@fireproof/core 0.3.12 → 0.3.14

package/dist/src/index.js

@@ -0,0 +1,1637 @@
+'use strict';
+
+var crypto = require('crypto');
+var Block = require('multiformats/block');
+var sha2 = require('multiformats/hashes/sha2');
+var dagcbor = require('@ipld/dag-cbor');
+var utils = require('prolly-trees/utils');
+var map = require('prolly-trees/map');
+var cache = require('prolly-trees/cache');
+var link = require('multiformats/link');
+var multiformats = require('multiformats');
+var car = require('@ipld/car');
+var cid = require('multiformats/cid');
+var CBW = require('@ipld/car/buffer-writer');
+var raw = require('multiformats/codecs/raw');
+var idb = require('idb');
+var cargoQueue = require('async/cargoQueue.js');
+var codec = require('encrypted-block');
+var cidSet = require('prolly-trees/cid-set');
+var buffer = require('buffer');
+var charwise = require('charwise');
+var dbIndex = require('prolly-trees/db-index');
+
+function _interopNamespaceDefault(e) {
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
+var Block__namespace = /*#__PURE__*/_interopNamespaceDefault(Block);
+var dagcbor__namespace = /*#__PURE__*/_interopNamespaceDefault(dagcbor);
+var CBW__namespace = /*#__PURE__*/_interopNamespaceDefault(CBW);
+var raw__namespace = /*#__PURE__*/_interopNamespaceDefault(raw);
+var codec__namespace = /*#__PURE__*/_interopNamespaceDefault(codec);
+
+async function advance(blocks, head, event) {
+  const events = new EventFetcher(blocks);
+  const headmap = new Map(head.map((cid) => [cid.toString(), cid]));
+  if (headmap.has(event.toString()))
+    return { head, cids: await events.all() };
+  let changed = false;
+  for (const cid of head) {
+    if (await contains(events, event, cid)) {
+      headmap.delete(cid.toString());
+      headmap.set(event.toString(), event);
+      changed = true;
+    }
+  }
+  if (changed) {
+    return { head: [...headmap.values()], cids: await events.all() };
+  }
+  for (const p of head) {
+    if (await contains(events, p, event)) {
+      return { head, cids: await events.all() };
+    }
+  }
+  return { head: head.concat(event), cids: await events.all() };
+}
+class EventBlock extends Block.Block {
+  /**
+   * @param {object} config
+   * @param {EventLink<T>} config.cid
+   * @param {Event} config.value
+   * @param {Uint8Array} config.bytes
+   */
+  constructor({ cid, value, bytes }) {
+    super({ cid, value, bytes });
+  }
+  /**
+   * @template T
+   * @param {T} data
+   * @param {EventLink<T>[]} [parents]
+   */
+  static create(data, parents) {
+    return encodeEventBlock({ data, parents: parents ?? [] });
+  }
+}
+class EventFetcher {
+  /** @param {import('./blockstore').TransactionBlockstore} blocks */
+  constructor(blocks) {
+    this._blocks = blocks;
+    this._cids = new utils.CIDCounter();
+    this._cache = /* @__PURE__ */ new Map();
+  }
+  /**
+   * @param {EventLink<T>} link
+   * @returns {Promise<EventBlockView<T>>}
+   */
+  async get(link) {
+    const slink = link.toString();
+    if (this._cache.has(slink))
+      return this._cache.get(slink);
+    const block = await this._blocks.get(link);
+    this._cids.add({ address: link });
+    if (!block)
+      throw new Error(`missing block: ${link}`);
+    const got = decodeEventBlock(block.bytes);
+    this._cache.set(slink, got);
+    return got;
+  }
+  async all() {
+    return this._cids.all();
+  }
+}
+async function encodeEventBlock(value) {
+  const { cid, bytes } = await Block.encode({ value, codec: dagcbor__namespace, hasher: sha2.sha256 });
+  return new Block.Block({ cid, value, bytes });
+}
+async function decodeEventBlock(bytes) {
+  const { cid, value } = await Block.decode({ bytes, codec: dagcbor__namespace, hasher: sha2.sha256 });
+  return new Block.Block({ cid, value, bytes });
+}
+async function contains(events, a, b) {
+  if (a.toString() === b.toString())
+    return true;
+  const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)]);
+  const links = [...aevent.parents];
+  while (links.length) {
+    const link = links.shift();
+    if (!link)
+      break;
+    if (link.toString() === b.toString())
+      return true;
+    if (bevent.parents.some((p) => link.toString() === p.toString()))
+      continue;
+    const { value: event } = await events.get(link);
+    links.push(...event.parents);
+  }
+  return false;
+}
+async function* vis$1(blocks, head, options = {}) {
+  const renderNodeLabel = options.renderNodeLabel ?? ((b) => b.value.data.value);
+  const events = new EventFetcher(blocks);
+  yield "digraph clock {";
+  yield '  node [shape=point fontname="Courier"]; head;';
+  const hevents = await Promise.all(head.map((link) => events.get(link)));
+  const links = [];
+  const nodes = /* @__PURE__ */ new Set();
+  for (const e of hevents) {
+    nodes.add(e.cid.toString());
+    yield `  node [shape=oval fontname="Courier"]; ${e.cid} [label="${renderNodeLabel(e)}"];`;
+    yield `  head -> ${e.cid};`;
+    for (const p of e.value.parents) {
+      yield `  ${e.cid} -> ${p};`;
+    }
+    links.push(...e.value.parents);
+  }
+  while (links.length) {
+    const link = links.shift();
+    if (!link)
+      break;
+    if (nodes.has(link.toString()))
+      continue;
+    nodes.add(link.toString());
+    const block = await events.get(link);
+    yield `  node [shape=oval]; ${link} [label="${renderNodeLabel(block)}" fontname="Courier"];`;
+    for (const p of block.value.parents) {
+      yield `  ${link} -> ${p};`;
+    }
+    links.push(...block.value.parents);
+  }
+  yield "}";
+}
+async function findEventsToSync(blocks, head) {
+  const events = new EventFetcher(blocks);
+  const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head);
+  const toSync = await asyncFilter(sorted, async (uks) => !await contains(events, ancestor, uks.cid));
+  return { cids: events.all(), events: toSync };
+}
+const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]));
+async function findCommonAncestorWithSortedEvents(events, children) {
+  const ancestor = await findCommonAncestor(events, children);
+  if (!ancestor) {
+    throw new Error("failed to find common ancestor event");
+  }
+  const sorted = await findSortedEvents(events, children, ancestor);
+  return { ancestor, sorted };
+}
+async function findCommonAncestor(events, children) {
+  if (!children.length)
+    return;
+  const candidates = children.map((c) => [c]);
+  while (true) {
+    let changed = false;
+    for (const c of candidates) {
+      const candidate = await findAncestorCandidate(events, c[c.length - 1]);
+      if (!candidate)
+        continue;
+      changed = true;
+      c.push(candidate);
+      const ancestor = findCommonString(candidates);
+      if (ancestor)
+        return ancestor;
+    }
+    if (!changed)
+      return;
+  }
+}
+async function findAncestorCandidate(events, root) {
+  const { value: event } = await events.get(root);
+  if (!event.parents.length)
+    return root;
+  return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents);
+}
+function findCommonString(arrays) {
+  arrays = arrays.map((a) => [...a]);
+  for (const arr of arrays) {
+    for (const item of arr) {
+      let matched = true;
+      for (const other of arrays) {
+        if (arr === other)
+          continue;
+        matched = other.some((i) => String(i) === String(item));
+        if (!matched)
+          break;
+      }
+      if (matched)
+        return item;
+    }
+  }
+}
+async function findSortedEvents(events, head, tail) {
+  const weights = /* @__PURE__ */ new Map();
+  const all = await Promise.all(head.map((h) => findEvents(events, h, tail)));
+  for (const arr of all) {
+    for (const { event, depth } of arr) {
+      const info = weights.get(event.cid.toString());
+      if (info) {
+        info.weight += depth;
+      } else {
+        weights.set(event.cid.toString(), { event, weight: depth });
+      }
+    }
+  }
+  const buckets = /* @__PURE__ */ new Map();
+  for (const { event, weight } of weights.values()) {
+    const bucket = buckets.get(weight);
+    if (bucket) {
+      bucket.push(event);
+    } else {
+      buckets.set(weight, [event]);
+    }
+  }
+  const sorted = Array.from(buckets).sort((a, b) => b[0] - a[0]).flatMap(([, es]) => es.sort((a, b) => String(a.cid) < String(b.cid) ? -1 : 1));
+  return sorted;
+}
+async function findEvents(events, start, end, depth = 0) {
+  const event = await events.get(start);
+  const acc = [{ event, depth }];
+  const { parents } = event.value;
+  if (parents.length === 1 && String(parents[0]) === String(end))
+    return acc;
+  const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)));
+  return acc.concat(...rest);
+}
+
+const createBlock = (bytes, cid) => Block.create({ cid, bytes, hasher: sha2.sha256, codec: codec__namespace });
+const encrypt = async function* ({ get, cids, hasher: hasher2, key, cache, chunker, root }) {
+  const set = /* @__PURE__ */ new Set();
+  let eroot;
+  for (const string of cids) {
+    const cid = multiformats.CID.parse(string);
+    const unencrypted = await get(cid);
+    const block2 = await Block.encode({ ...await codec__namespace.encrypt({ ...unencrypted, key }), codec: codec__namespace, hasher: hasher2 });
+    yield block2;
+    set.add(block2.cid.toString());
+    if (unencrypted.cid.equals(root))
+      eroot = block2.cid;
+  }
+  if (!eroot)
+    throw new Error("cids does not include root");
+  const list = [...set].map((s) => multiformats.CID.parse(s));
+  let last;
+  for await (const node of cidSet.create({ list, get, cache, chunker, hasher: hasher2, codec: dagcbor__namespace })) {
+    const block2 = await node.block;
+    yield block2;
+    last = block2;
+  }
+  const head = [eroot, last.cid];
+  const block = await Block.encode({ value: head, codec: dagcbor__namespace, hasher: hasher2 });
+  yield block;
+};
+const decrypt = async function* ({ root, get, key, cache, chunker, hasher: hasher2 }) {
+  const o = { ...await get(root), codec: dagcbor__namespace, hasher: hasher2 };
+  const decodedRoot = await Block.decode(o);
+  const { value: [eroot, tree] } = decodedRoot;
+  const rootBlock = await get(eroot);
+  const cidset = await cidSet.load({ cid: tree, get, cache, chunker, codec: codec__namespace, hasher: hasher2 });
+  const { result: nodes } = await cidset.getAllEntries();
+  const unwrap = async (eblock) => {
+    const { bytes, cid } = await codec__namespace.decrypt({ ...eblock, key }).catch((e) => {
+      console.log("ekey", e);
+      throw new Error("bad key: " + key.toString("hex"));
+    });
+    const block = await createBlock(bytes, cid);
+    return block;
+  };
+  const promises = [];
+  for (const { cid } of nodes) {
+    if (!rootBlock.cid.equals(cid))
+      promises.push(get(cid).then(unwrap));
+  }
+  yield* promises;
+  yield unwrap(rootBlock);
+};
+
+function rawSha1(b) {
+  let i = b.byteLength;
+  let bs = 0;
+  let A;
+  let B;
+  let C;
+  let D;
+  let G;
+  const H = Uint32Array.from([A = 1732584193, B = 4023233417, ~A, ~B, 3285377520]);
+  const W = new Uint32Array(80);
+  const nrWords = i / 4 + 2 | 15;
+  const words = new Uint32Array(nrWords + 1);
+  let j;
+  words[nrWords] = i * 8;
+  words[i >> 2] |= 128 << (~i << 3);
+  for (; i--; ) {
+    words[i >> 2] |= b[i] << (~i << 3);
+  }
+  for (A = H.slice(); bs < nrWords; bs += 16, A.set(H)) {
+    for (i = 0; i < 80; A[0] = (G = ((b = A[0]) << 5 | b >>> 27) + A[4] + (W[i] = i < 16 ? words[bs + i] : G << 1 | G >>> 31) + 1518500249, B = A[1], C = A[2], D = A[3], G + ((j = i / 5 >> 2) ? j !== 2 ? (B ^ C ^ D) + (j & 2 ? 1876969533 : 341275144) : (B & C | B & D | C & D) + 882459459 : B & C | ~B & D)), A[1] = b, A[2] = B << 30 | B >>> 2, A[3] = C, A[4] = D, ++i) {
+      G = W[i - 3] ^ W[i - 8] ^ W[i - 14] ^ W[i - 16];
+    }
+    for (i = 5; i; )
+      H[--i] = H[i] + A[i];
+  }
+  return new Uint8Array(H.buffer, H.byteOffset, H.byteLength);
+}
+
+const chunker = utils.bf(3);
+const NO_ENCRYPT = process.env?.NO_ENCRYPT;
+class Valet {
+  idb = null;
+  name = null;
+  uploadQueue = null;
+  alreadyEnqueued = /* @__PURE__ */ new Set();
+  keyMaterial = null;
+  keyId = "null";
+  /**
+   * Function installed by the database to upload car files
+   * @type {null|function(string, Uint8Array):Promise<void>}
+   */
+  uploadFunction = null;
+  constructor(name = "default", keyMaterial) {
+    this.name = name;
+    this.setKeyMaterial(keyMaterial);
+    this.uploadQueue = cargoQueue(async (tasks, callback) => {
+      console.log(
+        "queue worker",
+        tasks.length,
+        tasks.reduce((acc, t) => acc + t.value.length, 0)
+      );
+      if (this.uploadFunction) {
+        return await this.withDB(async (db) => {
+          for (const task of tasks) {
+            await this.uploadFunction(task.carCid, task.value);
+            const carMeta = await db.get("cidToCar", task.carCid);
+            delete carMeta.pending;
+            await db.put("cidToCar", carMeta);
+          }
+        });
+      }
+      callback();
+    });
+    this.uploadQueue.drain(async () => {
+      return await this.withDB(async (db) => {
+        const carKeys = (await db.getAllFromIndex("cidToCar", "pending")).map((c) => c.car);
+        for (const carKey of carKeys) {
+          await this.uploadFunction(carKey, await db.get("cars", carKey));
+          const carMeta = await db.get("cidToCar", carKey);
+          delete carMeta.pending;
+          await db.put("cidToCar", carMeta);
+        }
+      });
+    });
+  }
+  getKeyMaterial() {
+    return this.keyMaterial;
+  }
+  setKeyMaterial(km) {
+    if (km && !NO_ENCRYPT) {
+      const hex = Uint8Array.from(buffer.Buffer.from(km, "hex"));
+      this.keyMaterial = km;
+      const hash = rawSha1(hex);
+      this.keyId = buffer.Buffer.from(hash).toString("hex");
+    } else {
+      this.keyMaterial = null;
+      this.keyId = "null";
+    }
+  }
+  /**
+   * Group the blocks into a car and write it to the valet.
+   * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
+   * @param {Set<string>} cids
+   * @returns {Promise<void>}
+   * @memberof Valet
+   */
+  async writeTransaction(innerBlockstore, cids) {
+    if (innerBlockstore.lastCid) {
+      if (this.keyMaterial) {
+        const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial);
+        await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
+      } else {
+        const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
+        await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
+      }
+    }
+  }
+  withDB = async (dbWorkFun) => {
+    if (!this.idb) {
+      this.idb = await idb.openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
+        upgrade(db, oldVersion, newVersion, transaction) {
+          if (oldVersion < 1) {
+            db.createObjectStore("cars");
+            const cidToCar = db.createObjectStore("cidToCar", { keyPath: "car" });
+            cidToCar.createIndex("cids", "cids", { multiEntry: true });
+          }
+          if (oldVersion < 2) {
+            const cidToCar = transaction.objectStore("cidToCar");
+            cidToCar.createIndex("pending", "pending");
+          }
+        }
+      });
+    }
+    return await dbWorkFun(this.idb);
+  };
+  /**
+   *
+   * @param {string} carCid
+   * @param {*} value
+   */
+  async parkCar(carCid, value, cids) {
+    await this.withDB(async (db) => {
+      const tx = db.transaction(["cars", "cidToCar"], "readwrite");
+      await tx.objectStore("cars").put(value, carCid);
+      await tx.objectStore("cidToCar").put({ pending: "y", car: carCid, cids: Array.from(cids) });
+      return await tx.done;
+    });
+    if (this.uploadFunction) {
+      if (this.alreadyEnqueued.has(carCid)) {
+        return;
+      }
+      this.uploadQueue.push({ carCid, value });
+      this.alreadyEnqueued.add(carCid);
+    }
+  }
+  remoteBlockFunction = null;
+  async getBlock(dataCID) {
+    return await this.withDB(async (db) => {
+      const tx = db.transaction(["cars", "cidToCar"], "readonly");
+      const indexResp = await tx.objectStore("cidToCar").index("cids").get(dataCID);
+      const carCid = indexResp?.car;
+      if (!carCid) {
+        throw new Error("Missing block: " + dataCID);
+      }
+      const carBytes = await tx.objectStore("cars").get(carCid);
+      const reader = await car.CarReader.fromBytes(carBytes);
+      if (this.keyMaterial) {
+        const roots = await reader.getRoots();
+        const readerGetWithCodec = async (cid) => {
+          const got = await reader.get(cid);
+          let useCodec = codec__namespace;
+          if (cid.toString().indexOf("bafy") === 0) {
+            useCodec = dagcbor__namespace;
+          }
+          const decoded = await Block__namespace.decode({
+            ...got,
+            codec: useCodec,
+            hasher: sha2.sha256
+          });
+          return decoded;
+        };
+        const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
+        const block = blocks.find((b) => b.cid.toString() === dataCID);
+        if (block) {
+          return block.bytes;
+        }
+      } else {
+        const gotBlock = await reader.get(cid.CID.parse(dataCID));
+        if (gotBlock) {
+          return gotBlock.bytes;
+        }
+      }
+    });
+  }
+}
+const blocksToCarBlock = async (lastCid, blocks) => {
+  let size = 0;
+  const headerSize = CBW__namespace.headerLength({ roots: [lastCid] });
+  size += headerSize;
+  if (!Array.isArray(blocks)) {
+    blocks = Array.from(blocks.entries());
+  }
+  for (const { cid, bytes } of blocks) {
+    size += CBW__namespace.blockLength({ cid, bytes });
+  }
+  const buffer = new Uint8Array(size);
+  const writer = await CBW__namespace.createWriter(buffer, { headerSize });
+  writer.addRoot(lastCid);
+  for (const { cid, bytes } of blocks) {
+    writer.write({ cid, bytes });
+  }
+  await writer.close();
+  return await Block__namespace.encode({ value: writer.bytes, hasher: sha2.sha256, codec: raw__namespace });
+};
+const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
+  const encryptionKey = buffer.Buffer.from(keyMaterial, "hex");
+  const encryptedBlocks = [];
+  const theCids = [];
+  for (const { cid } of blocks.entries()) {
+    theCids.push(cid.toString());
+  }
+  let last;
+  for await (const block of encrypt({
+    cids: theCids,
+    get: async (cid) => blocks.get(cid),
+    // maybe we can just use blocks.get
+    key: encryptionKey,
+    hasher: sha2.sha256,
+    chunker,
+    cache: cache.nocache,
+    // codec: dagcbor, // should be crypto?
+    root: innerBlockStoreClockRootCid
+  })) {
+    encryptedBlocks.push(block);
+    last = block;
+  }
+  const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+  return encryptedCar;
+};
+const memoizeDecryptedCarBlocks = /* @__PURE__ */ new Map();
+const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+  if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+    return memoizeDecryptedCarBlocks.get(cid.toString());
+  } else {
+    const blocksPromise = (async () => {
+      const decryptionKey = buffer.Buffer.from(keyMaterial, "hex");
+      const cids = /* @__PURE__ */ new Set();
+      const decryptedBlocks = [];
+      for await (const block of decrypt({
+        root: cid,
+        get,
+        key: decryptionKey,
+        chunker,
+        hasher: sha2.sha256,
+        cache: cache.nocache
+        // codec: dagcbor
+      })) {
+        decryptedBlocks.push(block);
+        cids.add(block.cid.toString());
+      }
+      return { blocks: decryptedBlocks, cids };
+    })();
+    memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+    return blocksPromise;
+  }
+};
+
+const husherMap = /* @__PURE__ */ new Map();
+const husher = (id, workFn) => {
+  if (!husherMap.has(id)) {
+    husherMap.set(
+      id,
+      workFn().finally(() => setTimeout(() => husherMap.delete(id), 100))
+    );
+  }
+  return husherMap.get(id);
+};
+class TransactionBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  committedBlocks = /* @__PURE__ */ new Map();
+  valet = null;
+  instanceId = "blkz." + Math.random().toString(36).substring(2, 4);
+  inflightTransactions = /* @__PURE__ */ new Set();
+  constructor(name, encryptionKey) {
+    this.valet = new Valet(name, encryptionKey);
+  }
+  /**
+   * Get a block from the store.
+   *
+   * @param {import('./link').AnyLink} cid
+   * @returns {Promise<AnyBlock | undefined>}
+   */
+  async get(cid) {
+    const key = cid.toString();
+    const bytes = await Promise.any([this.transactionsGet(key), this.committedGet(key)]).catch((e) => {
+      return this.networkGet(key);
+    });
+    if (!bytes)
+      throw new Error("Missing block: " + key);
+    return { cid, bytes };
+  }
+  // this iterates over the in-flight transactions
+  // and returns the first matching block it finds
+  async transactionsGet(key) {
+    for (const transaction of this.inflightTransactions) {
+      const got = await transaction.get(key);
+      if (got && got.bytes)
+        return got.bytes;
+    }
+    throw new Error("Missing block: " + key);
+  }
+  async committedGet(key) {
+    const old = this.committedBlocks.get(key);
+    if (old)
+      return old;
+    const got = await this.valet.getBlock(key);
+    this.committedBlocks.set(key, got);
+    return got;
+  }
+  async clearCommittedCache() {
+    this.committedBlocks.clear();
+  }
+  async networkGet(key) {
+    if (this.valet.remoteBlockFunction) {
+      const value = await husher(key, async () => await this.valet.remoteBlockFunction(key));
+      if (value) {
+        doTransaction("networkGot: " + key, this, async (innerBlockstore) => {
+          await innerBlockstore.put(multiformats.CID.parse(key), value);
+        });
+        return value;
+      }
+    } else {
+      return false;
+    }
+  }
+  /**
+   * Add a block to the store. Usually bound to a transaction by a closure.
+   * It sets the lastCid property to the CID of the block that was put.
+   * This is used by the transaction as the head of the car when written to the valet.
+   * We don't have to worry about which transaction we are when we are here because
+   * we are the transactionBlockstore.
+   *
+   * @param {import('./link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  put(cid, bytes) {
+    throw new Error("use a transaction to put");
+  }
+  /**
+   * Iterate over all blocks in the store.
+   *
+   * @yields {AnyBlock}
+   * @returns {AsyncGenerator<AnyBlock>}
+   */
+  // * entries () {
+  //   // needs transaction blocks?
+  //   // for (const [str, bytes] of this.blocks) {
+  //   //   yield { cid: parse(str), bytes }
+  //   // }
+  //   for (const [str, bytes] of this.committedBlocks) {
+  //     yield { cid: parse(str), bytes }
+  //   }
+  // }
+  /**
+   * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
+   * Returns the blocks to read and write during the transaction.
+   * @returns {InnerBlockstore}
+   * @memberof TransactionBlockstore
+   */
+  begin(label = "") {
+    const innerTransactionBlockstore = new InnerBlockstore(label, this);
+    this.inflightTransactions.add(innerTransactionBlockstore);
+    return innerTransactionBlockstore;
+  }
+  /**
+   * Commit the transaction. Writes the blocks to the store.
+   * @returns {Promise<void>}
+   * @memberof TransactionBlockstore
+   */
+  async commit(innerBlockstore) {
+    await this.doCommit(innerBlockstore);
+  }
+  // first get the transaction blockstore from the map of transaction blockstores
+  // then copy it to committedBlocks
+  // then write the transaction blockstore to a car
+  // then write the car to the valet
+  // then remove the transaction blockstore from the map of transaction blockstores
+  doCommit = async (innerBlockstore) => {
+    const cids = /* @__PURE__ */ new Set();
+    for (const { cid, bytes } of innerBlockstore.entries()) {
+      const stringCid = cid.toString();
+      if (this.committedBlocks.has(stringCid)) ; else {
+        this.committedBlocks.set(stringCid, bytes);
+        cids.add(stringCid);
+      }
+    }
+    if (cids.size > 0) {
+      await this.valet.writeTransaction(innerBlockstore, cids);
+    }
+  };
+  /**
+   * Retire the transaction. Clears the uncommited blocks.
+   * @returns {void}
+   * @memberof TransactionBlockstore
+   */
+  retire(innerBlockstore) {
+    this.inflightTransactions.delete(innerBlockstore);
+  }
+}
+const doTransaction = async (label, blockstore, doFun) => {
+  if (!blockstore.commit)
+    return await doFun(blockstore);
+  const innerBlockstore = blockstore.begin(label);
+  try {
+    const result = await doFun(innerBlockstore);
+    await blockstore.commit(innerBlockstore);
+    return result;
+  } catch (e) {
+    console.error(`Transaction ${label} failed`, e, e.stack);
+    throw e;
+  } finally {
+    blockstore.retire(innerBlockstore);
+  }
+};
+class InnerBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  blocks = /* @__PURE__ */ new Map();
+  lastCid = null;
+  label = "";
+  parentBlockstore = null;
+  constructor(label, parentBlockstore) {
+    this.label = label;
+    this.parentBlockstore = parentBlockstore;
+  }
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @returns {Promise<AnyBlock | undefined>}
+   */
+  async get(cid) {
+    const key = cid.toString();
+    let bytes = this.blocks.get(key);
+    if (bytes) {
+      return { cid, bytes };
+    }
+    bytes = await this.parentBlockstore.committedGet(key);
+    if (bytes) {
+      return { cid, bytes };
+    }
+  }
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  async put(cid, bytes) {
+    this.blocks.set(cid.toString(), bytes);
+    this.lastCid = cid;
+  }
+  *entries() {
+    for (const [str, bytes] of this.blocks) {
+      yield { cid: link.parse(str), bytes };
+    }
+  }
+}
+
+const blockOpts = { cache: cache.nocache, chunker: utils.bf(3), codec: dagcbor__namespace, hasher: sha2.sha256, compare: utils.simpleCompare };
+const withLog = async (label, fn) => {
+  const resp = await fn();
+  return resp;
+};
+const makeGetBlock = (blocks) => {
+  const getBlockFn = async (address) => {
+    const { cid, bytes } = await withLog(address, () => blocks.get(address));
+    return Block.create({ cid, bytes, hasher: sha2.sha256, codec: dagcbor__namespace });
+  };
+  return {
+    // cids,
+    getBlock: getBlockFn
+  };
+};
+async function createAndSaveNewEvent({
+  inBlocks,
+  bigPut,
+  root: root2,
+  event: inEvent,
+  head,
+  additions,
+  removals = []
+}) {
+  let cids;
+  const { key, value, del } = inEvent;
+  const data = {
+    root: root2 ? {
+      cid: root2.cid,
+      bytes: root2.bytes,
+      // can we remove this?
+      value: root2.value
+      // can we remove this?
+    } : null,
+    key
+  };
+  if (del) {
+    data.value = null;
+    data.type = "del";
+  } else {
+    data.value = value;
+    data.type = "put";
+  }
+  const event = await EventBlock.create(data, head);
+  bigPut(event);
+  ({ head, cids } = await advance(inBlocks, head, event.cid));
+  return {
+    root: root2,
+    additions,
+    removals,
+    head,
+    clockCIDs: cids,
+    event
+  };
+}
+const makeGetAndPutBlock = (inBlocks) => {
+  const { getBlock, cids } = makeGetBlock(inBlocks);
+  const put2 = inBlocks.put.bind(inBlocks);
+  const bigPut = async (block, additions) => {
+    const { cid, bytes } = block;
+    put2(cid, bytes);
+    if (additions) {
+      additions.set(cid.toString(), block);
+    }
+  };
+  return { getBlock, bigPut, blocks: inBlocks, cids };
+};
+const bulkFromEvents = (sorted, event) => {
+  if (event) {
+    const update = { value: { data: { key: event.key } } };
+    if (event.del) {
+      update.value.data.type = "del";
+    } else {
+      update.value.data.type = "put";
+      update.value.data.value = event.value;
+    }
+    sorted.push(update);
+  }
+  const bulk = /* @__PURE__ */ new Map();
+  for (const { value: event2 } of sorted) {
+    const {
+      data: { type, value, key }
+    } = event2;
+    const bulkEvent = type === "put" ? { key, value } : { key, del: true };
+    bulk.set(bulkEvent.key, bulkEvent);
+  }
+  return Array.from(bulk.values());
+};
+const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
+  const event = await events.get(ancestor);
+  const { root: root2 } = event.value.data;
+  if (root2) {
+    return map.load({ cid: root2.cid, get: getBlock, ...blockOpts });
+  } else {
+    return null;
+  }
+};
+const doProllyBulk = async (inBlocks, head, event) => {
+  const { getBlock, blocks } = makeGetAndPutBlock(inBlocks);
+  let bulkSorted = [];
+  let prollyRootNode = null;
+  if (head.length) {
+    const events = new EventFetcher(blocks);
+    const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head);
+    bulkSorted = sorted;
+    prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock);
+  }
+  const bulkOperations = bulkFromEvents(bulkSorted, event);
+  if (!prollyRootNode) {
+    let root2;
+    const newBlocks = [];
+    if (bulkOperations.every((op) => op.del)) {
+      return { root: null, blocks: [] };
+    }
+    for await (const node of map.create({ get: getBlock, list: bulkOperations, ...blockOpts })) {
+      root2 = await node.block;
+      newBlocks.push(root2);
+    }
+    return { root: root2, blocks: newBlocks };
+  } else {
+    return await prollyRootNode.bulk(bulkOperations);
+  }
+};
+async function put(inBlocks, head, event, options) {
+  const { bigPut } = makeGetAndPutBlock(inBlocks);
+  if (!head.length) {
+    const additions = /* @__PURE__ */ new Map();
+    const { root: root2, blocks } = await doProllyBulk(inBlocks, head, event);
+    for (const b of blocks) {
+      bigPut(b, additions);
+    }
+    return createAndSaveNewEvent({ inBlocks, bigPut, root: root2, event, head, additions: Array.from(additions.values()) });
+  }
+  const { root: newProllyRootNode, blocks: newBlocks } = await doProllyBulk(inBlocks, head, event);
+  if (!newProllyRootNode) {
+    return createAndSaveNewEvent({
+      inBlocks,
+      bigPut,
+      root: null,
+      event,
+      head,
+      additions: []
+    });
+  } else {
+    const prollyRootBlock = await newProllyRootNode.block;
+    const additions = /* @__PURE__ */ new Map();
+    bigPut(prollyRootBlock, additions);
+    for (const nb of newBlocks) {
+      bigPut(nb, additions);
+    }
+    return createAndSaveNewEvent({
+      inBlocks,
+      bigPut,
+      root: prollyRootBlock,
+      event,
+      head,
+      additions: Array.from(additions.values())
+      /*, todo? Array.from(removals.values()) */
+    });
+  }
+}
+async function root(inBlocks, head) {
+  if (!head.length) {
+    throw new Error("no head");
+  }
+  const { root: newProllyRootNode, blocks: newBlocks, cids } = await doProllyBulk(inBlocks, head);
+  await doTransaction("root", inBlocks, async (transactionBlockstore) => {
+    const { bigPut } = makeGetAndPutBlock(transactionBlockstore);
+    for (const nb of newBlocks) {
+      bigPut(nb);
+    }
+  });
+  return { cids, node: newProllyRootNode };
+}
+async function eventsSince(blocks, head, since) {
+  if (!head.length) {
+    throw new Error("no head");
+  }
+  const sinceHead = [...since, ...head];
+  const { cids, events: unknownSorted3 } = await findEventsToSync(blocks, sinceHead);
+  return { clockCIDs: cids, result: unknownSorted3.map(({ value: { data } }) => data) };
+}
+async function getAll(blocks, head) {
+  if (!head.length) {
+    return { clockCIDs: new utils.CIDCounter(), cids: new utils.CIDCounter(), result: [] };
+  }
+  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head);
+  if (!prollyRootNode) {
+    return { clockCIDs, cids: new utils.CIDCounter(), result: [] };
+  }
+  const { result, cids } = await prollyRootNode.getAllEntries();
+  return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) };
+}
+async function get(blocks, head, key) {
+  if (!head.length) {
+    return { cids: new utils.CIDCounter(), result: null };
+  }
+  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head);
+  if (!prollyRootNode) {
+    return { clockCIDs, cids: new utils.CIDCounter(), result: null };
+  }
+  const { result, cids } = await prollyRootNode.get(key);
+  return { result, cids, clockCIDs };
+}
+async function* vis(blocks, head) {
+  if (!head.length) {
+    return { cids: new utils.CIDCounter(), result: null };
+  }
+  const { node: prollyRootNode, cids } = await root(blocks, head);
+  const lines = [];
+  for await (const line of prollyRootNode.vis()) {
+    yield line;
+    lines.push(line);
+  }
+  return { vis: lines.join("\n"), cids };
+}
+async function visMerkleTree(blocks, head) {
+  if (!head.length) {
+    return { cids: new utils.CIDCounter(), result: null };
+  }
+  const { node: prollyRootNode, cids } = await root(blocks, head);
+  const lines = [];
+  for await (const line of prollyRootNode.vis()) {
+    lines.push(line);
+  }
+  return { vis: lines.join("\n"), cids };
+}
+async function visMerkleClock(blocks, head) {
+  const lines = [];
+  for await (const line of vis$1(blocks, head)) {
+    lines.push(line);
+  }
+  return { vis: lines.join("\n") };
+}
+
+class Fireproof {
+  listeners = /* @__PURE__ */ new Set();
+  /**
+   * @function storage
+   * @memberof Fireproof
+   * Creates a new Fireproof instance with default storage settings
+   * Most apps should use this and not worry about the details.
+   * @static
+   * @returns {Fireproof} - a new Fireproof instance
+   */
+  static storage = (name = "global") => {
+    const instanceKey = crypto.randomBytes(32).toString("hex");
+    return new Fireproof(new TransactionBlockstore(name, instanceKey), [], { name });
+  };
+  constructor(blocks, clock, config, authCtx = {}) {
+    this.name = config?.name || "global";
+    this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`;
+    this.blocks = blocks;
+    this.clock = clock;
+    this.config = config;
+    this.authCtx = authCtx;
+    this.indexes = /* @__PURE__ */ new Map();
+  }
+  /**
+   * Renders the Fireproof instance as a JSON object.
+   * @returns {Object} - The JSON representation of the Fireproof instance. Includes clock heads for the database and its indexes.
+   * @memberof Fireproof
+   * @instance
+   */
+  toJSON() {
+    return {
+      clock: this.clockToJSON(),
+      name: this.name,
+      key: this.blocks.valet.getKeyMaterial(),
+      indexes: [...this.indexes.values()].map((index) => index.toJSON())
+    };
+  }
+  /**
+   * Returns the Merkle clock heads for the Fireproof instance.
+   * @returns {string[]} - The Merkle clock heads for the Fireproof instance.
+   * @memberof Fireproof
+   * @instance
+   */
+  clockToJSON() {
+    return this.clock.map((cid) => cid.toString());
+  }
+  hydrate({ clock, name, key }) {
+    this.name = name;
+    this.clock = clock;
+    this.blocks.valet.setKeyMaterial(key);
+    this.indexBlocks = null;
+  }
+  /**
+   * Triggers a notification to all listeners
+   * of the Fireproof instance so they can repaint UI, etc.
+   * @returns {Promise<void>}
+   * @memberof Fireproof
+   * @instance
+   */
+  async notifyReset() {
+    await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() });
+  }
+  // used be indexes etc to notify database listeners of new availability
+  async notifyExternal(source = "unknown") {
+    await this.notifyListeners({ _external: source, _clock: this.clockToJSON() });
+  }
+  /**
+   * Returns the changes made to the Fireproof instance since the specified event.
+   * @function changesSince
+   * @param {CID[]} [event] - The clock head to retrieve changes since. If null or undefined, retrieves all changes.
+   * @returns {Promise<{rows : Object[], clock: CID[], proof: {}}>} An object containing the rows and the head of the instance's clock.
+   * @memberof Fireproof
+   * @instance
+   */
+  async changesSince(event) {
+    let rows, dataCIDs, clockCIDs;
+    if (event) {
+      const resp = await eventsSince(this.blocks, this.clock, event);
+      const docsMap = /* @__PURE__ */ new Map();
+      for (const { key, type, value } of resp.result.map(decodeEvent)) {
+        if (type === "del") {
+          docsMap.set(key, { key, del: true });
+        } else {
+          docsMap.set(key, { key, value });
+        }
+      }
+      rows = Array.from(docsMap.values());
+      clockCIDs = resp.clockCIDs;
+    } else {
+      const allResp = await getAll(this.blocks, this.clock);
+      rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }));
+      dataCIDs = allResp.cids;
+    }
+    return {
+      rows,
+      clock: this.clockToJSON(),
+      proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
+    };
+  }
+  async allDocuments() {
+    const allResp = await getAll(this.blocks, this.clock);
+    const rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value })).map(({ key, value }) => ({ key, value: { _id: key, ...value } }));
+    return {
+      rows,
+      clock: this.clockToJSON(),
+      proof: await cidsToProof(allResp.cids)
+    };
+  }
+  /**
+   * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
+   *
+   * @param {Object} doc - The document to validate.
+   * @returns {Promise<void>}
+   * @throws {Error} - Throws an error if the document is invalid.
+   * @memberof Fireproof
+   * @instance
+   */
+  async runValidation(doc) {
+    if (this.config && this.config.validateChange) {
+      const oldDoc = await this.get(doc._id).then((doc2) => doc2).catch(() => ({}));
+      this.config.validateChange(doc, oldDoc, this.authCtx);
+    }
+  }
+  /**
+   * Retrieves the document with the specified ID from the database
+   *
+   * @param {string} key - the ID of the document to retrieve
+   * @param {Object} [opts] - options
+   * @returns {Promise<{_id: string}>} - the document with the specified ID
+   * @memberof Fireproof
+   * @instance
+   */
+  async get(key, opts = {}) {
+    const clock = opts.clock || this.clock;
+    const resp = await get(this.blocks, clock, charwise.encode(key));
+    if (!resp || resp.result === null) {
+      throw new Error("Not found");
+    }
+    const doc = resp.result;
+    if (opts.mvcc === true) {
+      doc._clock = this.clockToJSON();
+    }
+    doc._proof = {
+      data: await cidsToProof(resp.cids),
+      clock: this.clockToJSON()
+    };
+    doc._id = key;
+    return doc;
+  }
+  /**
+   * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
+   *
+   * @param {Object} doc - the document to be added
+   * @param {string} doc._id - the document ID. If not provided, a random ID will be generated.
+   * @param {CID[]} doc._clock - the document ID. If not provided, a random ID will be generated.
+   * @param {Proof} doc._proof - CIDs referenced by the update
+   * @returns {Promise<{ id: string, clock: CID[]  }>} - The result of adding the document to the database
+   * @memberof Fireproof
+   * @instance
+   */
+  async put({ _id, _proof, ...doc }) {
+    const id = _id || "f" + Math.random().toString(36).slice(2);
+    await this.runValidation({ _id: id, ...doc });
+    return await this.putToProllyTree({ key: id, value: doc }, doc._clock);
+  }
+  /**
+   * Deletes a document from the database
+   * @param {string | any} docOrId - the document ID
+   * @returns {Promise<{ id: string, clock: CID[] }>} - The result of deleting the document from the database
+   * @memberof Fireproof
+   * @instance
+   */
+  async del(docOrId) {
+    let id;
+    let clock = null;
+    if (docOrId._id) {
+      id = docOrId._id;
+      clock = docOrId._clock;
+    } else {
+      id = docOrId;
+    }
+    await this.runValidation({ _id: id, _deleted: true });
+    return await this.putToProllyTree({ key: id, del: true }, clock);
+  }
+  /**
+   * Updates the underlying storage with the specified event.
+   * @private
+   * @param {{del?: true, key : string, value?: any}} decodedEvent - the event to add
+   * @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
+   */
+  async putToProllyTree(decodedEvent, clock = null) {
+    const event = encodeEvent(decodedEvent);
+    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clockToJSON())) {
+      const resp = await eventsSince(this.blocks, this.clock, event.value._clock);
+      const missedChange = resp.result.find(({ key }) => key === event.key);
+      if (missedChange) {
+        throw new Error("MVCC conflict, document is changed, please reload the document and try again.");
+      }
+    }
+    const result = await doTransaction(
+      "putToProllyTree",
+      this.blocks,
+      async (blocks) => await put(blocks, this.clock, event)
+    );
+    if (!result) {
+      console.error("failed", event);
+      throw new Error("failed to put at storage layer");
+    }
+    this.clock = result.head;
+    await this.notifyListeners([decodedEvent]);
+    return {
+      id: decodedEvent.key,
+      clock: this.clockToJSON(),
+      proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
+    };
+  }
+  //   /**
+  //    * Advances the clock to the specified event and updates the root CID
+  //    *   Will be used by replication
+  //    */
+  //     async advance (event) {
+  //       this.clock = await advance(this.blocks, this.clock, event)
+  //       this.rootCid = await root(this.blocks, this.clock)
+  //       return this.clock
+  //     }
+  async *vis() {
+    return yield* vis(this.blocks, this.clock);
+  }
+  async visTree() {
+    return await visMerkleTree(this.blocks, this.clock);
+  }
+  async visClock() {
+    return await visMerkleClock(this.blocks, this.clock);
+  }
+  /**
+   * Registers a Listener to be called when the Fireproof instance's clock is updated.
+   * Recieves live changes from the database after they are committed.
+   * @param {Function} listener - The listener to be called when the clock is updated.
+   * @returns {Function} - A function that can be called to unregister the listener.
+   * @memberof Fireproof
+   */
+  registerListener(listener) {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+  async notifyListeners(changes) {
+    for (const listener of this.listeners) {
+      await listener(changes);
+    }
+  }
+  setCarUploader(carUploaderFn) {
+    this.blocks.valet.uploadFunction = carUploaderFn;
+  }
+  setRemoteBlockReader(remoteBlockReaderFn) {
+    this.blocks.valet.remoteBlockFunction = remoteBlockReaderFn;
+  }
+}
+async function cidsToProof(cids) {
+  if (!cids || !cids.all)
+    return [];
+  const all = await cids.all();
+  return [...all].map((cid) => cid.toString());
+}
+function decodeEvent(event) {
+  const decodedKey = charwise.decode(event.key);
+  return { ...event, key: decodedKey };
+}
+function encodeEvent(event) {
+  if (!(event && event.key))
+    return;
+  const encodedKey = charwise.encode(event.key);
+  return { ...event, key: encodedKey };
+}
+
+const compare = (a, b) => {
+  const [aKey, aRef] = a;
+  const [bKey, bRef] = b;
+  const comp = utils.simpleCompare(aKey, bKey);
+  if (comp !== 0)
+    return comp;
+  return refCompare(aRef, bRef);
+};
+const refCompare = (aRef, bRef) => {
+  if (Number.isNaN(aRef))
+    return -1;
+  if (Number.isNaN(bRef))
+    throw new Error("ref may not be Infinity or NaN");
+  if (aRef === Infinity)
+    return 1;
+  return utils.simpleCompare(aRef, bRef);
+};
+const dbIndexOpts = { cache: cache.nocache, chunker: utils.bf(3), codec: dagcbor__namespace, hasher: sha2.sha256, compare };
+const idIndexOpts = { cache: cache.nocache, chunker: utils.bf(3), codec: dagcbor__namespace, hasher: sha2.sha256, compare: utils.simpleCompare };
+const makeDoc = ({ key, value }) => ({ _id: key, ...value });
+const indexEntriesForChanges = (changes, mapFn) => {
+  const indexEntries = [];
+  changes.forEach(({ key, value, del }) => {
+    if (del || !value)
+      return;
+    mapFn(makeDoc({ key, value }), (k, v) => {
+      if (typeof v === "undefined" || typeof k === "undefined")
+        return;
+      indexEntries.push({
+        key: [charwise.encode(k), key],
+        value: v
+      });
+    });
+  });
+  return indexEntries;
+};
+class DbIndex {
+  constructor(database, mapFn, clock, opts = {}) {
+    this.database = database;
+    if (!database.indexBlocks) {
+      database.indexBlocks = new TransactionBlockstore(database.name + ".indexes", database.blocks.valet.getKeyMaterial());
+    }
+    if (typeof mapFn === "string") {
+      this.mapFnString = mapFn;
+    } else {
+      this.mapFn = mapFn;
+      this.mapFnString = mapFn.toString();
+    }
+    this.name = opts.name || this.makeName();
+    this.indexById = { root: null, cid: null };
+    this.indexByKey = { root: null, cid: null };
+    this.dbHead = null;
+    if (clock) {
+      this.indexById.cid = clock.byId;
+      this.indexByKey.cid = clock.byKey;
+      this.dbHead = clock.db;
+    }
+    this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`;
+    this.updateIndexPromise = null;
+    if (!opts.temporary) {
+      DbIndex.registerWithDatabase(this, this.database);
+    }
+  }
+  makeName() {
+    const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
+    const matches = Array.from(this.mapFnString.matchAll(regex), (match) => match[1].trim());
+    return matches[1];
+  }
+  static registerWithDatabase(inIndex, database) {
+    if (!database.indexes.has(inIndex.mapFnString)) {
+      database.indexes.set(inIndex.mapFnString, inIndex);
+    } else {
+      const existingIndex = database.indexes.get(inIndex.mapFnString);
+      if (existingIndex.mapFn) {
+        existingIndex.dbHead = inIndex.dbHead;
+        existingIndex.indexById.cid = inIndex.indexById.cid;
+        existingIndex.indexByKey.cid = inIndex.indexByKey.cid;
+      } else {
+        inIndex.dbHead = existingIndex.dbHead;
+        inIndex.indexById.cid = existingIndex.indexById.cid;
+        inIndex.indexByKey.cid = existingIndex.indexByKey.cid;
+        database.indexes.set(inIndex.mapFnString, inIndex);
+      }
+    }
+  }
+  toJSON() {
+    const indexJson = { name: this.name, code: this.mapFnString, clock: { db: null, byId: null, byKey: null } };
+    indexJson.clock.db = this.dbHead?.map((cid) => cid.toString());
+    indexJson.clock.byId = this.indexById.cid?.toString();
+    indexJson.clock.byKey = this.indexByKey.cid?.toString();
+    return indexJson;
+  }
+  static fromJSON(database, { code, clock, name }) {
+    return new DbIndex(database, code, clock, { name });
+  }
+  /**
+   * JSDoc for Query type.
+   * @typedef {Object} DbQuery
+   * @property {string[]} [range] - The range to query.
+   * @memberof DbIndex
+   */
+  /**
+   * Query object can have {range}
+   * @param {DbQuery} query - the query range to use
+   * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
+   * @memberof DbIndex
+   * @instance
+   */
+  async query(query, update = true) {
+    update && await this.updateIndex(this.database.indexBlocks);
+    const response = await doIndexQuery(this.database.indexBlocks, this.indexByKey, query);
+    return {
+      proof: { index: await cidsToProof(response.cids) },
+      rows: response.result.map(({ id, key, row }) => {
+        return { id, key: charwise.decode(key), value: row };
+      })
+    };
+  }
+  /**
+   * Update the DbIndex with the latest changes
+   * @private
+   * @returns {Promise<void>}
+   */
+  async updateIndex(blocks) {
+    if (this.updateIndexPromise)
+      return this.updateIndexPromise;
+    this.updateIndexPromise = this.innerUpdateIndex(blocks);
+    this.updateIndexPromise.finally(() => {
+      this.updateIndexPromise = null;
+    });
+    return this.updateIndexPromise;
+  }
+  async innerUpdateIndex(inBlocks) {
+    const result = await this.database.changesSince(this.dbHead);
+    if (result.rows.length === 0) {
+      this.dbHead = result.clock;
+      return;
+    }
+    await doTransaction("updateIndex", inBlocks, async (blocks) => {
+      let oldIndexEntries = [];
+      let removeByIdIndexEntries = [];
+      await loadIndex(blocks, this.indexById, idIndexOpts);
+      await loadIndex(blocks, this.indexByKey, dbIndexOpts);
+      if (this.dbHead) {
+        const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key));
+        oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }));
+        removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }));
+      }
+      if (!this.mapFn) {
+        throw new Error("No live map function installed for index, cannot update. Make sure your index definition runs before any queries." + (this.mapFnString ? " Your code should match the stored map function source:\n" + this.mapFnString : ""));
+      }
+      const indexEntries = indexEntriesForChanges(result.rows, this.mapFn);
+      const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }));
+      this.indexById = await bulkIndex(blocks, this.indexById, removeByIdIndexEntries.concat(byIdIndexEntries), idIndexOpts);
+      this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts);
+      this.dbHead = result.clock;
+    });
+    this.database.notifyExternal("dbIndex");
+  }
+}
+async function bulkIndex(blocks, inIndex, indexEntries, opts) {
+  if (!indexEntries.length)
+    return inIndex;
+  const putBlock = blocks.put.bind(blocks);
+  const { getBlock } = makeGetBlock(blocks);
+  let returnRootBlock;
+  let returnNode;
+  if (!inIndex.root) {
+    const cid = inIndex.cid;
+    if (!cid) {
+      for await (const node of await dbIndex.create({ get: getBlock, list: indexEntries, ...opts })) {
+        const block = await node.block;
+        await putBlock(block.cid, block.bytes);
+        returnRootBlock = block;
+        returnNode = node;
+      }
+      return { root: returnNode, cid: returnRootBlock.cid };
+    }
+    inIndex.root = await dbIndex.load({ cid, get: getBlock, ...dbIndexOpts });
+  }
+  const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
+  returnRootBlock = await root.block;
+  returnNode = root;
+  for await (const block of newBlocks) {
+    await putBlock(block.cid, block.bytes);
+  }
+  await putBlock(returnRootBlock.cid, returnRootBlock.bytes);
+  return { root: returnNode, cid: returnRootBlock.cid };
+}
+async function loadIndex(blocks, index, indexOpts) {
+  if (!index.root) {
+    const cid = index.cid;
+    if (!cid)
+      return;
+    const { getBlock } = makeGetBlock(blocks);
+    index.root = await dbIndex.load({ cid, get: getBlock, ...indexOpts });
+  }
+  return index.root;
+}
+async function applyLimit(results, limit) {
+  results.result = results.result.slice(0, limit);
+  return results;
+}
+async function doIndexQuery(blocks, indexByKey, query = {}) {
+  await loadIndex(blocks, indexByKey, dbIndexOpts);
+  if (!indexByKey.root)
+    return { result: [] };
+  if (query.range) {
+    const encodedRange = query.range.map((key) => charwise.encode(key));
+    return applyLimit(await indexByKey.root.range(...encodedRange), query.limit);
+  } else if (query.key) {
+    const encodedKey = charwise.encode(query.key);
+    return indexByKey.root.get(encodedKey);
+  } else {
+    const { result, ...all } = await indexByKey.root.getAllEntries();
+    return applyLimit({ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }, query.limit);
+  }
+}
+
+class Listener {
+  subcribers = /* @__PURE__ */ new Map();
+  doStopListening = null;
+  /**
+   * @param {import('./fireproof.js').Fireproof} database
+   * @param {(_: any, emit: any) => void} routingFn
+   */
+  constructor(database, routingFn) {
+    this.database = database;
+    this.doStopListening = database.registerListener((changes) => this.onChanges(changes));
+    this.routingFn = routingFn || function(_, emit) {
+      emit("*");
+    };
+    this.dbHead = null;
+  }
+  /**
+   * Subscribe to a topic emitted by the event function.
+   * @param {string} topic - The topic to subscribe to.
+   * @param {Function} subscriber - The function to call when the topic is emitted.
+   * @returns {Function} A function to unsubscribe from the topic.
+   * @memberof Listener
+   * @instance
+   * @param {any} since
+   */
+  on(topic, subscriber, since) {
+    const listOfTopicSubscribers = getTopicList(this.subcribers, topic);
+    listOfTopicSubscribers.push(subscriber);
+    if (typeof since !== "undefined") {
+      this.database.changesSince(since).then(({ rows: changes }) => {
+        const keys = topicsForChanges(changes, this.routingFn).get(topic);
+        if (keys)
+          keys.forEach((key) => subscriber(key));
+      });
+    }
+    return () => {
+      const index = listOfTopicSubscribers.indexOf(subscriber);
+      if (index > -1)
+        listOfTopicSubscribers.splice(index, 1);
+    };
+  }
+  /**
+   * @typedef {import('./db-index').ChangeEvent} ChangeEvent
+   */
+  /**
+   * @param {ChangeEvent[]} changes
+   */
+  onChanges(changes) {
+    if (Array.isArray(changes)) {
+      const seenTopics = topicsForChanges(changes, this.routingFn);
+      for (const [topic, keys] of seenTopics) {
+        const listOfTopicSubscribers = getTopicList(this.subcribers, topic);
+        listOfTopicSubscribers.forEach(
+          (subscriber) => keys.forEach((key) => subscriber(key))
+        );
+      }
+    } else {
+      for (const [, listOfTopicSubscribers] of this.subcribers) {
+        listOfTopicSubscribers.forEach((subscriber) => subscriber(changes));
+      }
+    }
+  }
+}
+function getTopicList(subscribersMap, name) {
+  let topicList = subscribersMap.get(name);
+  if (!topicList) {
+    topicList = [];
+    subscribersMap.set(name, topicList);
+  }
+  return topicList;
+}
+const topicsForChanges = (changes, routingFn) => {
+  const seenTopics = /* @__PURE__ */ new Map();
+  changes.forEach(({ key, value, del }) => {
+    if (del || !value)
+      value = { _deleted: true };
+    routingFn({ _id: key, ...value }, (t) => {
+      const topicList = getTopicList(seenTopics, t);
+      topicList.push(key);
+    });
+  });
+  return seenTopics;
+};
+
+const parseCID = (cid) => typeof cid === "string" ? multiformats.CID.parse(cid) : cid;
+class Hydrator {
+  static fromJSON(json, database) {
+    database.hydrate({ clock: json.clock.map((c) => parseCID(c)), name: json.name, key: json.key });
+    if (json.indexes) {
+      for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
+        DbIndex.fromJSON(database, {
+          clock: {
+            byId: byId ? parseCID(byId) : null,
+            byKey: byKey ? parseCID(byKey) : null,
+            db: db ? db.map((c) => parseCID(c)) : null
+          },
+          code,
+          name
+        });
+      }
+    }
+    return database;
+  }
+  static snapshot(database, clock) {
+    const definition = database.toJSON();
+    const withBlocks = new Fireproof(database.blocks);
+    if (clock) {
+      definition.clock = clock.map((c) => parseCID(c));
+      definition.indexes.forEach((index) => {
+        index.clock.byId = null;
+        index.clock.byKey = null;
+        index.clock.db = null;
+      });
+    }
+    const snappedDb = this.fromJSON(definition, withBlocks);
+    [...database.indexes.values()].forEach((index) => {
+      snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn;
+    });
+    return snappedDb;
+  }
+  static async zoom(database, clock) {
+    [...database.indexes.values()].forEach((index) => {
+      index.indexById = { root: null, cid: null };
+      index.indexByKey = { root: null, cid: null };
+      index.dbHead = null;
+    });
+    database.clock = clock.map((c) => parseCID(c));
+    await database.notifyReset();
+    return database;
+  }
+}
+
+exports.Fireproof = Fireproof;
+exports.Hydrator = Hydrator;
+exports.Index = DbIndex;
+exports.Listener = Listener;
+//# sourceMappingURL=index.js.map