@fireproof/core 0.16.1 → 0.16.3

package/dist/memory/fireproof.js

@@ -0,0 +1,1114 @@
+// src/database.ts
+import { uuidv7 } from "uuidv7";
+
+// src/write-queue.ts
+function writeQueue(worker, payload = Infinity, unbounded = false) {
+  const queue = [];
+  let isProcessing = false;
+  async function process() {
+    if (isProcessing || queue.length === 0)
+      return;
+    isProcessing = true;
+    const tasksToProcess = queue.splice(0, payload);
+    const updates = tasksToProcess.map((item) => item.task);
+    if (unbounded) {
+      const promises = updates.map(async (update, index2) => {
+        try {
+          const result = await worker([update]);
+          tasksToProcess[index2].resolve(result);
+        } catch (error) {
+          tasksToProcess[index2].reject(error);
+        }
+      });
+      await Promise.all(promises);
+    } else {
+      try {
+        const result = await worker(updates);
+        tasksToProcess.forEach((task) => task.resolve(result));
+      } catch (error) {
+        tasksToProcess.forEach((task) => task.reject(error));
+      }
+    }
+    isProcessing = false;
+    void process();
+  }
+  return {
+    push(task) {
+      return new Promise((resolve, reject) => {
+        queue.push({ task, resolve, reject });
+        void process();
+      });
+    }
+  };
+}
+
+// src/crdt.ts
+import {
+  EncryptedBlockstore
+} from "@fireproof/encrypted-blockstore";
+
+// src/eb-web.ts
+import * as crypto from "@fireproof/encrypted-blockstore/crypto-web";
+import * as store from "@fireproof/encrypted-blockstore/store-web";
+
+// src/crdt-helpers.ts
+import { encode, decode, Block } from "multiformats/block";
+import { parse } from "multiformats/link";
+import { sha256 as hasher } from "multiformats/hashes/sha2";
+import * as codec from "@ipld/dag-cbor";
+import { put, get, entries, root } from "@alanshaw/pail/crdt";
+import { EventFetcher, vis } from "@alanshaw/pail/clock";
+import {
+  CarTransaction
+} from "@fireproof/encrypted-blockstore";
+
+// src/files.ts
+import * as UnixFS from "@ipld/unixfs";
+import * as raw from "multiformats/codecs/raw";
+import { withMaxChunkSize } from "@ipld/unixfs/file/chunker/fixed";
+import { withWidth } from "@ipld/unixfs/file/layout/balanced";
+import { exporter } from "ipfs-unixfs-exporter";
+var queuingStrategy = UnixFS.withCapacity();
+var settings = UnixFS.configure({
+  fileChunkEncoder: raw,
+  smallFileEncoder: raw,
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
+  chunker: withMaxChunkSize(1024 * 1024),
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
+  fileLayout: withWidth(1024)
+});
+async function encodeFile(blob) {
+  const readable = createFileEncoderStream(blob);
+  const blocks = await collect(readable);
+  return { cid: blocks.at(-1).cid, blocks };
+}
+async function decodeFile(blocks, cid, meta) {
+  const entry = await exporter(cid.toString(), blocks, { length: meta.size });
+  const chunks = [];
+  for await (const chunk of entry.content())
+    chunks.push(chunk);
+  return new File(chunks, entry.name, { type: meta.type, lastModified: 0 });
+}
+function createFileEncoderStream(blob) {
+  const { readable, writable } = new TransformStream({}, queuingStrategy);
+  const unixfsWriter = UnixFS.createWriter({ writable, settings });
+  const fileBuilder = new UnixFSFileBuilder("", blob);
+  void (async () => {
+    await fileBuilder.finalize(unixfsWriter);
+    await unixfsWriter.close();
+  })();
+  return readable;
+}
+async function collect(collectable) {
+  const chunks = [];
+  await collectable.pipeTo(
+    new WritableStream({
+      write(chunk) {
+        chunks.push(chunk);
+      }
+    })
+  );
+  return chunks;
+}
+var UnixFSFileBuilder = class {
+  #file;
+  name;
+  constructor(name, file) {
+    this.name = name;
+    this.#file = file;
+  }
+  async finalize(writer) {
+    const unixfsFileWriter = UnixFS.createFileWriter(writer);
+    await this.#file.stream().pipeTo(
+      new WritableStream({
+        async write(chunk) {
+          await unixfsFileWriter.write(chunk);
+        }
+      })
+    );
+    return await unixfsFileWriter.close();
+  }
+};
+
+// src/crdt-helpers.ts
+async function applyBulkUpdateToCrdt(tblocks, head, updates, options) {
+  let result;
+  for (const update of updates) {
+    const link = await writeDocContent(tblocks, update);
+    result = await put(tblocks, head, update.key, link, options);
+    const resRoot = result.root.toString();
+    const isReturned = result.additions.some((a) => a.cid.toString() === resRoot);
+    if (!isReturned) {
+      const hasRoot = await tblocks.get(result.root);
+      if (!hasRoot) {
+        throw new Error(
+          `missing root in additions: ${result.additions.length} ${resRoot} keys: ${updates.map((u) => u.key).toString()}`
+        );
+        result.head = head;
+      }
+    }
+    if (result.event) {
+      for (const { cid, bytes } of [...result.additions, result.event]) {
+        tblocks.putSync(cid, bytes);
+      }
+      head = result.head;
+    }
+  }
+  return { head };
+}
+async function writeDocContent(blocks, update) {
+  let value;
+  if (update.del) {
+    value = { del: true };
+  } else {
+    await processFiles(blocks, update.value);
+    value = { doc: update.value };
+  }
+  const block = await encode({ value, hasher, codec });
+  blocks.putSync(block.cid, block.bytes);
+  return block.cid;
+}
+async function processFiles(blocks, doc) {
+  if (doc._files) {
+    await processFileset(blocks, doc._files);
+  }
+  if (doc._publicFiles) {
+    await processFileset(blocks, doc._publicFiles, true);
+  }
+}
+async function processFileset(blocks, files, publicFiles = false) {
+  const dbBlockstore = blocks.parent;
+  const t = new CarTransaction(dbBlockstore);
+  const didPut = [];
+  for (const filename in files) {
+    if (File === files[filename].constructor) {
+      const file = files[filename];
+      const { cid, blocks: fileBlocks } = await encodeFile(file);
+      didPut.push(filename);
+      for (const block of fileBlocks) {
+        t.putSync(block.cid, block.bytes);
+      }
+      files[filename] = { cid, type: file.type, size: file.size };
+    }
+  }
+  if (didPut.length) {
+    const car = await dbBlockstore.loader?.commitFiles(t, { files }, {
+      public: publicFiles
+    });
+    if (car) {
+      for (const name of didPut) {
+        files[name] = { car, ...files[name] };
+      }
+    }
+  }
+}
+async function getValueFromCrdt(blocks, head, key) {
+  if (!head.length)
+    throw new Error("Getting from an empty database");
+  const link = await get(blocks, head, key);
+  if (!link)
+    throw new Error(`Missing key ${key}`);
+  return await getValueFromLink(blocks, link);
+}
+function readFiles(blocks, { doc }) {
+  if (!doc)
+    return;
+  if (doc._files) {
+    readFileset(blocks, doc._files);
+  }
+  if (doc._publicFiles) {
+    readFileset(blocks, doc._publicFiles, true);
+  }
+}
+function readFileset(blocks, files, isPublic = false) {
+  for (const filename in files) {
+    const fileMeta = files[filename];
+    if (fileMeta.cid) {
+      if (isPublic) {
+        fileMeta.url = `https://${fileMeta.cid.toString()}.ipfs.w3s.link/`;
+      }
+      if (fileMeta.car) {
+        fileMeta.file = async () => await decodeFile(
+          {
+            get: async (cid) => {
+              return await blocks.getFile(fileMeta.car, cid, isPublic);
+            }
+          },
+          fileMeta.cid,
+          fileMeta
+        );
+      }
+    }
+    files[filename] = fileMeta;
+  }
+}
+async function getValueFromLink(blocks, link) {
+  const block = await blocks.get(link);
+  if (!block)
+    throw new Error(`Missing linked block ${link.toString()}`);
+  const { value } = await decode({ bytes: block.bytes, hasher, codec });
+  readFiles(blocks, value);
+  return value;
+}
+var DirtyEventFetcher = class extends EventFetcher {
+  // @ts-ignore
+  async get(link) {
+    try {
+      return await super.get(link);
+    } catch (e) {
+      console.error("missing event", link.toString(), e);
+      return { value: null };
+    }
+  }
+};
+async function clockChangesSince(blocks, head, since, opts) {
+  const eventsFetcher = opts.dirty ? new DirtyEventFetcher(blocks) : new EventFetcher(blocks);
+  const keys = /* @__PURE__ */ new Set();
+  const updates = await gatherUpdates(
+    blocks,
+    eventsFetcher,
+    head,
+    since,
+    [],
+    keys,
+    /* @__PURE__ */ new Set(),
+    opts.limit || Infinity
+  );
+  return { result: updates.reverse(), head };
+}
+async function gatherUpdates(blocks, eventsFetcher, head, since, updates = [], keys, didLinks, limit) {
+  if (limit <= 0)
+    return updates;
+  const sHead = head.map((l) => l.toString());
+  for (const link of since) {
+    if (sHead.includes(link.toString())) {
+      return updates;
+    }
+  }
+  for (const link of head) {
+    if (didLinks.has(link.toString()))
+      continue;
+    didLinks.add(link.toString());
+    const { value: event } = await eventsFetcher.get(link);
+    if (!event)
+      continue;
+    const { key, value } = event.data;
+    if (keys.has(key)) {
+      if (event.parents) {
+        updates = await gatherUpdates(
+          blocks,
+          eventsFetcher,
+          event.parents,
+          since,
+          updates,
+          keys,
+          didLinks,
+          limit
+        );
+      }
+    } else {
+      keys.add(key);
+      const docValue = await getValueFromLink(blocks, value);
+      updates.push({ key, value: docValue.doc, del: docValue.del, clock: link });
+      limit--;
+      if (event.parents) {
+        updates = await gatherUpdates(
+          blocks,
+          eventsFetcher,
+          event.parents,
+          since,
+          updates,
+          keys,
+          didLinks,
+          limit
+        );
+      }
+    }
+  }
+  return updates;
+}
+async function* getAllEntries(blocks, head) {
+  for await (const [key, link] of entries(blocks, head)) {
+    const docValue = await getValueFromLink(blocks, link);
+    yield { key, value: docValue.doc, del: docValue.del };
+  }
+}
+async function* clockVis(blocks, head) {
+  for await (const line of vis(blocks, head)) {
+    yield line;
+  }
+}
+var isCompacting = false;
+async function doCompact(blockLog, head) {
+  if (isCompacting) {
+    console.log("already compacting");
+    return;
+  }
+  isCompacting = true;
+  for (const cid of head) {
+    const bl = await blockLog.get(cid);
+    if (!bl)
+      throw new Error("Missing head block: " + cid.toString());
+  }
+  for await (const entry of getAllEntries(blockLog, head)) {
+  }
+  for await (const [, link] of entries(blockLog, head)) {
+    const bl = await blockLog.get(link);
+    if (!bl)
+      throw new Error("Missing entry block: " + link.toString());
+  }
+  for await (const _line of vis(blockLog, head)) {
+  }
+  const result = await root(blockLog, head);
+  for (const { cid, bytes } of [...result.additions, ...result.removals]) {
+    blockLog.loggedBlocks.putSync(cid, bytes);
+  }
+  await clockChangesSince(blockLog, head, [], {});
+  isCompacting = false;
+}
+async function getBlock(blocks, cidString) {
+  const block = await blocks.get(parse(cidString));
+  if (!block)
+    throw new Error(`Missing block ${cidString}`);
+  const { cid, value } = await decode({ bytes: block.bytes, codec, hasher });
+  return new Block({ cid, value, bytes: block.bytes });
+}
+
+// src/indexer-helpers.ts
+import { create } from "multiformats/block";
+import { sha256 as hasher2 } from "multiformats/hashes/sha2";
+import * as codec2 from "@ipld/dag-cbor";
+import charwise from "charwise";
+import * as DbIndex from "prolly-trees/db-index";
+import { bf, simpleCompare } from "prolly-trees/utils";
+import { nocache as cache } from "prolly-trees/cache";
+var IndexTree = class {
+  cid = null;
+  root = null;
+};
+var refCompare = (aRef, bRef) => {
+  if (Number.isNaN(aRef))
+    return -1;
+  if (Number.isNaN(bRef))
+    throw new Error("ref may not be Infinity or NaN");
+  if (aRef === Infinity)
+    return 1;
+  return simpleCompare(aRef, bRef);
+};
+var compare = (a, b) => {
+  const [aKey, aRef] = a;
+  const [bKey, bRef] = b;
+  const comp = simpleCompare(aKey, bKey);
+  if (comp !== 0)
+    return comp;
+  return refCompare(aRef, bRef);
+};
+var byKeyOpts = { cache, chunker: bf(30), codec: codec2, hasher: hasher2, compare };
+var byIdOpts = { cache, chunker: bf(30), codec: codec2, hasher: hasher2, compare: simpleCompare };
+function indexEntriesForChanges(changes, mapFn) {
+  const indexEntries = [];
+  changes.forEach(({ key: _id, value, del }) => {
+    if (del || !value)
+      return;
+    let mapCalled = false;
+    const mapReturn = mapFn({ _id, ...value }, (k, v) => {
+      mapCalled = true;
+      if (typeof k === "undefined")
+        return;
+      indexEntries.push({
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+        key: [charwise.encode(k), _id],
+        value: v || null
+      });
+    });
+    if (!mapCalled && mapReturn) {
+      indexEntries.push({
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+        key: [charwise.encode(mapReturn), _id],
+        value: null
+      });
+    }
+  });
+  return indexEntries;
+}
+function makeProllyGetBlock(blocks) {
+  return async (address) => {
+    const block = await blocks.get(address);
+    if (!block)
+      throw new Error(`Missing block ${address.toString()}`);
+    const { cid, bytes } = block;
+    return create({ cid, bytes, hasher: hasher2, codec: codec2 });
+  };
+}
+async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
+  if (!indexEntries.length)
+    return inIndex;
+  if (!inIndex.root) {
+    if (!inIndex.cid) {
+      let returnRootBlock = null;
+      let returnNode = null;
+      for await (const node of await DbIndex.create({ get: makeProllyGetBlock(tblocks), list: indexEntries, ...opts })) {
+        const block = await node.block;
+        await tblocks.put(block.cid, block.bytes);
+        returnRootBlock = block;
+        returnNode = node;
+      }
+      if (!returnNode || !returnRootBlock)
+        throw new Error("failed to create index");
+      return { root: returnNode, cid: returnRootBlock.cid };
+    } else {
+      inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
+    }
+  }
+  const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
+  if (root3) {
+    for await (const block of newBlocks) {
+      await tblocks.put(block.cid, block.bytes);
+    }
+    return { root: root3, cid: (await root3.block).cid };
+  } else {
+    return { root: null, cid: null };
+  }
+}
+async function loadIndex(tblocks, cid, opts) {
+  return await DbIndex.load({ cid, get: makeProllyGetBlock(tblocks), ...opts });
+}
+async function applyQuery(crdt, resp, query) {
+  if (query.descending) {
+    resp.result = resp.result.reverse();
+  }
+  if (query.limit) {
+    resp.result = resp.result.slice(0, query.limit);
+  }
+  if (query.includeDocs) {
+    resp.result = await Promise.all(
+      resp.result.map(async (row) => {
+        const val = await crdt.get(row.id);
+        const doc = val ? { _id: row.id, ...val.doc } : null;
+        return { ...row, doc };
+      })
+    );
+  }
+  return {
+    rows: resp.result.map((row) => {
+      row.key = charwise.decode(row.key);
+      if (row.row && !row.value) {
+        row.value = row.row;
+        delete row.row;
+      }
+      return row;
+    })
+  };
+}
+function encodeRange(range) {
+  return range.map((key) => charwise.encode(key));
+}
+function encodeKey(key) {
+  return charwise.encode(key);
+}
+
+// src/index.ts
+function index({ _crdt }, name, mapFn, meta) {
+  if (mapFn && meta)
+    throw new Error("cannot provide both mapFn and meta");
+  if (mapFn && mapFn.constructor.name !== "Function")
+    throw new Error("mapFn must be a function");
+  if (_crdt.indexers.has(name)) {
+    const idx = _crdt.indexers.get(name);
+    idx.applyMapFn(name, mapFn, meta);
+  } else {
+    const idx = new Index(_crdt, name, mapFn, meta);
+    _crdt.indexers.set(name, idx);
+  }
+  return _crdt.indexers.get(name);
+}
+var Index = class {
+  blockstore;
+  crdt;
+  name = null;
+  mapFn = null;
+  mapFnString = "";
+  byKey = new IndexTree();
+  byId = new IndexTree();
+  indexHead = void 0;
+  includeDocsDefault = false;
+  initError = null;
+  ready;
+  constructor(crdt, name, mapFn, meta) {
+    this.blockstore = crdt.indexBlockstore;
+    this.crdt = crdt;
+    this.applyMapFn(name, mapFn, meta);
+    if (!(this.mapFnString || this.initError))
+      throw new Error("missing mapFnString");
+    this.ready = this.blockstore.ready.then(() => {
+    });
+  }
+  applyMapFn(name, mapFn, meta) {
+    if (mapFn && meta)
+      throw new Error("cannot provide both mapFn and meta");
+    if (this.name && this.name !== name)
+      throw new Error("cannot change name");
+    this.name = name;
+    try {
+      if (meta) {
+        if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
+          throw new Error("cannot apply meta to existing index");
+        }
+        if (this.mapFnString) {
+          if (this.mapFnString !== meta.map) {
+            console.log(
+              "cannot apply different mapFn meta: old mapFnString",
+              this.mapFnString,
+              "new mapFnString",
+              meta.map
+            );
+          } else {
+            this.byId.cid = meta.byId;
+            this.byKey.cid = meta.byKey;
+            this.indexHead = meta.head;
+          }
+        } else {
+          this.mapFnString = meta.map;
+          this.byId.cid = meta.byId;
+          this.byKey.cid = meta.byKey;
+          this.indexHead = meta.head;
+        }
+      } else {
+        if (this.mapFn) {
+          if (mapFn) {
+            if (this.mapFn.toString() !== mapFn.toString())
+              throw new Error("cannot apply different mapFn app2");
+          }
+        } else {
+          if (!mapFn) {
+            mapFn = makeMapFnFromName(name);
+          }
+          if (this.mapFnString) {
+            if (this.mapFnString !== mapFn.toString())
+              throw new Error("cannot apply different mapFn app");
+          } else {
+            this.mapFnString = mapFn.toString();
+          }
+          this.mapFn = mapFn;
+        }
+      }
+      const matches = /=>\s*(.*)/.test(this.mapFnString);
+      this.includeDocsDefault = matches;
+    } catch (e) {
+      this.initError = e;
+    }
+  }
+  async query(opts = {}) {
+    await this._updateIndex();
+    await this._hydrateIndex();
+    if (!this.byKey.root)
+      return await applyQuery(this.crdt, { result: [] }, opts);
+    if (this.includeDocsDefault && opts.includeDocs === void 0)
+      opts.includeDocs = true;
+    if (opts.range) {
+      const { result: result2, ...all2 } = await this.byKey.root.range(...encodeRange(opts.range));
+      return await applyQuery(this.crdt, { result: result2, ...all2 }, opts);
+    }
+    if (opts.key) {
+      const encodedKey = encodeKey(opts.key);
+      return await applyQuery(this.crdt, await this.byKey.root.get(encodedKey), opts);
+    }
+    if (Array.isArray(opts.keys)) {
+      const results = await Promise.all(
+        opts.keys.map(async (key) => {
+          const encodedKey = encodeKey(key);
+          return (await applyQuery(this.crdt, await this.byKey.root.get(encodedKey), opts)).rows;
+        })
+      );
+      return { rows: results.flat() };
+    }
+    if (opts.prefix) {
+      if (!Array.isArray(opts.prefix))
+        opts.prefix = [opts.prefix];
+      const start = [...opts.prefix, NaN];
+      const end = [...opts.prefix, Infinity];
+      const encodedR = encodeRange([start, end]);
+      return await applyQuery(this.crdt, await this.byKey.root.range(...encodedR), opts);
+    }
+    const { result, ...all } = await this.byKey.root.getAllEntries();
+    return await applyQuery(
+      this.crdt,
+      {
+        result: result.map(({ key: [k, id], value }) => ({ key: k, id, value })),
+        ...all
+      },
+      opts
+    );
+  }
+  _resetIndex() {
+    this.byId = new IndexTree();
+    this.byKey = new IndexTree();
+    this.indexHead = void 0;
+  }
+  async _hydrateIndex() {
+    if (this.byId.root && this.byKey.root)
+      return;
+    if (!this.byId.cid || !this.byKey.cid)
+      return;
+    this.byId.root = await loadIndex(this.blockstore, this.byId.cid, byIdOpts);
+    this.byKey.root = await loadIndex(this.blockstore, this.byKey.cid, byKeyOpts);
+  }
+  async _updateIndex() {
+    await this.ready;
+    if (this.initError)
+      throw this.initError;
+    if (!this.mapFn)
+      throw new Error("No map function defined");
+    let result, head;
+    if (!this.indexHead || this.indexHead.length === 0) {
+      ;
+      ({ result, head } = await this.crdt.allDocs());
+    } else {
+      ;
+      ({ result, head } = await this.crdt.changes(this.indexHead));
+    }
+    if (result.length === 0) {
+      this.indexHead = head;
+      return { byId: this.byId, byKey: this.byKey };
+    }
+    let staleKeyIndexEntries = [];
+    let removeIdIndexEntries = [];
+    if (this.byId.root) {
+      const removeIds = result.map(({ key }) => key);
+      const { result: oldChangeEntries } = await this.byId.root.getMany(removeIds);
+      staleKeyIndexEntries = oldChangeEntries.map((key) => ({ key, del: true }));
+      removeIdIndexEntries = oldChangeEntries.map((key) => ({ key: key[1], del: true }));
+    }
+    const indexEntries = indexEntriesForChanges(result, this.mapFn);
+    const byIdIndexEntries = indexEntries.map(({ key }) => ({
+      key: key[1],
+      value: key
+    }));
+    const indexerMeta = { indexes: /* @__PURE__ */ new Map() };
+    for (const [name, indexer] of this.crdt.indexers) {
+      if (indexer.indexHead) {
+        indexerMeta.indexes.set(name, {
+          byId: indexer.byId.cid,
+          byKey: indexer.byKey.cid,
+          head: indexer.indexHead,
+          map: indexer.mapFnString,
+          name: indexer.name
+        });
+      }
+    }
+    return await this.blockstore.transaction(async (tblocks) => {
+      this.byId = await bulkIndex(
+        tblocks,
+        this.byId,
+        removeIdIndexEntries.concat(byIdIndexEntries),
+        byIdOpts
+      );
+      this.byKey = await bulkIndex(
+        tblocks,
+        this.byKey,
+        staleKeyIndexEntries.concat(indexEntries),
+        byKeyOpts
+      );
+      this.indexHead = head;
+      const idxMeta = {
+        byId: this.byId.cid,
+        byKey: this.byKey.cid,
+        head,
+        map: this.mapFnString,
+        name: this.name
+      };
+      indexerMeta.indexes.set(this.name, idxMeta);
+      return indexerMeta;
+    });
+  }
+};
+function makeMapFnFromName(name) {
+  return (doc) => doc[name] ?? void 0;
+}
+
+// src/crdt-clock.ts
+import { advance } from "@alanshaw/pail/clock";
+import { root as root2 } from "@alanshaw/pail/crdt";
+
+// src/apply-head-queue.ts
+function applyHeadQueue(worker) {
+  const queue = [];
+  let isProcessing = false;
+  async function* process() {
+    if (isProcessing || queue.length === 0)
+      return;
+    isProcessing = true;
+    const allUpdates = [];
+    try {
+      while (queue.length > 0) {
+        queue.sort((a, b) => b.updates ? 1 : -1);
+        const task = queue.shift();
+        if (!task)
+          continue;
+        await worker(task.newHead, task.prevHead);
+        if (task.updates) {
+          allUpdates.push(...task.updates);
+        }
+        if (!queue.some((t) => t.updates) || task.updates) {
+          const allTasksHaveUpdates = queue.every((task2) => task2.updates !== null);
+          yield { updates: allUpdates, all: allTasksHaveUpdates };
+          allUpdates.length = 0;
+        }
+      }
+    } finally {
+      isProcessing = false;
+      const generator = process();
+      let result = await generator.next();
+      while (!result.done) {
+        result = await generator.next();
+      }
+    }
+  }
+  return {
+    push(task) {
+      queue.push(task);
+      return process();
+    }
+  };
+}
+
+// src/crdt-clock.ts
+var CRDTClock = class {
+  // todo: track local and remote clocks independently, merge on read
+  // that way we can drop the whole remote if we need to
+  // should go with making sure the local clock only references locally available blockstore on write
+  head = [];
+  zoomers = /* @__PURE__ */ new Set();
+  watchers = /* @__PURE__ */ new Set();
+  emptyWatchers = /* @__PURE__ */ new Set();
+  blockstore = null;
+  applyHeadQueue;
+  constructor() {
+    this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this));
+  }
+  setHead(head) {
+    this.head = head;
+  }
+  async applyHead(newHead, prevHead, updates = null) {
+    for await (const { updates: updatesAcc, all } of this.applyHeadQueue.push({
+      newHead,
+      prevHead,
+      updates
+    })) {
+      this.processUpdates(updatesAcc, all, prevHead);
+    }
+  }
+  async processUpdates(updatesAcc, all, prevHead) {
+    let internalUpdates = updatesAcc;
+    if (this.watchers.size && !all) {
+      const changes = await clockChangesSince(this.blockstore, this.head, prevHead, {});
+      internalUpdates = changes.result;
+    }
+    this.zoomers.forEach((fn) => fn());
+    this.notifyWatchers(internalUpdates || []);
+  }
+  notifyWatchers(updates) {
+    this.emptyWatchers.forEach((fn) => fn());
+    this.watchers.forEach((fn) => fn(updates || []));
+  }
+  onTick(fn) {
+    this.watchers.add(fn);
+  }
+  onTock(fn) {
+    this.emptyWatchers.add(fn);
+  }
+  onZoom(fn) {
+    this.zoomers.add(fn);
+  }
+  async int_applyHead(newHead, prevHead) {
+    const ogHead = sortClockHead(this.head);
+    newHead = sortClockHead(newHead);
+    if (compareClockHeads(ogHead, newHead)) {
+      return;
+    }
+    const ogPrev = sortClockHead(prevHead);
+    if (compareClockHeads(ogHead, ogPrev)) {
+      this.setHead(newHead);
+      return;
+    }
+    let head = this.head;
+    const noLoader = false;
+    if (!this.blockstore)
+      throw new Error("missing blockstore");
+    await validateBlocks(newHead, this.blockstore);
+    await this.blockstore.transaction(
+      async (tblocks) => {
+        head = await advanceBlocks(newHead, tblocks, head);
+        const result = await root2(tblocks, head);
+        for (const { cid, bytes } of [...result.additions, ...result.removals]) {
+          tblocks.putSync(cid, bytes);
+        }
+        return { head };
+      },
+      { noLoader }
+    );
+    this.setHead(head);
+  }
+};
+function sortClockHead(clockHead) {
+  return clockHead.sort((a, b) => a.toString().localeCompare(b.toString()));
+}
+async function validateBlocks(newHead, blockstore) {
+  newHead.map(async (cid) => {
+    const got = await blockstore.get(cid);
+    if (!got) {
+      throw new Error("int_applyHead missing block: " + cid.toString());
+    }
+  });
+}
+function compareClockHeads(head1, head2) {
+  return head1.toString() === head2.toString();
+}
+async function advanceBlocks(newHead, tblocks, head) {
+  for (const cid of newHead) {
+    try {
+      head = await advance(tblocks, head, cid);
+    } catch (e) {
+      console.error("failed to advance", cid.toString(), e);
+      continue;
+    }
+  }
+  return head;
+}
+
+// src/crdt.ts
+var CRDT = class {
+  name;
+  opts = {};
+  ready;
+  blockstore;
+  indexBlockstore;
+  indexers = /* @__PURE__ */ new Map();
+  clock = new CRDTClock();
+  constructor(name, opts) {
+    this.name = name || null;
+    this.opts = opts || this.opts;
+    this.blockstore = new EncryptedBlockstore({
+      name,
+      applyMeta: async (meta) => {
+        const crdtMeta = meta;
+        await this.clock.applyHead(crdtMeta.head, []);
+      },
+      compact: async (blocks) => {
+        await doCompact(blocks, this.clock.head);
+        return { head: this.clock.head };
+      },
+      autoCompact: this.opts.autoCompact || 100,
+      crypto,
+      store,
+      public: this.opts.public,
+      meta: this.opts.meta
+    });
+    this.clock.blockstore = this.blockstore;
+    this.indexBlockstore = new EncryptedBlockstore({
+      name: this.opts.persistIndexes && this.name ? this.name + ".idx" : void 0,
+      applyMeta: async (meta) => {
+        const idxCarMeta = meta;
+        for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
+          index({ _crdt: this }, name2, void 0, idx);
+        }
+      },
+      crypto,
+      public: this.opts.public,
+      store
+    });
+    this.ready = Promise.all([this.blockstore.ready, this.indexBlockstore.ready]).then(() => {
+    });
+    this.clock.onZoom(() => {
+      for (const idx of this.indexers.values()) {
+        idx._resetIndex();
+      }
+    });
+  }
+  async bulk(updates, options) {
+    await this.ready;
+    const prevHead = [...this.clock.head];
+    const meta = await this.blockstore.transaction(
+      async (blocks) => {
+        const { head } = await applyBulkUpdateToCrdt(blocks, this.clock.head, updates, options);
+        updates = updates.map(({ key, value, del, clock }) => {
+          readFiles(this.blockstore, { doc: value });
+          return { key, value, del, clock };
+        });
+        return { head };
+      }
+    );
+    await this.clock.applyHead(meta.head, prevHead, updates);
+    return meta;
+  }
+  // if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
+  async allDocs() {
+    await this.ready;
+    const result = [];
+    for await (const entry of getAllEntries(this.blockstore, this.clock.head)) {
+      result.push(entry);
+    }
+    return { result, head: this.clock.head };
+  }
+  async vis() {
+    await this.ready;
+    const txt = [];
+    for await (const line of clockVis(this.blockstore, this.clock.head)) {
+      txt.push(line);
+    }
+    return txt.join("\n");
+  }
+  async getBlock(cidString) {
+    await this.ready;
+    return await getBlock(this.blockstore, cidString);
+  }
+  async get(key) {
+    await this.ready;
+    const result = await getValueFromCrdt(this.blockstore, this.clock.head, key);
+    if (result.del)
+      return null;
+    return result;
+  }
+  async changes(since = [], opts = {}) {
+    await this.ready;
+    return await clockChangesSince(this.blockstore, this.clock.head, since, opts);
+  }
+  async compact() {
+    return await this.blockstore.compact();
+  }
+};
+
+// src/database.ts
+var Database = class {
+  static databases = /* @__PURE__ */ new Map();
+  name;
+  opts = {};
+  _listening = false;
+  _listeners = /* @__PURE__ */ new Set();
+  _noupdate_listeners = /* @__PURE__ */ new Set();
+  _crdt;
+  _writeQueue;
+  blockstore;
+  constructor(name, opts) {
+    this.name = name || null;
+    this.opts = opts || this.opts;
+    this._crdt = new CRDT(name, this.opts);
+    this.blockstore = this._crdt.blockstore;
+    this._writeQueue = writeQueue(async (updates) => {
+      return await this._crdt.bulk(updates);
+    });
+    this._crdt.clock.onTock(() => {
+      this._no_update_notify();
+    });
+  }
+  async get(id) {
+    const got = await this._crdt.get(id).catch((e) => {
+      e.message = `Not found: ${id} - ` + e.message;
+      throw e;
+    });
+    if (!got)
+      throw new Error(`Not found: ${id}`);
+    const { doc } = got;
+    return { _id: id, ...doc };
+  }
+  async put(doc) {
+    const { _id, ...value } = doc;
+    const docId = _id || uuidv7();
+    const result = await this._writeQueue.push({ key: docId, value });
+    return { id: docId, clock: result?.head };
+  }
+  async del(id) {
+    const result = await this._writeQueue.push({ key: id, del: true });
+    return { id, clock: result?.head };
+  }
+  async changes(since = [], opts = {}) {
+    const { result, head } = await this._crdt.changes(since, opts);
+    const rows = result.map(({ key, value, del, clock }) => ({
+      key,
+      value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
+      clock
+    }));
+    return { rows, clock: head };
+  }
+  async allDocs() {
+    const { result, head } = await this._crdt.allDocs();
+    const rows = result.map(({ key, value, del }) => ({
+      key,
+      value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
+    }));
+    return { rows, clock: head };
+  }
+  async allDocuments() {
+    return this.allDocs();
+  }
+  subscribe(listener, updates) {
+    if (updates) {
+      if (!this._listening) {
+        this._listening = true;
+        this._crdt.clock.onTick((updates2) => {
+          void this._notify(updates2);
+        });
+      }
+      this._listeners.add(listener);
+      return () => {
+        this._listeners.delete(listener);
+      };
+    } else {
+      this._noupdate_listeners.add(listener);
+      return () => {
+        this._noupdate_listeners.delete(listener);
+      };
+    }
+  }
+  // todo if we add this onto dbs in fireproof.ts then we can make index.ts a separate package
+  async query(field, opts = {}) {
+    const idx = typeof field === "string" ? index({ _crdt: this._crdt }, field) : index({ _crdt: this._crdt }, makeName(field.toString()), field);
+    return await idx.query(opts);
+  }
+  async compact() {
+    await this._crdt.compact();
+  }
+  async _notify(updates) {
+    if (this._listeners.size) {
+      const docs = updates.map(({ key, value }) => ({ _id: key, ...value }));
+      for (const listener of this._listeners) {
+        await (async () => await listener(docs))().catch((e) => {
+          console.error("subscriber error", e);
+        });
+      }
+    }
+  }
+  async _no_update_notify() {
+    if (this._noupdate_listeners.size) {
+      for (const listener of this._noupdate_listeners) {
+        await (async () => await listener([]))().catch((e) => {
+          console.error("subscriber error", e);
+        });
+      }
+    }
+  }
+};
+function fireproof(name, opts) {
+  if (!Database.databases.has(name)) {
+    Database.databases.set(name, new Database(name, opts));
+  }
+  return Database.databases.get(name);
+}
+function makeName(fnString) {
+  const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
+  let found = null;
+  const matches = Array.from(fnString.matchAll(regex), (match) => match[1].trim());
+  if (matches.length === 0) {
+    found = /=>\s*(.*)/.exec(fnString);
+  }
+  if (!found) {
+    return fnString;
+  } else {
+    return found[1];
+  }
+}
+export {
+  Database,
+  fireproof
+};
+//# sourceMappingURL=fireproof.js.map