@fireproof/core 0.17.2 → 0.17.4

package/dist/node/fireproof.global.js

@@ -4136,22 +4136,6 @@ var Fireproof = (() => {
       }
     }
   };
-  var MultiBlockFetcher = class {
-    /** @type {API.BlockFetcher[]} */
-    #fetchers;
-    /** @param {API.BlockFetcher[]} fetchers */
-    constructor(...fetchers) {
-      this.#fetchers = fetchers;
-    }
-    /** @type {API.BlockFetcher['get']} */
-    async get(link2) {
-      for (const f of this.#fetchers) {
-        const v = await f.get(link2);
-        if (v)
-          return v;
-      }
-    }
-  };
 
   // ../../node_modules/.pnpm/yocto-queue@1.0.0/node_modules/yocto-queue/index.js
   var Node = class {
@@ -9952,8 +9936,7 @@ You can use close({ resize: true }) to resize header`);
     return uint8Array;
   }
   async function encryptedEncodeCarFile(crypto22, key, rootCid, t) {
-    const encryptionKeyUint8 = hexStringToUint8Array(key);
-    const encryptionKey = encryptionKeyUint8.buffer.slice(0, encryptionKeyUint8.byteLength);
+    const encryptionKey = hexStringToUint8Array(key);
     const encryptedBlocks = new MemoryBlockstore();
     const cidsToEncrypt = [];
     for (const { cid } of t.entries()) {
@@ -10030,7 +10013,7 @@ You can use close({ resize: true }) to resize header`);
     }
     return bytes;
   }
-  var PACKAGE_VERSION = "0.16.10";
+  var PACKAGE_VERSION = "0.17.3";
   var match = PACKAGE_VERSION.match(/^([^.]*\.[^.]*)/);
   if (!match)
     throw new Error("invalid version: " + PACKAGE_VERSION);
@@ -10106,11 +10089,11 @@ You can use close({ resize: true }) to resize header`);
     constructor(name7, ebOpts) {
       this.name = name7;
       this.ebOpts = ebOpts;
-      this.metaStore = new ebOpts.store.MetaStore(this.name);
-      this.carStore = new ebOpts.store.DataStore(this.name);
-      this.fileStore = new ebOpts.store.DataStore(this.name);
-      this.remoteWAL = new ebOpts.store.RemoteWAL(this);
+      this.carStore = ebOpts.store.makeDataStore(this.name);
+      this.fileStore = ebOpts.store.makeDataStore(this.name);
+      this.remoteWAL = ebOpts.store.makeRemoteWAL(this);
       this.ready = Promise.resolve().then(async () => {
+        this.metaStore = ebOpts.store.makeMetaStore(this);
         if (!this.metaStore || !this.carStore || !this.remoteWAL)
           throw new Error("stores not initialized");
         const metas = this.ebOpts.meta ? [this.ebOpts.meta] : await this.metaStore.load("main");
@@ -10284,19 +10267,30 @@ You can use close({ resize: true }) to resize header`);
       const sCid = cid.toString();
       if (this.getBlockCache.has(sCid))
         return this.getBlockCache.get(sCid);
-      const got = await Promise.any(
-        // maybe worth taking this in chunks of 5? to allow cache to favor recent files, carLog order is newest first
-        this.carLog.map(async (carCid) => {
-          const reader = await this.loadCar(carCid);
-          if (!reader) {
-            throw new Error(`missing car reader ${carCid.toString()}`);
-          }
-          await this.cacheCarReader(reader);
-          if (this.getBlockCache.has(sCid))
-            return this.getBlockCache.get(sCid);
-          throw new Error(`block not in reader: ${cid.toString()}`);
-        })
-      ).catch(() => void 0);
+      const getCarCid = async (carCid) => {
+        const reader = await this.loadCar(carCid);
+        if (!reader) {
+          throw new Error(`missing car reader ${carCid.toString()}`);
+        }
+        await this.cacheCarReader(reader);
+        if (this.getBlockCache.has(sCid))
+          return this.getBlockCache.get(sCid);
+        throw new Error(`block not in reader: ${cid.toString()}`);
+      };
+      let got;
+      const batchSize = 5;
+      for (let i = 0; i < this.carLog.length; i += batchSize) {
+        const promises = [];
+        for (let j = i; j < Math.min(i + batchSize, this.carLog.length); j++) {
+          promises.push(getCarCid(this.carLog[j]));
+        }
+        try {
+          got = await Promise.any(promises);
+        } catch {
+        }
+        if (got)
+          break;
+      }
       if (got) {
         this.getBlockCache.set(sCid, got);
       }
@@ -18766,12 +18760,15 @@ ${key.data.toString("base64")}
     DataStore: () => DataStore2,
     MetaStore: () => MetaStore2,
     RemoteWAL: () => RemoteWAL2,
+    makeDataStore: () => makeDataStore,
+    makeMetaStore: () => makeMetaStore,
+    makeRemoteWAL: () => makeRemoteWAL,
     testConfig: () => testConfig
   });
   var import_path = __require("path");
   var import_os = __require("os");
   var import_promises = __require("fs/promises");
-  var PACKAGE_VERSION2 = "0.16.10";
+  var PACKAGE_VERSION2 = "0.17.3";
   var match2 = PACKAGE_VERSION2.match(/^([^.]*\.[^.]*)/);
   if (!match2)
     throw new Error("invalid version: " + PACKAGE_VERSION2);
@@ -18859,14 +18856,14 @@ ${key.data.toString("base64")}
     processQueue = new CommitQueue2();
     constructor(loader) {
       this.loader = loader;
-      this.ready = (async () => {
+      this.ready = Promise.resolve().then(async () => {
         const walState = await this.load().catch((e) => {
           console.error("error loading wal", e);
           return null;
         });
         this.walState.operations = walState?.operations || [];
         this.walState.fileOperations = walState?.fileOperations || [];
-      })();
+      });
     }
     async enqueue(dbMeta, opts) {
       await this.ready;
@@ -18965,6 +18962,9 @@ ${key.data.toString("base64")}
       await rmlp;
     }
   };
+  var makeDataStore = (name7) => new DataStore2(name7);
+  var makeMetaStore = (loader) => new MetaStore2(loader.name);
+  var makeRemoteWAL = (loader) => new RemoteWAL2(loader);
   var RemoteWAL2 = class extends RemoteWAL {
     filePathForBranch(branch) {
       return (0, import_path.join)(MetaStore2.dataDir, this.loader.name, "wal", branch + ".json");
@@ -19034,7 +19034,7 @@ ${key.data.toString("base64")}
     return await (0, import_promises.writeFile)(path, data);
   }
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.0/node_modules/@web3-storage/pail/src/clock/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/clock/index.js
   var advance = async (blocks, head, event) => {
     const events = new EventFetcher(blocks);
     const headmap = new Map(head.map((cid) => [cid.toString(), cid]));
@@ -19160,7 +19160,7 @@ ${key.data.toString("base64")}
   };
   var shortLink = (l) => `${String(l).slice(0, 4)}..${String(l).slice(-4)}`;
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.0/node_modules/@web3-storage/pail/src/shard.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/shard.js
   var MaxKeyLength = 64;
   var MaxShardSize = 512 * 1024;
   var CID_TAG2 = new Token(Type.tag, 42);
@@ -19291,41 +19291,8 @@ ${key.data.toString("base64")}
       }
     }
   };
-  var encodedLength = (shard) => {
-    let entriesLength = 0;
-    for (const entry of shard.entries) {
-      entriesLength += entryEncodedLength(entry);
-    }
-    const tokens = [
-      new Token(Type.map, 3),
-      new Token(Type.string, "entries"),
-      new Token(Type.array, shard.entries.length),
-      new Token(Type.string, "maxKeyLength"),
-      new Token(Type.uint, shard.maxKeyLength),
-      new Token(Type.string, "maxSize"),
-      new Token(Type.uint, shard.maxSize)
-    ];
-    return tokensToLength(tokens) + entriesLength;
-  };
-  var entryEncodedLength = (entry) => {
-    const tokens = [
-      new Token(Type.array, entry.length),
-      new Token(Type.string, entry[0])
-    ];
-    if (Array.isArray(entry[1])) {
-      tokens.push(new Token(Type.array, entry[1].length));
-      for (const link2 of entry[1]) {
-        tokens.push(CID_TAG2);
-        tokens.push(new Token(Type.bytes, { length: link2.byteLength + 1 }));
-      }
-    } else {
-      tokens.push(CID_TAG2);
-      tokens.push(new Token(Type.bytes, { length: entry[1].byteLength + 1 }));
-    }
-    return tokensToLength(tokens);
-  };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.0/node_modules/@web3-storage/pail/src/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/index.js
   var put = async (blocks, root2, key, value) => {
     const shards = new ShardFetcher(blocks);
     const rshard = await shards.get(root2);
@@ -19528,15 +19495,118 @@ ${key.data.toString("base64")}
     return [shard];
   };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.0/node_modules/@web3-storage/pail/src/batch/shard.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/block.js
+  var MemoryBlockstore2 = class {
+    /** @type {Map<string, Uint8Array>} */
+    #blocks = /* @__PURE__ */ new Map();
+    /**
+     * @param {Array<import('multiformats').Block>} [blocks]
+     */
+    constructor(blocks) {
+      if (blocks) {
+        this.#blocks = new Map(blocks.map((b) => [b.cid.toString(), b.bytes]));
+      }
+    }
+    /** @type {API.BlockFetcher['get']} */
+    async get(cid) {
+      const bytes = this.#blocks.get(cid.toString());
+      if (!bytes)
+        return;
+      return { cid, bytes };
+    }
+    /**
+     * @param {API.UnknownLink} cid
+     * @param {Uint8Array} bytes
+     */
+    async put(cid, bytes) {
+      this.#blocks.set(cid.toString(), bytes);
+    }
+    /**
+     * @param {API.UnknownLink} cid
+     * @param {Uint8Array} bytes
+     */
+    putSync(cid, bytes) {
+      this.#blocks.set(cid.toString(), bytes);
+    }
+    /** @param {API.UnknownLink} cid */
+    async delete(cid) {
+      this.#blocks.delete(cid.toString());
+    }
+    /** @param {API.UnknownLink} cid */
+    deleteSync(cid) {
+      this.#blocks.delete(cid.toString());
+    }
+    *entries() {
+      for (const [str, bytes] of this.#blocks) {
+        yield { cid: parse(str), bytes };
+      }
+    }
+  };
+  var MultiBlockFetcher = class {
+    /** @type {API.BlockFetcher[]} */
+    #fetchers;
+    /** @param {API.BlockFetcher[]} fetchers */
+    constructor(...fetchers) {
+      this.#fetchers = fetchers;
+    }
+    /** @type {API.BlockFetcher['get']} */
+    async get(link2) {
+      for (const f of this.#fetchers) {
+        const v = await f.get(link2);
+        if (v)
+          return v;
+      }
+    }
+  };
+
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/batch/shard.js
+  var ShardLinkByteLength = 36;
+  var CID_TAG3 = new Token(Type.tag, 42);
   var create8 = (init2) => ({
     base: init2?.base,
     prefix: init2?.prefix ?? "",
-    entries: init2?.entries ?? [],
+    entries: [...init2?.entries ?? []],
     ...configure(init2)
   });
+  var encodedLength = (shard) => {
+    let entriesLength = 0;
+    for (const entry of shard.entries) {
+      entriesLength += entryEncodedLength(entry);
+    }
+    const tokens = [
+      new Token(Type.map, 3),
+      new Token(Type.string, "entries"),
+      new Token(Type.array, shard.entries.length),
+      new Token(Type.string, "maxKeyLength"),
+      new Token(Type.uint, shard.maxKeyLength),
+      new Token(Type.string, "maxSize"),
+      new Token(Type.uint, shard.maxSize)
+    ];
+    return tokensToLength(tokens) + entriesLength;
+  };
+  var entryEncodedLength = (entry) => {
+    const tokens = [
+      new Token(Type.array, entry.length),
+      new Token(Type.string, entry[0])
+    ];
+    if (Array.isArray(entry[1])) {
+      tokens.push(new Token(Type.array, entry[1].length));
+      for (const item of entry[1]) {
+        tokens.push(CID_TAG3);
+        if (isLink(item)) {
+          tokens.push(new Token(Type.bytes, { length: item.byteLength + 1 }));
+        } else {
+          tokens.push(new Token(Type.bytes, { length: ShardLinkByteLength + 1 }));
+        }
+      }
+    } else {
+      tokens.push(CID_TAG3);
+      tokens.push(new Token(Type.bytes, { length: entry[1].byteLength + 1 }));
+    }
+    return tokensToLength(tokens);
+  };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.0/node_modules/@web3-storage/pail/src/batch/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/batch/index.js
   var Batcher = class _Batcher {
     #committed = false;
     /**
@@ -19551,7 +19621,7 @@ ${key.data.toString("base64")}
     constructor({ blocks, entries: entries3, prefix, maxSize, maxKeyLength, base: base4 }) {
       this.blocks = blocks;
       this.prefix = prefix;
-      this.entries = entries3;
+      this.entries = [...entries3];
       this.base = base4;
       this.maxSize = maxSize;
       this.maxKeyLength = maxKeyLength;
@@ -19581,7 +19651,7 @@ ${key.data.toString("base64")}
     static async create({ blocks, link: link2, prefix }) {
       const shards = new ShardFetcher(blocks);
       const base4 = await shards.get(link2);
-      return new _Batcher({ blocks, entries: base4.value.entries, prefix, base: base4, ...configure(base4.value) });
+      return new _Batcher({ blocks, prefix, base: base4, ...base4.value });
     }
   };
   var put2 = async (blocks, shard, key, value) => {
@@ -19618,7 +19688,7 @@ ${key.data.toString("base64")}
       entry = [pfxskeys[0].key, [batcher]];
     }
     shard.entries = putEntry(asShardEntries(shard.entries), asShardEntry(entry));
-    const size = encodedLength(withEntries(asShardEntries(shard.entries), shard));
+    const size = encodedLength(shard);
     if (size > shard.maxSize) {
       const common = findCommonPrefix(
         asShardEntries(shard.entries),
@@ -19655,14 +19725,16 @@ ${key.data.toString("base64")}
     }
   };
   var traverse2 = async (shards, key, shard) => {
-    for (const e of shard.entries) {
-      const [k, v] = e;
+    for (let i = 0; i < shard.entries.length; i++) {
+      const [k, v] = shard.entries[i];
       if (key <= k)
         break;
       if (key.startsWith(k) && Array.isArray(v)) {
         if (isShardLink(v[0])) {
           const blk = await shards.get(v[0], shard.prefix + k);
-          v[0] = create8({ base: blk, prefix: blk.prefix, ...blk.value });
+          const batcher = create8({ base: blk, prefix: blk.prefix, ...blk.value });
+          shard.entries[i] = [k, v[1] == null ? [batcher] : [batcher, v[1]]];
+          return traverse2(shards, key.slice(k.length), batcher);
         }
         return traverse2(shards, key.slice(k.length), v[0]);
       }
@@ -19716,9 +19788,9 @@ ${key.data.toString("base64")}
     static code = "ERR_BATCH_COMMITTED";
   };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.0/node_modules/@web3-storage/pail/src/crdt/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/crdt/index.js
   var put3 = async (blocks, head, key, value) => {
-    const mblocks = new MemoryBlockstore();
+    const mblocks = new MemoryBlockstore2();
     blocks = new MultiBlockFetcher(mblocks, blocks);
     if (!head.length) {
       const shard = await ShardBlock.create();
@@ -19802,7 +19874,7 @@ ${key.data.toString("base64")}
   var root = async (blocks, head) => {
     if (!head.length)
       throw new Error("cannot determine root of headless clock");
-    const mblocks = new MemoryBlockstore();
+    const mblocks = new MemoryBlockstore2();
     blocks = new MultiBlockFetcher(mblocks, blocks);
     const events = new EventFetcher(blocks);
     if (head.length === 1) {
@@ -19861,7 +19933,7 @@ ${key.data.toString("base64")}
       return;
     const result = await root(blocks, head);
     if (result.additions.length) {
-      blocks = new MultiBlockFetcher(new MemoryBlockstore(result.additions), blocks);
+      blocks = new MultiBlockFetcher(new MemoryBlockstore2(result.additions), blocks);
     }
     return get2(blocks, result.root, key);
   };
@@ -19870,7 +19942,7 @@ ${key.data.toString("base64")}
       return;
     const result = await root(blocks, head);
     if (result.additions.length) {
-      blocks = new MultiBlockFetcher(new MemoryBlockstore(result.additions), blocks);
+      blocks = new MultiBlockFetcher(new MemoryBlockstore2(result.additions), blocks);
     }
     yield* entries(blocks, result.root, options);
   };
@@ -19954,7 +20026,7 @@ ${key.data.toString("base64")}
     return acc.concat(...rest);
   };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.0/node_modules/@web3-storage/pail/src/crdt/batch/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/crdt/batch/index.js
   var Batcher2 = class _Batcher {
     #committed = false;
     /**
@@ -19973,7 +20045,7 @@ ${key.data.toString("base64")}
       this.blocks = blocks;
       this.head = head;
       this.prefix = prefix;
-      this.entries = entries3;
+      this.entries = [...entries3];
       this.base = base4;
       this.maxSize = maxSize;
       this.maxKeyLength = maxKeyLength;
@@ -19999,7 +20071,7 @@ ${key.data.toString("base64")}
       const res = await commit(this);
       const data = { type: "batch", ops: this.ops, root: res.root };
       const event = await EventBlock.create(data, this.head);
-      const mblocks = new MemoryBlockstore();
+      const mblocks = new MemoryBlockstore2();
       const blocks = new MultiBlockFetcher(mblocks, this.blocks);
       mblocks.putSync(event.cid, event.bytes);
       const head = await advance(blocks, this.head, event.cid);
@@ -20039,7 +20111,7 @@ ${key.data.toString("base64")}
      * @param {string} init.prefix
      */
     static async create({ blocks, head, prefix }) {
-      const mblocks = new MemoryBlockstore();
+      const mblocks = new MemoryBlockstore2();
       blocks = new MultiBlockFetcher(mblocks, blocks);
       if (!head.length) {
         const base5 = await ShardBlock.create();
@@ -28480,10 +28552,8 @@ ${key.data.toString("base64")}
 
   // src/crdt-helpers.ts
   function time(tag2) {
-    console.time(tag2);
   }
   function timeEnd(tag2) {
-    console.timeEnd(tag2);
   }
   async function applyBulkUpdateToCrdt(tblocks, head, updates) {
     let result = null;
@@ -28610,6 +28680,7 @@ ${key.data.toString("base64")}
     if (!block)
       throw new Error(`Missing linked block ${link2.toString()}`);
     const { value } = await decode11({ bytes: block.bytes, hasher: sha256, codec: src_exports });
+    value.cid = link2;
     readFiles(blocks, value);
     return value;
   }
@@ -29531,7 +29602,7 @@ ${key.data.toString("base64")}
       try {
         head = await advance(tblocks, head, cid);
       } catch (e) {
-        console.error("failed to advance", cid.toString(), e);
+        console.log("failed to advance head:", cid.toString());
         continue;
       }
     }
@@ -29561,8 +29632,8 @@ ${key.data.toString("base64")}
           return { head: this.clock.head };
         },
         autoCompact: this.opts.autoCompact || 100,
-        crypto: crypto_node_exports,
-        store: store_node_exports,
+        crypto: this.opts.crypto || crypto_node_exports,
+        store: this.opts.store || store_node_exports,
         public: this.opts.public,
         meta: this.opts.meta
       });