@fireproof/core 0.17.4 → 0.17.6

package/dist/node/fireproof.global.js

@@ -4089,7 +4089,7 @@ var Fireproof = (() => {
   };
   var parse = (source, base4) => CID.parse(source, base4);
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.0/node_modules/@web3-storage/pail/src/block.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.2/node_modules/@web3-storage/pail/src/block.js
   var MemoryBlockstore = class {
     /** @type {Map<string, Uint8Array>} */
     #blocks = /* @__PURE__ */ new Map();
@@ -4136,6 +4136,22 @@ var Fireproof = (() => {
       }
     }
   };
+  var MultiBlockFetcher = class {
+    /** @type {API.BlockFetcher[]} */
+    #fetchers;
+    /** @param {API.BlockFetcher[]} fetchers */
+    constructor(...fetchers) {
+      this.#fetchers = fetchers;
+    }
+    /** @type {API.BlockFetcher['get']} */
+    async get(link2) {
+      for (const f of this.#fetchers) {
+        const v = await f.get(link2);
+        if (v)
+          return v;
+      }
+    }
+  };
 
   // ../../node_modules/.pnpm/yocto-queue@1.0.0/node_modules/yocto-queue/index.js
   var Node = class {
@@ -10013,7 +10029,7 @@ You can use close({ resize: true }) to resize header`);
     }
     return bytes;
   }
-  var PACKAGE_VERSION = "0.17.3";
+  var PACKAGE_VERSION = "0.17.5";
   var match = PACKAGE_VERSION.match(/^([^.]*\.[^.]*)/);
   if (!match)
     throw new Error("invalid version: " + PACKAGE_VERSION);
@@ -10083,6 +10099,7 @@ You can use close({ resize: true }) to resize header`);
     key;
     keyId;
     seenCompacted = /* @__PURE__ */ new Set();
+    processedCars = /* @__PURE__ */ new Set();
     writing = Promise.resolve();
     getBlockCache = /* @__PURE__ */ new Map();
     seenMeta = /* @__PURE__ */ new Set();
@@ -10171,7 +10188,7 @@ You can use close({ resize: true }) to resize header`);
     // can these skip the queue? or have a file queue?
     async _commitInternalFiles(t, done, opts = { noLoader: false, compact: false }) {
       await this.ready;
-      const { files: roots } = this.makeFileCarHeader(done, this.carLog, !!opts.compact);
+      const { files: roots } = this.makeFileCarHeader(done);
       const { cid, bytes } = await this.prepareCarFile(roots[0], t, !!opts.public);
       await this.fileStore.save({ cid, bytes });
       await this.remoteWAL.enqueueFile(cid, !!opts.public);
@@ -10191,7 +10208,10 @@ You can use close({ resize: true }) to resize header`);
         }
       }
     }
-    async cacheCarReader(reader) {
+    async cacheCarReader(carCidStr, reader) {
+      if (this.processedCars.has(carCidStr))
+        return;
+      this.processedCars.add(carCidStr);
       for await (const block of reader.blocks()) {
         const sBlock = block.cid.toString();
         if (!this.getBlockCache.has(sBlock)) {
@@ -10221,7 +10241,7 @@ You can use close({ resize: true }) to resize header`);
       const theKey = isPublic ? null : await this._getKey();
       return theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.ebOpts.crypto, theKey, root2, t) : await encodeCarFile([root2], t);
     }
-    makeFileCarHeader(result, cars, compact = false) {
+    makeFileCarHeader(result) {
       const files = [];
       for (const [, meta] of Object.entries(result.files)) {
         if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
@@ -10256,10 +10276,27 @@ You can use close({ resize: true }) to resize header`);
     //     await this.remoteWAL!.enqueue(dbMeta, { public: false })
     //   }
     // }
-    async *entries() {
+    async *entries(cache22 = true) {
       await this.ready;
-      for (const [, block] of this.getBlockCache) {
-        yield block;
+      if (cache22) {
+        for (const [, block] of this.getBlockCache) {
+          yield block;
+        }
+      } else {
+        for (const [, block] of this.getBlockCache) {
+          yield block;
+        }
+        for (const cid of this.carLog) {
+          const reader = await this.loadCar(cid);
+          if (!reader)
+            throw new Error(`missing car reader ${cid.toString()}`);
+          for await (const block of reader.blocks()) {
+            const sCid = block.cid.toString();
+            if (!this.getBlockCache.has(sCid)) {
+              yield block;
+            }
+          }
+        }
       }
     }
     async getBlock(cid) {
@@ -10272,11 +10309,37 @@ You can use close({ resize: true }) to resize header`);
         if (!reader) {
           throw new Error(`missing car reader ${carCid.toString()}`);
         }
-        await this.cacheCarReader(reader);
+        await this.cacheCarReader(carCid.toString(), reader).catch((e) => {
+        });
         if (this.getBlockCache.has(sCid))
           return this.getBlockCache.get(sCid);
         throw new Error(`block not in reader: ${cid.toString()}`);
       };
+      const getCompactCarCids = async (carCid) => {
+        const reader = await this.loadCar(carCid);
+        if (!reader) {
+          throw new Error(`missing car reader ${carCid.toString()}`);
+        }
+        const header = await parseCarFile(reader);
+        const compacts = header.compact;
+        let got2;
+        const batchSize2 = 5;
+        for (let i = 0; i < compacts.length; i += batchSize2) {
+          const promises = [];
+          for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
+            promises.push(getCarCid(compacts[j]));
+          }
+          try {
+            got2 = await Promise.any(promises);
+          } catch {
+          }
+          if (got2)
+            break;
+        }
+        if (this.getBlockCache.has(sCid))
+          return this.getBlockCache.get(sCid);
+        throw new Error(`block not in compact reader: ${cid.toString()}`);
+      };
       let got;
       const batchSize = 5;
       for (let i = 0; i < this.carLog.length; i += batchSize) {
@@ -10291,8 +10354,19 @@ You can use close({ resize: true }) to resize header`);
         if (got)
           break;
       }
-      if (got) {
-        this.getBlockCache.set(sCid, got);
+      if (!got) {
+        for (let i = 0; i < this.carLog.length; i += batchSize) {
+          const promises = [];
+          for (let j = i; j < Math.min(i + batchSize, this.carLog.length); j++) {
+            promises.push(getCompactCarCids(this.carLog[j]));
+          }
+          try {
+            got = await Promise.any(promises);
+          } catch {
+          }
+          if (got)
+            break;
+        }
       }
       return got;
     }
@@ -10399,6 +10473,7 @@ You can use close({ resize: true }) to resize header`);
     compacting = false;
     ebOpts;
     transactions = /* @__PURE__ */ new Set();
+    lastTxMeta = null;
     constructor(ebOpts) {
       this.ebOpts = ebOpts;
       const { name: name7 } = ebOpts;
@@ -10413,6 +10488,7 @@ You can use close({ resize: true }) to resize header`);
     async transaction(fn, opts = { noLoader: false }) {
       const t = new CarTransaction(this);
       const done = await fn(t);
+      this.lastTxMeta = done;
       if (this.loader) {
         const car = await this.loader.commit(t, done, opts);
         if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
@@ -10458,7 +10534,7 @@ You can use close({ resize: true }) to resize header`);
         throw new Error("loader required to compact");
       if (this.loader.carLog.length < 2)
         return;
-      const compactFn = this.ebOpts.compact;
+      const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks));
       if (!compactFn || this.compacting)
         return;
       const blockLog = new CompactionFetcher(this);
@@ -10470,6 +10546,23 @@ You can use close({ resize: true }) to resize header`);
       });
       this.compacting = false;
     }
+    async defaultCompact(blocks) {
+      if (!this.loader) {
+        throw new Error("no loader");
+      }
+      if (!this.lastTxMeta) {
+        throw new Error("no lastTxMeta");
+      }
+      for await (const blk of this.loader.entries(false)) {
+        blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
+      }
+      for (const t of this.transactions) {
+        for await (const blk of t.entries()) {
+          blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
+        }
+      }
+      return this.lastTxMeta;
+    }
     async *entries() {
       const seen = /* @__PURE__ */ new Set();
       if (this.loader) {
@@ -18768,7 +18861,7 @@ ${key.data.toString("base64")}
   var import_path = __require("path");
   var import_os = __require("os");
   var import_promises = __require("fs/promises");
-  var PACKAGE_VERSION2 = "0.17.3";
+  var PACKAGE_VERSION2 = "0.17.5";
   var match2 = PACKAGE_VERSION2.match(/^([^.]*\.[^.]*)/);
   if (!match2)
     throw new Error("invalid version: " + PACKAGE_VERSION2);
@@ -18906,7 +18999,7 @@ ${key.data.toString("base64")}
             const car = await this.loader.carStore.load(dbMeta.car).catch(() => null);
             if (!car) {
               if (cidListIncludes2(this.loader.carLog, dbMeta.car))
-                throw new Error(`missing car ${dbMeta.car.toString()}`);
+                throw new Error(`missing local car ${dbMeta.car.toString()}`);
             } else {
               await this.loader.remoteCarStore.save(car);
             }
@@ -18919,7 +19012,7 @@ ${key.data.toString("base64")}
             const car = await this.loader.carStore.load(dbMeta.car).catch(() => null);
             if (!car) {
               if (cidListIncludes2(this.loader.carLog, dbMeta.car))
-                throw new Error(`missing car ${dbMeta.car.toString()}`);
+                throw new Error(`missing local car ${dbMeta.car.toString()}`);
             } else {
               await this.loader.remoteCarStore.save(car);
             }
@@ -18944,7 +19037,7 @@ ${key.data.toString("base64")}
           const res = await Promise.allSettled(uploads);
           const errors = res.filter((r) => r.status === "rejected");
           if (errors.length) {
-            console.error("error uploading", JSON.stringify(errors));
+            console.error("error uploading", JSON.stringify(errors.map((e) => e.reason)));
             throw errors[0].reason;
           }
           if (operations.length) {
@@ -19034,7 +19127,7 @@ ${key.data.toString("base64")}
     return await (0, import_promises.writeFile)(path, data);
   }
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/clock/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.2/node_modules/@web3-storage/pail/src/clock/index.js
   var advance = async (blocks, head, event) => {
     const events = new EventFetcher(blocks);
     const headmap = new Map(head.map((cid) => [cid.toString(), cid]));
@@ -19160,7 +19253,7 @@ ${key.data.toString("base64")}
   };
   var shortLink = (l) => `${String(l).slice(0, 4)}..${String(l).slice(-4)}`;
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/shard.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.2/node_modules/@web3-storage/pail/src/shard.js
   var MaxKeyLength = 64;
   var MaxShardSize = 512 * 1024;
   var CID_TAG2 = new Token(Type.tag, 42);
@@ -19292,7 +19385,7 @@ ${key.data.toString("base64")}
     }
   };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.2/node_modules/@web3-storage/pail/src/index.js
   var put = async (blocks, root2, key, value) => {
     const shards = new ShardFetcher(blocks);
     const rshard = await shards.get(root2);
@@ -19495,71 +19588,7 @@ ${key.data.toString("base64")}
     return [shard];
   };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/block.js
-  var MemoryBlockstore2 = class {
-    /** @type {Map<string, Uint8Array>} */
-    #blocks = /* @__PURE__ */ new Map();
-    /**
-     * @param {Array<import('multiformats').Block>} [blocks]
-     */
-    constructor(blocks) {
-      if (blocks) {
-        this.#blocks = new Map(blocks.map((b) => [b.cid.toString(), b.bytes]));
-      }
-    }
-    /** @type {API.BlockFetcher['get']} */
-    async get(cid) {
-      const bytes = this.#blocks.get(cid.toString());
-      if (!bytes)
-        return;
-      return { cid, bytes };
-    }
-    /**
-     * @param {API.UnknownLink} cid
-     * @param {Uint8Array} bytes
-     */
-    async put(cid, bytes) {
-      this.#blocks.set(cid.toString(), bytes);
-    }
-    /**
-     * @param {API.UnknownLink} cid
-     * @param {Uint8Array} bytes
-     */
-    putSync(cid, bytes) {
-      this.#blocks.set(cid.toString(), bytes);
-    }
-    /** @param {API.UnknownLink} cid */
-    async delete(cid) {
-      this.#blocks.delete(cid.toString());
-    }
-    /** @param {API.UnknownLink} cid */
-    deleteSync(cid) {
-      this.#blocks.delete(cid.toString());
-    }
-    *entries() {
-      for (const [str, bytes] of this.#blocks) {
-        yield { cid: parse(str), bytes };
-      }
-    }
-  };
-  var MultiBlockFetcher = class {
-    /** @type {API.BlockFetcher[]} */
-    #fetchers;
-    /** @param {API.BlockFetcher[]} fetchers */
-    constructor(...fetchers) {
-      this.#fetchers = fetchers;
-    }
-    /** @type {API.BlockFetcher['get']} */
-    async get(link2) {
-      for (const f of this.#fetchers) {
-        const v = await f.get(link2);
-        if (v)
-          return v;
-      }
-    }
-  };
-
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/batch/shard.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.2/node_modules/@web3-storage/pail/src/batch/shard.js
   var ShardLinkByteLength = 36;
   var CID_TAG3 = new Token(Type.tag, 42);
   var create8 = (init2) => ({
@@ -19606,7 +19635,7 @@ ${key.data.toString("base64")}
     return tokensToLength(tokens);
   };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/batch/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.2/node_modules/@web3-storage/pail/src/batch/index.js
   var Batcher = class _Batcher {
     #committed = false;
     /**
@@ -19788,9 +19817,9 @@ ${key.data.toString("base64")}
     static code = "ERR_BATCH_COMMITTED";
   };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/crdt/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.2/node_modules/@web3-storage/pail/src/crdt/index.js
   var put3 = async (blocks, head, key, value) => {
-    const mblocks = new MemoryBlockstore2();
+    const mblocks = new MemoryBlockstore();
     blocks = new MultiBlockFetcher(mblocks, blocks);
     if (!head.length) {
       const shard = await ShardBlock.create();
@@ -19874,7 +19903,7 @@ ${key.data.toString("base64")}
   var root = async (blocks, head) => {
     if (!head.length)
       throw new Error("cannot determine root of headless clock");
-    const mblocks = new MemoryBlockstore2();
+    const mblocks = new MemoryBlockstore();
     blocks = new MultiBlockFetcher(mblocks, blocks);
     const events = new EventFetcher(blocks);
     if (head.length === 1) {
@@ -19933,7 +19962,7 @@ ${key.data.toString("base64")}
       return;
     const result = await root(blocks, head);
     if (result.additions.length) {
-      blocks = new MultiBlockFetcher(new MemoryBlockstore2(result.additions), blocks);
+      blocks = new MultiBlockFetcher(new MemoryBlockstore(result.additions), blocks);
     }
     return get2(blocks, result.root, key);
   };
@@ -19942,7 +19971,7 @@ ${key.data.toString("base64")}
       return;
     const result = await root(blocks, head);
     if (result.additions.length) {
-      blocks = new MultiBlockFetcher(new MemoryBlockstore2(result.additions), blocks);
+      blocks = new MultiBlockFetcher(new MemoryBlockstore(result.additions), blocks);
     }
     yield* entries(blocks, result.root, options);
   };
@@ -20026,7 +20055,7 @@ ${key.data.toString("base64")}
     return acc.concat(...rest);
   };
 
-  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.1/node_modules/@web3-storage/pail/src/crdt/batch/index.js
+  // ../../node_modules/.pnpm/@web3-storage+pail@0.4.2/node_modules/@web3-storage/pail/src/crdt/batch/index.js
   var Batcher2 = class _Batcher {
     #committed = false;
     /**
@@ -20071,7 +20100,7 @@ ${key.data.toString("base64")}
       const res = await commit(this);
       const data = { type: "batch", ops: this.ops, root: res.root };
       const event = await EventBlock.create(data, this.head);
-      const mblocks = new MemoryBlockstore2();
+      const mblocks = new MemoryBlockstore();
       const blocks = new MultiBlockFetcher(mblocks, this.blocks);
       mblocks.putSync(event.cid, event.bytes);
       const head = await advance(blocks, this.head, event.cid);
@@ -20111,7 +20140,7 @@ ${key.data.toString("base64")}
      * @param {string} init.prefix
      */
     static async create({ blocks, head, prefix }) {
-      const mblocks = new MemoryBlockstore2();
+      const mblocks = new MemoryBlockstore();
       blocks = new MultiBlockFetcher(mblocks, blocks);
       if (!head.length) {
         const base5 = await ShardBlock.create();
@@ -28771,7 +28800,6 @@ ${key.data.toString("base64")}
   var isCompacting = false;
   async function doCompact(blockLog, head) {
     if (isCompacting) {
-      console.log("already compacting");
       return;
     }
     isCompacting = true;
@@ -29478,7 +29506,10 @@ ${key.data.toString("base64")}
           const task = queue.shift();
           if (!task)
             continue;
-          await worker(task.newHead, task.prevHead, task.updates !== null);
+          await worker(task.newHead, task.prevHead, task.updates !== null).catch((e) => {
+            console.error("int_applyHead worker error", e);
+            throw e;
+          });
           if (task.updates) {
             allUpdates.push(...task.updates);
           }
@@ -29501,6 +29532,9 @@ ${key.data.toString("base64")}
       push(task) {
         queue.push(task);
         return process2();
+      },
+      size() {
+        return queue.length;
       }
     };
   }
@@ -29602,7 +29636,6 @@ ${key.data.toString("base64")}
       try {
         head = await advance(tblocks, head, cid);
       } catch (e) {
-        console.log("failed to advance head:", cid.toString());
         continue;
       }
     }