@fireproof/core 0.17.5 → 0.17.6

package/dist/node/fireproof.global.js

@@ -10029,7 +10029,7 @@ You can use close({ resize: true }) to resize header`);
     }
     return bytes;
   }
-  var PACKAGE_VERSION = "0.17.4";
+  var PACKAGE_VERSION = "0.17.5";
   var match = PACKAGE_VERSION.match(/^([^.]*\.[^.]*)/);
   if (!match)
     throw new Error("invalid version: " + PACKAGE_VERSION);
@@ -10099,6 +10099,7 @@ You can use close({ resize: true }) to resize header`);
     key;
     keyId;
     seenCompacted = /* @__PURE__ */ new Set();
+    processedCars = /* @__PURE__ */ new Set();
     writing = Promise.resolve();
     getBlockCache = /* @__PURE__ */ new Map();
     seenMeta = /* @__PURE__ */ new Set();
@@ -10187,7 +10188,7 @@ You can use close({ resize: true }) to resize header`);
     // can these skip the queue? or have a file queue?
     async _commitInternalFiles(t, done, opts = { noLoader: false, compact: false }) {
       await this.ready;
-      const { files: roots } = this.makeFileCarHeader(done, this.carLog, !!opts.compact);
+      const { files: roots } = this.makeFileCarHeader(done);
       const { cid, bytes } = await this.prepareCarFile(roots[0], t, !!opts.public);
       await this.fileStore.save({ cid, bytes });
       await this.remoteWAL.enqueueFile(cid, !!opts.public);
@@ -10207,7 +10208,10 @@ You can use close({ resize: true }) to resize header`);
         }
       }
     }
-    async cacheCarReader(reader) {
+    async cacheCarReader(carCidStr, reader) {
+      if (this.processedCars.has(carCidStr))
+        return;
+      this.processedCars.add(carCidStr);
       for await (const block of reader.blocks()) {
         const sBlock = block.cid.toString();
         if (!this.getBlockCache.has(sBlock)) {
@@ -10237,7 +10241,7 @@ You can use close({ resize: true }) to resize header`);
       const theKey = isPublic ? null : await this._getKey();
       return theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.ebOpts.crypto, theKey, root2, t) : await encodeCarFile([root2], t);
     }
-    makeFileCarHeader(result, cars, compact = false) {
+    makeFileCarHeader(result) {
       const files = [];
       for (const [, meta] of Object.entries(result.files)) {
         if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
@@ -10272,10 +10276,27 @@ You can use close({ resize: true }) to resize header`);
     //     await this.remoteWAL!.enqueue(dbMeta, { public: false })
     //   }
     // }
-    async *entries() {
+    async *entries(cache22 = true) {
       await this.ready;
-      for (const [, block] of this.getBlockCache) {
-        yield block;
+      if (cache22) {
+        for (const [, block] of this.getBlockCache) {
+          yield block;
+        }
+      } else {
+        for (const [, block] of this.getBlockCache) {
+          yield block;
+        }
+        for (const cid of this.carLog) {
+          const reader = await this.loadCar(cid);
+          if (!reader)
+            throw new Error(`missing car reader ${cid.toString()}`);
+          for await (const block of reader.blocks()) {
+            const sCid = block.cid.toString();
+            if (!this.getBlockCache.has(sCid)) {
+              yield block;
+            }
+          }
+        }
       }
     }
     async getBlock(cid) {
@@ -10288,11 +10309,37 @@ You can use close({ resize: true }) to resize header`);
         if (!reader) {
           throw new Error(`missing car reader ${carCid.toString()}`);
         }
-        await this.cacheCarReader(reader);
+        await this.cacheCarReader(carCid.toString(), reader).catch((e) => {
+        });
         if (this.getBlockCache.has(sCid))
           return this.getBlockCache.get(sCid);
         throw new Error(`block not in reader: ${cid.toString()}`);
       };
+      const getCompactCarCids = async (carCid) => {
+        const reader = await this.loadCar(carCid);
+        if (!reader) {
+          throw new Error(`missing car reader ${carCid.toString()}`);
+        }
+        const header = await parseCarFile(reader);
+        const compacts = header.compact;
+        let got2;
+        const batchSize2 = 5;
+        for (let i = 0; i < compacts.length; i += batchSize2) {
+          const promises = [];
+          for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
+            promises.push(getCarCid(compacts[j]));
+          }
+          try {
+            got2 = await Promise.any(promises);
+          } catch {
+          }
+          if (got2)
+            break;
+        }
+        if (this.getBlockCache.has(sCid))
+          return this.getBlockCache.get(sCid);
+        throw new Error(`block not in compact reader: ${cid.toString()}`);
+      };
       let got;
       const batchSize = 5;
       for (let i = 0; i < this.carLog.length; i += batchSize) {
@@ -10307,8 +10354,19 @@ You can use close({ resize: true }) to resize header`);
         if (got)
           break;
       }
-      if (got) {
-        this.getBlockCache.set(sCid, got);
+      if (!got) {
+        for (let i = 0; i < this.carLog.length; i += batchSize) {
+          const promises = [];
+          for (let j = i; j < Math.min(i + batchSize, this.carLog.length); j++) {
+            promises.push(getCompactCarCids(this.carLog[j]));
+          }
+          try {
+            got = await Promise.any(promises);
+          } catch {
+          }
+          if (got)
+            break;
+        }
       }
       return got;
     }
@@ -10415,6 +10473,7 @@ You can use close({ resize: true }) to resize header`);
     compacting = false;
     ebOpts;
     transactions = /* @__PURE__ */ new Set();
+    lastTxMeta = null;
     constructor(ebOpts) {
       this.ebOpts = ebOpts;
       const { name: name7 } = ebOpts;
@@ -10429,6 +10488,7 @@ You can use close({ resize: true }) to resize header`);
     async transaction(fn, opts = { noLoader: false }) {
       const t = new CarTransaction(this);
       const done = await fn(t);
+      this.lastTxMeta = done;
       if (this.loader) {
         const car = await this.loader.commit(t, done, opts);
         if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
@@ -10474,7 +10534,7 @@ You can use close({ resize: true }) to resize header`);
         throw new Error("loader required to compact");
       if (this.loader.carLog.length < 2)
         return;
-      const compactFn = this.ebOpts.compact;
+      const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks));
       if (!compactFn || this.compacting)
         return;
       const blockLog = new CompactionFetcher(this);
@@ -10486,6 +10546,23 @@ You can use close({ resize: true }) to resize header`);
       });
       this.compacting = false;
     }
+    async defaultCompact(blocks) {
+      if (!this.loader) {
+        throw new Error("no loader");
+      }
+      if (!this.lastTxMeta) {
+        throw new Error("no lastTxMeta");
+      }
+      for await (const blk of this.loader.entries(false)) {
+        blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
+      }
+      for (const t of this.transactions) {
+        for await (const blk of t.entries()) {
+          blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
+        }
+      }
+      return this.lastTxMeta;
+    }
     async *entries() {
       const seen = /* @__PURE__ */ new Set();
       if (this.loader) {
@@ -18784,7 +18861,7 @@ ${key.data.toString("base64")}
   var import_path = __require("path");
   var import_os = __require("os");
   var import_promises = __require("fs/promises");
-  var PACKAGE_VERSION2 = "0.17.4";
+  var PACKAGE_VERSION2 = "0.17.5";
   var match2 = PACKAGE_VERSION2.match(/^([^.]*\.[^.]*)/);
   if (!match2)
     throw new Error("invalid version: " + PACKAGE_VERSION2);
@@ -18922,7 +18999,7 @@ ${key.data.toString("base64")}
             const car = await this.loader.carStore.load(dbMeta.car).catch(() => null);
             if (!car) {
               if (cidListIncludes2(this.loader.carLog, dbMeta.car))
-                throw new Error(`missing car ${dbMeta.car.toString()}`);
+                throw new Error(`missing local car ${dbMeta.car.toString()}`);
             } else {
               await this.loader.remoteCarStore.save(car);
             }
@@ -18935,7 +19012,7 @@ ${key.data.toString("base64")}
             const car = await this.loader.carStore.load(dbMeta.car).catch(() => null);
             if (!car) {
               if (cidListIncludes2(this.loader.carLog, dbMeta.car))
-                throw new Error(`missing car ${dbMeta.car.toString()}`);
+                throw new Error(`missing local car ${dbMeta.car.toString()}`);
             } else {
               await this.loader.remoteCarStore.save(car);
             }
@@ -18960,7 +19037,7 @@ ${key.data.toString("base64")}
           const res = await Promise.allSettled(uploads);
           const errors = res.filter((r) => r.status === "rejected");
           if (errors.length) {
-            console.error("error uploading", JSON.stringify(errors));
+            console.error("error uploading", JSON.stringify(errors.map((e) => e.reason)));
             throw errors[0].reason;
           }
           if (operations.length) {
@@ -28723,7 +28800,6 @@ ${key.data.toString("base64")}
   var isCompacting = false;
   async function doCompact(blockLog, head) {
     if (isCompacting) {
-      console.log("already compacting");
       return;
     }
     isCompacting = true;
@@ -29430,7 +29506,10 @@ ${key.data.toString("base64")}
           const task = queue.shift();
           if (!task)
             continue;
-          await worker(task.newHead, task.prevHead, task.updates !== null);
+          await worker(task.newHead, task.prevHead, task.updates !== null).catch((e) => {
+            console.error("int_applyHead worker error", e);
+            throw e;
+          });
           if (task.updates) {
             allUpdates.push(...task.updates);
           }
@@ -29453,6 +29532,9 @@ ${key.data.toString("base64")}
       push(task) {
         queue.push(task);
         return process2();
+      },
+      size() {
+        return queue.length;
       }
     };
   }
@@ -29554,7 +29636,6 @@ ${key.data.toString("base64")}
       try {
         head = await advance(tblocks, head, cid);
       } catch (e) {
-        console.log("failed to advance head:", cid.toString());
         continue;
       }
     }