@fireproof/core 0.19.11-dev-dryrun6 → 0.19.99-dev-100

package/index.global.js

@@ -12614,6 +12614,237 @@ ${end.comment}` : end.comment;
     }
   });
 
+  // node_modules/.pnpm/retry@0.13.1/node_modules/retry/lib/retry_operation.js
+  var require_retry_operation = __commonJS({
+    "node_modules/.pnpm/retry@0.13.1/node_modules/retry/lib/retry_operation.js"(exports2, module2) {
+      "use strict";
+      function RetryOperation(timeouts, options) {
+        if (typeof options === "boolean") {
+          options = { forever: options };
+        }
+        this._originalTimeouts = JSON.parse(JSON.stringify(timeouts));
+        this._timeouts = timeouts;
+        this._options = options || {};
+        this._maxRetryTime = options && options.maxRetryTime || Infinity;
+        this._fn = null;
+        this._errors = [];
+        this._attempts = 1;
+        this._operationTimeout = null;
+        this._operationTimeoutCb = null;
+        this._timeout = null;
+        this._operationStart = null;
+        this._timer = null;
+        if (this._options.forever) {
+          this._cachedTimeouts = this._timeouts.slice(0);
+        }
+      }
+      module2.exports = RetryOperation;
+      RetryOperation.prototype.reset = function() {
+        this._attempts = 1;
+        this._timeouts = this._originalTimeouts.slice(0);
+      };
+      RetryOperation.prototype.stop = function() {
+        if (this._timeout) {
+          clearTimeout(this._timeout);
+        }
+        if (this._timer) {
+          clearTimeout(this._timer);
+        }
+        this._timeouts = [];
+        this._cachedTimeouts = null;
+      };
+      RetryOperation.prototype.retry = function(err) {
+        if (this._timeout) {
+          clearTimeout(this._timeout);
+        }
+        if (!err) {
+          return false;
+        }
+        var currentTime = (/* @__PURE__ */ new Date()).getTime();
+        if (err && currentTime - this._operationStart >= this._maxRetryTime) {
+          this._errors.push(err);
+          this._errors.unshift(new Error("RetryOperation timeout occurred"));
+          return false;
+        }
+        this._errors.push(err);
+        var timeout = this._timeouts.shift();
+        if (timeout === void 0) {
+          if (this._cachedTimeouts) {
+            this._errors.splice(0, this._errors.length - 1);
+            timeout = this._cachedTimeouts.slice(-1);
+          } else {
+            return false;
+          }
+        }
+        var self2 = this;
+        this._timer = setTimeout(function() {
+          self2._attempts++;
+          if (self2._operationTimeoutCb) {
+            self2._timeout = setTimeout(function() {
+              self2._operationTimeoutCb(self2._attempts);
+            }, self2._operationTimeout);
+            if (self2._options.unref) {
+              self2._timeout.unref();
+            }
+          }
+          self2._fn(self2._attempts);
+        }, timeout);
+        if (this._options.unref) {
+          this._timer.unref();
+        }
+        return true;
+      };
+      RetryOperation.prototype.attempt = function(fn, timeoutOps) {
+        this._fn = fn;
+        if (timeoutOps) {
+          if (timeoutOps.timeout) {
+            this._operationTimeout = timeoutOps.timeout;
+          }
+          if (timeoutOps.cb) {
+            this._operationTimeoutCb = timeoutOps.cb;
+          }
+        }
+        var self2 = this;
+        if (this._operationTimeoutCb) {
+          this._timeout = setTimeout(function() {
+            self2._operationTimeoutCb();
+          }, self2._operationTimeout);
+        }
+        this._operationStart = (/* @__PURE__ */ new Date()).getTime();
+        this._fn(this._attempts);
+      };
+      RetryOperation.prototype.try = function(fn) {
+        console.log("Using RetryOperation.try() is deprecated");
+        this.attempt(fn);
+      };
+      RetryOperation.prototype.start = function(fn) {
+        console.log("Using RetryOperation.start() is deprecated");
+        this.attempt(fn);
+      };
+      RetryOperation.prototype.start = RetryOperation.prototype.try;
+      RetryOperation.prototype.errors = function() {
+        return this._errors;
+      };
+      RetryOperation.prototype.attempts = function() {
+        return this._attempts;
+      };
+      RetryOperation.prototype.mainError = function() {
+        if (this._errors.length === 0) {
+          return null;
+        }
+        var counts = {};
+        var mainError = null;
+        var mainErrorCount = 0;
+        for (var i = 0; i < this._errors.length; i++) {
+          var error = this._errors[i];
+          var message2 = error.message;
+          var count = (counts[message2] || 0) + 1;
+          counts[message2] = count;
+          if (count >= mainErrorCount) {
+            mainError = error;
+            mainErrorCount = count;
+          }
+        }
+        return mainError;
+      };
+    }
+  });
+
+  // node_modules/.pnpm/retry@0.13.1/node_modules/retry/lib/retry.js
+  var require_retry = __commonJS({
+    "node_modules/.pnpm/retry@0.13.1/node_modules/retry/lib/retry.js"(exports2) {
+      "use strict";
+      var RetryOperation = require_retry_operation();
+      exports2.operation = function(options) {
+        var timeouts = exports2.timeouts(options);
+        return new RetryOperation(timeouts, {
+          forever: options && (options.forever || options.retries === Infinity),
+          unref: options && options.unref,
+          maxRetryTime: options && options.maxRetryTime
+        });
+      };
+      exports2.timeouts = function(options) {
+        if (options instanceof Array) {
+          return [].concat(options);
+        }
+        var opts = {
+          retries: 10,
+          factor: 2,
+          minTimeout: 1 * 1e3,
+          maxTimeout: Infinity,
+          randomize: false
+        };
+        for (var key in options) {
+          opts[key] = options[key];
+        }
+        if (opts.minTimeout > opts.maxTimeout) {
+          throw new Error("minTimeout is greater than maxTimeout");
+        }
+        var timeouts = [];
+        for (var i = 0; i < opts.retries; i++) {
+          timeouts.push(this.createTimeout(i, opts));
+        }
+        if (options && options.forever && !timeouts.length) {
+          timeouts.push(this.createTimeout(i, opts));
+        }
+        timeouts.sort(function(a, b) {
+          return a - b;
+        });
+        return timeouts;
+      };
+      exports2.createTimeout = function(attempt, opts) {
+        var random = opts.randomize ? Math.random() + 1 : 1;
+        var timeout = Math.round(random * Math.max(opts.minTimeout, 1) * Math.pow(opts.factor, attempt));
+        timeout = Math.min(timeout, opts.maxTimeout);
+        return timeout;
+      };
+      exports2.wrap = function(obj, options, methods) {
+        if (options instanceof Array) {
+          methods = options;
+          options = null;
+        }
+        if (!methods) {
+          methods = [];
+          for (var key in obj) {
+            if (typeof obj[key] === "function") {
+              methods.push(key);
+            }
+          }
+        }
+        for (var i = 0; i < methods.length; i++) {
+          var method = methods[i];
+          var original = obj[method];
+          obj[method] = function retryWrapper(original2) {
+            var op = exports2.operation(options);
+            var args = Array.prototype.slice.call(arguments, 1);
+            var callback = args.pop();
+            args.push(function(err) {
+              if (op.retry(err)) {
+                return;
+              }
+              if (err) {
+                arguments[0] = op.mainError();
+              }
+              callback.apply(this, arguments);
+            });
+            op.attempt(function() {
+              original2.apply(obj, args);
+            });
+          }.bind(obj, original);
+          obj[method].options = options;
+        }
+      };
+    }
+  });
+
+  // node_modules/.pnpm/retry@0.13.1/node_modules/retry/index.js
+  var require_retry2 = __commonJS({
+    "node_modules/.pnpm/retry@0.13.1/node_modules/retry/index.js"(exports2, module2) {
+      "use strict";
+      module2.exports = require_retry();
+    }
+  });
+
   // src/runtime/gateways/indexdb/version.ts
   var INDEXDB_VERSION;
   var init_version = __esm({
@@ -24772,6 +25003,29 @@ ${end.comment}` : end.comment;
     }
   };
 
+  // src/blockstore/store.ts
+  init_cement();
+
+  // src/types.ts
+  function isFalsy(value) {
+    return value === false && value === null && value === void 0;
+  }
+  function throwFalsy(value) {
+    if (isFalsy(value)) {
+      throw new Error("value is Falsy");
+    }
+    return value;
+  }
+  function falsyToUndef(value) {
+    if (isFalsy(value)) {
+      return void 0;
+    }
+    return value;
+  }
+
+  // src/blockstore/store.ts
+  init_utils();
+
   // node_modules/.pnpm/yocto-queue@1.1.1/node_modules/yocto-queue/index.js
   var Node2 = class {
     value;
@@ -24903,29 +25157,6 @@ ${end.comment}` : end.comment;
     }
   }
 
-  // src/blockstore/store.ts
-  init_cement();
-
-  // src/types.ts
-  function isFalsy(value) {
-    return value === false && value === null && value === void 0;
-  }
-  function throwFalsy(value) {
-    if (isFalsy(value)) {
-      throw new Error("value is Falsy");
-    }
-    return value;
-  }
-  function falsyToUndef(value) {
-    if (isFalsy(value)) {
-      return void 0;
-    }
-    return value;
-  }
-
-  // src/blockstore/store.ts
-  init_utils();
-
   // node_modules/.pnpm/@ipld+car@5.3.2/node_modules/@ipld/car/src/decoder-common.js
   var import_varint2 = __toESM(require_varint(), 1);
   var CIDV0_BYTES = {
@@ -27111,6 +27342,232 @@ You can use close({ resize: true }) to resize header`);
     return sthis.txt.encode(JSON.stringify(crdtEntries));
   }
 
+  // node_modules/.pnpm/p-retry@6.2.0/node_modules/p-retry/index.js
+  var import_retry = __toESM(require_retry2(), 1);
+
+  // node_modules/.pnpm/is-network-error@1.1.0/node_modules/is-network-error/index.js
+  var objectToString = Object.prototype.toString;
+  var isError = (value) => objectToString.call(value) === "[object Error]";
+  var errorMessages = /* @__PURE__ */ new Set([
+    "network error",
+    // Chrome
+    "Failed to fetch",
+    // Chrome
+    "NetworkError when attempting to fetch resource.",
+    // Firefox
+    "The Internet connection appears to be offline.",
+    // Safari 16
+    "Load failed",
+    // Safari 17+
+    "Network request failed",
+    // `cross-fetch`
+    "fetch failed",
+    // Undici (Node.js)
+    "terminated"
+    // Undici (Node.js)
+  ]);
+  function isNetworkError(error) {
+    const isValid = error && isError(error) && error.name === "TypeError" && typeof error.message === "string";
+    if (!isValid) {
+      return false;
+    }
+    if (error.message === "Load failed") {
+      return error.stack === void 0;
+    }
+    return errorMessages.has(error.message);
+  }
+
+  // node_modules/.pnpm/p-retry@6.2.0/node_modules/p-retry/index.js
+  var AbortError3 = class extends Error {
+    constructor(message2) {
+      super();
+      if (message2 instanceof Error) {
+        this.originalError = message2;
+        ({ message: message2 } = message2);
+      } else {
+        this.originalError = new Error(message2);
+        this.originalError.stack = this.stack;
+      }
+      this.name = "AbortError";
+      this.message = message2;
+    }
+  };
+  var decorateErrorWithCounts = (error, attemptNumber, options) => {
+    const retriesLeft = options.retries - (attemptNumber - 1);
+    error.attemptNumber = attemptNumber;
+    error.retriesLeft = retriesLeft;
+    return error;
+  };
+  async function pRetry(input, options) {
+    return new Promise((resolve7, reject) => {
+      options = {
+        onFailedAttempt() {
+        },
+        retries: 10,
+        shouldRetry: () => true,
+        ...options
+      };
+      const operation = import_retry.default.operation(options);
+      const abortHandler = () => {
+        operation.stop();
+        reject(options.signal?.reason);
+      };
+      if (options.signal && !options.signal.aborted) {
+        options.signal.addEventListener("abort", abortHandler, { once: true });
+      }
+      const cleanUp = () => {
+        options.signal?.removeEventListener("abort", abortHandler);
+        operation.stop();
+      };
+      operation.attempt(async (attemptNumber) => {
+        try {
+          const result = await input(attemptNumber);
+          cleanUp();
+          resolve7(result);
+        } catch (error) {
+          try {
+            if (!(error instanceof Error)) {
+              throw new TypeError(`Non-error was thrown: "${error}". You should only throw errors.`);
+            }
+            if (error instanceof AbortError3) {
+              throw error.originalError;
+            }
+            if (error instanceof TypeError && !isNetworkError(error)) {
+              throw error;
+            }
+            decorateErrorWithCounts(error, attemptNumber, options);
+            if (!await options.shouldRetry(error)) {
+              operation.stop();
+              reject(error);
+            }
+            await options.onFailedAttempt(error);
+            if (!operation.retry(error)) {
+              throw operation.mainError();
+            }
+          } catch (finalError) {
+            decorateErrorWithCounts(finalError, attemptNumber, options);
+            cleanUp();
+            reject(finalError);
+          }
+        }
+      });
+    });
+  }
+
+  // node_modules/.pnpm/p-map@7.0.2/node_modules/p-map/index.js
+  async function pMap(iterable, mapper, {
+    concurrency = Number.POSITIVE_INFINITY,
+    stopOnError = true,
+    signal
+  } = {}) {
+    return new Promise((resolve7, reject_) => {
+      if (iterable[Symbol.iterator] === void 0 && iterable[Symbol.asyncIterator] === void 0) {
+        throw new TypeError(`Expected \`input\` to be either an \`Iterable\` or \`AsyncIterable\`, got (${typeof iterable})`);
+      }
+      if (typeof mapper !== "function") {
+        throw new TypeError("Mapper function is required");
+      }
+      if (!(Number.isSafeInteger(concurrency) && concurrency >= 1 || concurrency === Number.POSITIVE_INFINITY)) {
+        throw new TypeError(`Expected \`concurrency\` to be an integer from 1 and up or \`Infinity\`, got \`${concurrency}\` (${typeof concurrency})`);
+      }
+      const result = [];
+      const errors = [];
+      const skippedIndexesMap = /* @__PURE__ */ new Map();
+      let isRejected = false;
+      let isResolved = false;
+      let isIterableDone = false;
+      let resolvingCount = 0;
+      let currentIndex = 0;
+      const iterator = iterable[Symbol.iterator] === void 0 ? iterable[Symbol.asyncIterator]() : iterable[Symbol.iterator]();
+      const reject = (reason) => {
+        isRejected = true;
+        isResolved = true;
+        reject_(reason);
+      };
+      if (signal) {
+        if (signal.aborted) {
+          reject(signal.reason);
+        }
+        signal.addEventListener("abort", () => {
+          reject(signal.reason);
+        });
+      }
+      const next = async () => {
+        if (isResolved) {
+          return;
+        }
+        const nextItem = await iterator.next();
+        const index2 = currentIndex;
+        currentIndex++;
+        if (nextItem.done) {
+          isIterableDone = true;
+          if (resolvingCount === 0 && !isResolved) {
+            if (!stopOnError && errors.length > 0) {
+              reject(new AggregateError(errors));
+              return;
+            }
+            isResolved = true;
+            if (skippedIndexesMap.size === 0) {
+              resolve7(result);
+              return;
+            }
+            const pureResult = [];
+            for (const [index3, value] of result.entries()) {
+              if (skippedIndexesMap.get(index3) === pMapSkip) {
+                continue;
+              }
+              pureResult.push(value);
+            }
+            resolve7(pureResult);
+          }
+          return;
+        }
+        resolvingCount++;
+        (async () => {
+          try {
+            const element = await nextItem.value;
+            if (isResolved) {
+              return;
+            }
+            const value = await mapper(element, index2);
+            if (value === pMapSkip) {
+              skippedIndexesMap.set(index2, value);
+            }
+            result[index2] = value;
+            resolvingCount--;
+            await next();
+          } catch (error) {
+            if (stopOnError) {
+              reject(error);
+            } else {
+              errors.push(error);
+              resolvingCount--;
+              try {
+                await next();
+              } catch (error2) {
+                reject(error2);
+              }
+            }
+          }
+        })();
+      };
+      (async () => {
+        for (let index2 = 0; index2 < concurrency; index2++) {
+          try {
+            await next();
+          } catch (error) {
+            reject(error);
+            break;
+          }
+          if (isIterableDone || isRejected) {
+            break;
+          }
+        }
+      })();
+    });
+  }
+  var pMapSkip = Symbol("skip");
+
   // src/blockstore/store.ts
   function guardVersion(url) {
     if (!url.hasParam("version")) {
@@ -27371,72 +27828,82 @@ You can use close({ resize: true }) to resize header`);
     }
     async _doProcess() {
       if (!this.loader.remoteCarStore) return;
-      const rmlp = (async () => {
-        const operations = [...this.walState.operations];
-        const fileOperations = [...this.walState.fileOperations];
-        const uploads = [];
-        const noLoaderOps = [...this.walState.noLoaderOps];
-        const limit = pLimit(3);
-        if (operations.length + fileOperations.length + noLoaderOps.length === 0) return;
-        for (const dbMeta of noLoaderOps) {
-          const uploadP = limit(async () => {
-            for (const cid of dbMeta.cars) {
-              const car = await (await this.loader.carStore()).load(cid);
-              if (!car) {
-                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
-                  throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
-              } else {
-                await throwFalsy(this.loader.remoteCarStore).save(car);
+      const operations = [...this.walState.operations];
+      const noLoaderOps = [...this.walState.noLoaderOps];
+      const fileOperations = [...this.walState.fileOperations];
+      if (operations.length + noLoaderOps.length + fileOperations.length === 0) return;
+      const concurrencyLimit = 3;
+      const retryableUpload = (fn, description) => pRetry(fn, {
+        retries: 5,
+        onFailedAttempt: (error) => {
+          this.logger.Warn().Msg(`Attempt ${error.attemptNumber} failed for ${description}. There are ${error.retriesLeft} retries left.`);
+        }
+      });
+      try {
+        await pMap(
+          noLoaderOps,
+          async (dbMeta) => {
+            await retryableUpload(async () => {
+              for (const cid of dbMeta.cars) {
+                const car = await (await this.loader.carStore()).load(cid);
+                if (!car) {
+                  if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                    throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
+                  }
+                } else {
+                  await throwFalsy(this.loader.remoteCarStore).save(car);
+                }
               }
               this.walState.noLoaderOps = this.walState.noLoaderOps.filter((op) => op !== dbMeta);
-            }
-          });
-          uploads.push(uploadP);
-        }
-        for (const dbMeta of operations) {
-          const uploadP = limit(async () => {
-            for (const cid of dbMeta.cars) {
-              const car = await (await this.loader.carStore()).load(cid).catch(() => null);
-              if (!car) {
-                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
-                  throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
-              } else {
-                await throwFalsy(this.loader.remoteCarStore).save(car);
+            }, `noLoaderOp with dbMeta.cars=${dbMeta.cars.toString()}`);
+          },
+          { concurrency: concurrencyLimit }
+        );
+        await pMap(
+          operations,
+          async (dbMeta) => {
+            await retryableUpload(async () => {
+              for (const cid of dbMeta.cars) {
+                const car = await (await this.loader.carStore()).load(cid);
+                if (!car) {
+                  if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                    throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
+                  }
+                } else {
+                  await throwFalsy(this.loader.remoteCarStore).save(car);
+                }
               }
-            }
-            this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
-          });
-          uploads.push(uploadP);
-        }
-        if (fileOperations.length) {
-          const dbLoader = this.loader;
-          for (const { cid: fileCid, public: publicFile } of fileOperations) {
-            const uploadP = limit(async () => {
-              const fileBlock = await (await dbLoader.fileStore()).load(fileCid);
-              await dbLoader.remoteFileStore?.save(fileBlock, { public: publicFile });
+              this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
+            }, `operation with dbMeta.cars=${dbMeta.cars.toString()}`);
+          },
+          { concurrency: concurrencyLimit }
+        );
+        await pMap(
+          fileOperations,
+          async ({ cid: fileCid, public: publicFile }) => {
+            await retryableUpload(async () => {
+              const fileBlock = await (await this.loader.fileStore()).load(fileCid);
+              if (!fileBlock) {
+                throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
+              }
+              await this.loader.remoteFileStore?.save(fileBlock, { public: publicFile });
               this.walState.fileOperations = this.walState.fileOperations.filter((op) => op.cid !== fileCid);
-            });
-            uploads.push(uploadP);
-          }
-        }
-        try {
-          const res = await Promise.allSettled(uploads);
-          const errors = res.filter((r) => r.status === "rejected");
-          if (errors.length) {
-            throw this.logger.Error().Any("errors", errors).Msg("error uploading").AsError();
-          }
-          if (operations.length) {
-            const lastOp = operations[operations.length - 1];
-            await this.loader.remoteMetaStore?.save(lastOp).catch((e) => {
-              this.walState.operations.push(lastOp);
-              throw this.logger.Error().Any("error", e).Msg("error saving remote meta").AsError();
-            });
-          }
-        } finally {
-          await this.save(this.walState);
+            }, `fileOperation with cid=${fileCid.toString()}`);
+          },
+          { concurrency: concurrencyLimit }
+        );
+        if (operations.length) {
+          const lastOp = operations[operations.length - 1];
+          await retryableUpload(async () => {
+            await this.loader.remoteMetaStore?.save(lastOp);
+          }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
         }
-      })();
-      await rmlp;
+      } catch (error) {
+        this.logger.Error().Any("error", error).Msg("Processing failed");
+        return;
+      } finally {
+        await this.save(this.walState);
+      }
     }
     async load() {
       this.logger.Debug().Msg("loading");
@@ -30062,7 +30529,7 @@ You can use close({ resize: true }) to resize header`);
 
   // src/version.ts
   var PACKAGE_VERSION = Object.keys({
-    "0.19.11-dev-dryrun6": "xxxx"
+    "0.19.99-dev-100": "xxxx"
   })[0];
   return __toCommonJS(src_exports6);
 })();