@fireproof/core 0.19.8-dev-series-1 → 0.19.8-dev-series-2

package/index.js

@@ -252,7 +252,6 @@ var UnixFSFileBuilder = class {
 };
 
 // src/blockstore/store.ts
-import pLimit2 from "p-limit";
 import { format as format2, parse as parse2 } from "@ipld/dag-json";
 import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result5 } from "@adviser/cement";
 
@@ -1667,6 +1666,8 @@ async function encodeEventsWithParents(sthis, events, parents) {
 }
 
 // src/blockstore/store.ts
+import pRetry from "p-retry";
+import pMap from "p-map";
 function guardVersion(url) {
   if (!url.hasParam("version")) {
     return Result5.Err(`missing version: ${url.toString()}`);
@@ -1926,72 +1927,82 @@ var WALStoreImpl = class extends BaseStoreImpl {
   }
   async _doProcess() {
     if (!this.loader.remoteCarStore) return;
-    const rmlp = (async () => {
-      const operations = [...this.walState.operations];
-      const fileOperations = [...this.walState.fileOperations];
-      const uploads = [];
-      const noLoaderOps = [...this.walState.noLoaderOps];
-      const limit = pLimit2(3);
-      if (operations.length + fileOperations.length + noLoaderOps.length === 0) return;
-      for (const dbMeta of noLoaderOps) {
-        const uploadP = limit(async () => {
-          for (const cid of dbMeta.cars) {
-            const car = await (await this.loader.carStore()).load(cid);
-            if (!car) {
-              if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
-                throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
-            } else {
-              await throwFalsy(this.loader.remoteCarStore).save(car);
+    const operations = [...this.walState.operations];
+    const noLoaderOps = [...this.walState.noLoaderOps];
+    const fileOperations = [...this.walState.fileOperations];
+    if (operations.length + noLoaderOps.length + fileOperations.length === 0) return;
+    const concurrencyLimit = 3;
+    const retryableUpload = (fn, description) => pRetry(fn, {
+      retries: 5,
+      onFailedAttempt: (error) => {
+        this.logger.Warn().Msg(`Attempt ${error.attemptNumber} failed for ${description}. There are ${error.retriesLeft} retries left.`);
+      }
+    });
+    try {
+      await pMap(
+        noLoaderOps,
+        async (dbMeta) => {
+          await retryableUpload(async () => {
+            for (const cid of dbMeta.cars) {
+              const car = await (await this.loader.carStore()).load(cid);
+              if (!car) {
+                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                  throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
+                }
+              } else {
+                await throwFalsy(this.loader.remoteCarStore).save(car);
+              }
             }
             this.walState.noLoaderOps = this.walState.noLoaderOps.filter((op) => op !== dbMeta);
-          }
-        });
-        uploads.push(uploadP);
-      }
-      for (const dbMeta of operations) {
-        const uploadP = limit(async () => {
-          for (const cid of dbMeta.cars) {
-            const car = await (await this.loader.carStore()).load(cid).catch(() => null);
-            if (!car) {
-              if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
-                throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
-            } else {
-              await throwFalsy(this.loader.remoteCarStore).save(car);
+          }, `noLoaderOp with dbMeta.cars=${dbMeta.cars.toString()}`);
+        },
+        { concurrency: concurrencyLimit }
+      );
+      await pMap(
+        operations,
+        async (dbMeta) => {
+          await retryableUpload(async () => {
+            for (const cid of dbMeta.cars) {
+              const car = await (await this.loader.carStore()).load(cid);
+              if (!car) {
+                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                  throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
+                }
+              } else {
+                await throwFalsy(this.loader.remoteCarStore).save(car);
+              }
             }
-          }
-          this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
-        });
-        uploads.push(uploadP);
-      }
-      if (fileOperations.length) {
-        const dbLoader = this.loader;
-        for (const { cid: fileCid, public: publicFile } of fileOperations) {
-          const uploadP = limit(async () => {
-            const fileBlock = await (await dbLoader.fileStore()).load(fileCid);
-            await dbLoader.remoteFileStore?.save(fileBlock, { public: publicFile });
+            this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
+          }, `operation with dbMeta.cars=${dbMeta.cars.toString()}`);
+        },
+        { concurrency: concurrencyLimit }
+      );
+      await pMap(
+        fileOperations,
+        async ({ cid: fileCid, public: publicFile }) => {
+          await retryableUpload(async () => {
+            const fileBlock = await (await this.loader.fileStore()).load(fileCid);
+            if (!fileBlock) {
+              throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
+            }
+            await this.loader.remoteFileStore?.save(fileBlock, { public: publicFile });
             this.walState.fileOperations = this.walState.fileOperations.filter((op) => op.cid !== fileCid);
-          });
-          uploads.push(uploadP);
-        }
-      }
-      try {
-        const res = await Promise.allSettled(uploads);
-        const errors = res.filter((r) => r.status === "rejected");
-        if (errors.length) {
-          throw this.logger.Error().Any("errors", errors).Msg("error uploading").AsError();
-        }
-        if (operations.length) {
-          const lastOp = operations[operations.length - 1];
-          await this.loader.remoteMetaStore?.save(lastOp).catch((e) => {
-            this.walState.operations.push(lastOp);
-            throw this.logger.Error().Any("error", e).Msg("error saving remote meta").AsError();
-          });
-        }
-      } finally {
-        await this.save(this.walState);
-      }
-    })();
-    await rmlp;
+          }, `fileOperation with cid=${fileCid.toString()}`);
+        },
+        { concurrency: concurrencyLimit }
+      );
+      if (operations.length) {
+        const lastOp = operations[operations.length - 1];
+        await retryableUpload(async () => {
+          await this.loader.remoteMetaStore?.save(lastOp);
+        }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
+      }
+    } catch (error) {
+      this.logger.Error().Any("error", error).Msg("Processing failed");
+      return;
+    } finally {
+      await this.save(this.walState);
+    }
   }
   async load() {
     this.logger.Debug().Msg("loading");
@@ -3416,7 +3427,7 @@ import { runtimeFn as runtimeFn2 } from "@adviser/cement";
 
 // src/version.ts
 var PACKAGE_VERSION = Object.keys({
-  "0.19.8-dev-series-1": "xxxx"
+  "0.19.8-dev-series-2": "xxxx"
 })[0];
 export {
   CRDT,