@lodestar/state-transition 1.41.0-dev.f2caa915ab → 1.41.0-dev.f7a5f4ddda

package/src/block/processExecutionPayloadEnvelope.ts

@@ -7,18 +7,25 @@ import {
 } from "@lodestar/params";
 import {gloas, ssz} from "@lodestar/types";
 import {byteArrayEquals, toHex, toRootHex} from "@lodestar/utils";
-import {CachedBeaconStateGloas} from "../types.ts";
-import {computeSigningRoot, computeTimeAtSlot} from "../util/index.ts";
-import {processConsolidationRequest} from "./processConsolidationRequest.ts";
-import {processDepositRequest} from "./processDepositRequest.ts";
-import {processWithdrawalRequest} from "./processWithdrawalRequest.ts";
-
-// This function does not call execution engine to verify payload. Need to call it from other place
+import {CachedBeaconStateGloas} from "../types.js";
+import {computeSigningRoot, computeTimeAtSlot} from "../util/index.js";
+import {processConsolidationRequest} from "./processConsolidationRequest.js";
+import {processDepositRequest} from "./processDepositRequest.js";
+import {processWithdrawalRequest} from "./processWithdrawalRequest.js";
+
+export type ProcessExecutionPayloadEnvelopeOpts = {
+  dontTransferCache?: boolean;
+};
+
+// Unlike other block processing functions which mutate state in-place, this function
+// clones the state and returns the post-state, similar to stateTransition().
+// This function does not call execution engine to verify payload. Need to call it from other place.
 export function processExecutionPayloadEnvelope(
   state: CachedBeaconStateGloas,
   signedEnvelope: gloas.SignedExecutionPayloadEnvelope,
-  verify: boolean
-): void {
+  verify: boolean,
+  opts?: ProcessExecutionPayloadEnvelopeOpts
+): CachedBeaconStateGloas {
   const envelope = signedEnvelope.message;
   const payload = envelope.payload;
   const fork = state.config.getForkSeq(envelope.slot);
@@ -27,42 +34,49 @@ export function processExecutionPayloadEnvelope(
     throw Error(`Execution payload envelope has invalid signature builderIndex=${envelope.builderIndex}`);
   }
 
-  validateExecutionPayloadEnvelope(state, envelope);
+  // .clone() before mutating state, similar to stateTransition()
+  const postState = state.clone(opts?.dontTransferCache) as CachedBeaconStateGloas;
+
+  validateExecutionPayloadEnvelope(postState, envelope);
 
   const requests = envelope.executionRequests;
 
   for (const deposit of requests.deposits) {
-    processDepositRequest(fork, state, deposit);
+    processDepositRequest(fork, postState, deposit);
   }
 
   for (const withdrawal of requests.withdrawals) {
-    processWithdrawalRequest(fork, state, withdrawal);
+    processWithdrawalRequest(fork, postState, withdrawal);
   }
 
   for (const consolidation of requests.consolidations) {
-    processConsolidationRequest(state, consolidation);
+    processConsolidationRequest(postState, consolidation);
   }
 
   // Queue the builder payment
-  const paymentIndex = SLOTS_PER_EPOCH + (state.slot % SLOTS_PER_EPOCH);
-  const payment = state.builderPendingPayments.get(paymentIndex).clone();
+  const paymentIndex = SLOTS_PER_EPOCH + (postState.slot % SLOTS_PER_EPOCH);
+  const payment = postState.builderPendingPayments.get(paymentIndex).clone();
   const amount = payment.withdrawal.amount;
 
   if (amount > 0) {
-    state.builderPendingWithdrawals.push(payment.withdrawal);
+    postState.builderPendingWithdrawals.push(payment.withdrawal);
   }
 
-  state.builderPendingPayments.set(paymentIndex, ssz.gloas.BuilderPendingPayment.defaultViewDU());
+  postState.builderPendingPayments.set(paymentIndex, ssz.gloas.BuilderPendingPayment.defaultViewDU());
 
   // Cache the execution payload hash
-  state.executionPayloadAvailability.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, true);
-  state.latestBlockHash = payload.blockHash;
+  postState.executionPayloadAvailability.set(postState.slot % SLOTS_PER_HISTORICAL_ROOT, true);
+  postState.latestBlockHash = payload.blockHash;
 
-  if (verify && !byteArrayEquals(envelope.stateRoot, state.hashTreeRoot())) {
+  postState.commit();
+
+  if (verify && !byteArrayEquals(envelope.stateRoot, postState.hashTreeRoot())) {
     throw new Error(
-      `Envelope's state root does not match state envelope=${toRootHex(envelope.stateRoot)} state=${toRootHex(state.hashTreeRoot())}`
+      `Envelope's state root does not match state envelope=${toRootHex(envelope.stateRoot)} state=${toRootHex(postState.hashTreeRoot())}`
     );
   }
+
+  return postState;
 }
 
 function validateExecutionPayloadEnvelope(

package/src/block/processOperations.ts

@@ -14,7 +14,7 @@ import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js";
 import {processConsolidationRequest} from "./processConsolidationRequest.js";
 import {processDeposit} from "./processDeposit.js";
 import {processDepositRequest} from "./processDepositRequest.js";
-import {processPayloadAttestation} from "./processPayloadAttestation.ts";
+import {processPayloadAttestation} from "./processPayloadAttestation.js";
 import {processProposerSlashing} from "./processProposerSlashing.js";
 import {processVoluntaryExit} from "./processVoluntaryExit.js";
 import {processWithdrawalRequest} from "./processWithdrawalRequest.js";

package/src/block/processPayloadAttestation.ts

@@ -1,7 +1,7 @@
 import {gloas} from "@lodestar/types";
 import {byteArrayEquals} from "@lodestar/utils";
-import {CachedBeaconStateGloas} from "../types.ts";
-import {isValidIndexedPayloadAttestation} from "./isValidIndexedPayloadAttestation.ts";
+import {CachedBeaconStateGloas} from "../types.js";
+import {isValidIndexedPayloadAttestation} from "./isValidIndexedPayloadAttestation.js";
 
 export function processPayloadAttestation(
   state: CachedBeaconStateGloas,

package/src/block/processWithdrawals.ts

@@ -16,7 +16,7 @@ import {
   convertValidatorIndexToBuilderIndex,
   isBuilderIndex,
   isParentBlockFull,
-} from "../util/gloas.ts";
+} from "../util/gloas.js";
 import {
   decreaseBalance,
   getMaxEffectiveBalance,

package/src/cache/epochCache.ts

@@ -226,7 +226,9 @@ export class EpochCache {
   /** TODO: Indexed SyncCommitteeCache */
   nextSyncCommitteeIndexed: SyncCommitteeCache;
 
-  // TODO GLOAS: See if we need to cached PTC for prev/next epoch
+  // TODO GLOAS: See if we need to cache PTC for next epoch
+  // PTC for previous epoch, required for slot N block validating slot N-1 attestations
+  previousPayloadTimelinessCommittees: Uint32Array[];
   // PTC for current epoch, computed eagerly at epoch transition
   payloadTimelinessCommittees: Uint32Array[];
 
@@ -266,6 +268,7 @@ export class EpochCache {
     previousTargetUnslashedBalanceIncrements: number;
     currentSyncCommitteeIndexed: SyncCommitteeCache;
     nextSyncCommitteeIndexed: SyncCommitteeCache;
+    previousPayloadTimelinessCommittees: Uint32Array[];
     payloadTimelinessCommittees: Uint32Array[];
     epoch: Epoch;
     syncPeriod: SyncPeriod;
@@ -296,6 +299,7 @@ export class EpochCache {
     this.previousTargetUnslashedBalanceIncrements = data.previousTargetUnslashedBalanceIncrements;
     this.currentSyncCommitteeIndexed = data.currentSyncCommitteeIndexed;
     this.nextSyncCommitteeIndexed = data.nextSyncCommitteeIndexed;
+    this.previousPayloadTimelinessCommittees = data.previousPayloadTimelinessCommittees;
     this.payloadTimelinessCommittees = data.payloadTimelinessCommittees;
     this.epoch = data.epoch;
     this.syncPeriod = data.syncPeriod;
@@ -447,7 +451,8 @@ export class EpochCache {
       nextSyncCommitteeIndexed = new SyncCommitteeCacheEmpty();
     }
 
-    // Compute PTC eagerly for all slots in the epoch
+    // Compute PTC for all slots in the prev/current epoch
+    let previousPayloadTimelinessCommittees: Uint32Array[] = [];
     let payloadTimelinessCommittees: Uint32Array[] = [];
     if (currentEpoch >= config.GLOAS_FORK_EPOCH) {
       payloadTimelinessCommittees = computePayloadTimelinessCommitteesForEpoch(
@@ -456,6 +461,15 @@ export class EpochCache {
         currentShuffling.committees,
         effectiveBalanceIncrements
       );
+
+      if (!isGenesis && previousEpoch >= config.GLOAS_FORK_EPOCH) {
+        previousPayloadTimelinessCommittees = computePayloadTimelinessCommitteesForEpoch(
+          state,
+          previousEpoch,
+          previousShuffling.committees,
+          effectiveBalanceIncrements
+        );
+      }
     }
 
     // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute everytime the
@@ -530,6 +544,7 @@ export class EpochCache {
       currentTargetUnslashedBalanceIncrements,
       currentSyncCommitteeIndexed,
       nextSyncCommitteeIndexed,
+      previousPayloadTimelinessCommittees,
       payloadTimelinessCommittees,
       epoch: currentEpoch,
       syncPeriod: computeSyncPeriodAtEpoch(currentEpoch),
@@ -575,6 +590,7 @@ export class EpochCache {
       currentTargetUnslashedBalanceIncrements: this.currentTargetUnslashedBalanceIncrements,
       currentSyncCommitteeIndexed: this.currentSyncCommitteeIndexed,
       nextSyncCommitteeIndexed: this.nextSyncCommitteeIndexed,
+      previousPayloadTimelinessCommittees: this.previousPayloadTimelinessCommittees,
       payloadTimelinessCommittees: this.payloadTimelinessCommittees,
       epoch: this.epoch,
       syncPeriod: this.syncPeriod,
@@ -686,6 +702,8 @@ export class EpochCache {
 
     this.proposersPrevEpoch = this.proposers;
     if (upcomingEpoch >= this.config.GLOAS_FORK_EPOCH) {
+      // Shift and compute current epoch PTC eagerly for all slots
+      this.previousPayloadTimelinessCommittees = this.payloadTimelinessCommittees;
       this.payloadTimelinessCommittees = computePayloadTimelinessCommitteesForEpoch(
         state,
         upcomingEpoch,
@@ -1016,11 +1034,15 @@ export class EpochCache {
       throw new Error("Payload Timeliness Committee is not available before gloas fork");
     }
 
-    if (epoch !== this.epoch) {
-      throw new Error(`Payload Timeliness Committee is not available for slot=${slot}`);
+    if (epoch === this.epoch) {
+      return this.payloadTimelinessCommittees[slot % SLOTS_PER_EPOCH];
+    }
+
+    if (epoch === this.epoch - 1 && this.previousPayloadTimelinessCommittees.length > 0) {
+      return this.previousPayloadTimelinessCommittees[slot % SLOTS_PER_EPOCH];
     }
 
-    return this.payloadTimelinessCommittees[slot % SLOTS_PER_EPOCH];
+    throw new Error(`Payload Timeliness Committee is not available for slot=${slot}`);
   }
 
   getIndexedPayloadAttestation(

package/src/epoch/index.ts

@@ -16,7 +16,7 @@ import {
   CachedBeaconStatePhase0,
   EpochTransitionCache,
 } from "../types.js";
-import {processBuilderPendingPayments} from "./processBuilderPendingPayments.ts";
+import {processBuilderPendingPayments} from "./processBuilderPendingPayments.js";
 import {processEffectiveBalanceUpdates} from "./processEffectiveBalanceUpdates.js";
 import {processEth1DataReset} from "./processEth1DataReset.js";
 import {processHistoricalRootsUpdate} from "./processHistoricalRootsUpdate.js";

package/src/epoch/processBuilderPendingPayments.ts

@@ -1,7 +1,7 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {ssz} from "@lodestar/types";
-import {CachedBeaconStateGloas} from "../types.ts";
-import {getBuilderPaymentQuorumThreshold} from "../util/gloas.ts";
+import {CachedBeaconStateGloas} from "../types.js";
+import {getBuilderPaymentQuorumThreshold} from "../util/gloas.js";
 
 /**
  * Processes the builder pending payments from the previous epoch.

package/src/testUtils/cache.ts

@@ -0,0 +1,8 @@
+import path from "node:path";
+import {fileURLToPath} from "node:url";
+
+// Global variable __dirname no longer available in ES6 modules.
+// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+export const testCachePath = path.join(__dirname, "../../test/test-cache");

package/src/testUtils/index.ts

@@ -0,0 +1,5 @@
+export * from "./cache.js";
+export * from "./interop.js";
+export * from "./params.js";
+export * from "./testFileCache.js";
+export * from "./util.js";

package/src/testUtils/infura.ts

@@ -0,0 +1,10 @@
+import {NetworkName} from "@lodestar/config/networks";
+
+export function getInfuraBeaconUrl(network: NetworkName): string {
+  const INFURA_ETH2_CREDENTIALS = process.env.INFURA_ETH2_CREDENTIALS;
+  if (!INFURA_ETH2_CREDENTIALS) {
+    throw Error("Must set ENV INFURA_ETH2_CREDENTIALS");
+  }
+
+  return `https://${INFURA_ETH2_CREDENTIALS}@eth2-beacon-${network}.infura.io`;
+}

package/src/testUtils/interop.ts

@@ -0,0 +1,29 @@
+import fs from "node:fs";
+import path from "node:path";
+import {fromHexString, toHexString} from "@chainsafe/ssz";
+import {interopSecretKey} from "../index.js";
+import {testCachePath} from "./cache.js";
+
+const interopPubkeysCachedPath = path.join(testCachePath, "interop-pubkeys.json");
+
+export function interopPubkeysCached(validatorCount: number): Uint8Array[] {
+  fs.mkdirSync(path.dirname(interopPubkeysCachedPath), {recursive: true});
+
+  const cachedKeysHex = fs.existsSync(interopPubkeysCachedPath)
+    ? (JSON.parse(fs.readFileSync(interopPubkeysCachedPath, "utf8")) as string[])
+    : ([] as string[]);
+
+  const keys = cachedKeysHex.slice(0, validatorCount).map((hex) => fromHexString(hex));
+
+  if (cachedKeysHex.length < validatorCount) {
+    for (let i = cachedKeysHex.length; i < validatorCount; i++) {
+      const sk = interopSecretKey(i);
+      const pk = sk.toPublicKey().toBytes();
+      keys.push(pk);
+    }
+    const keysHex = keys.map((pk) => toHexString(pk));
+    fs.writeFileSync(interopPubkeysCachedPath, JSON.stringify(keysHex, null, 2));
+  }
+
+  return keys;
+}

package/src/testUtils/params.ts

@@ -0,0 +1,23 @@
+// DON'T MODIFY OR FORMAT THIS FILE
+// USE FOR GA CACHE
+
+export const phase0State = {
+  network: "mainnet" as const,
+  epoch: 58758, // Pre-altair fork
+};
+
+export const altairState = {
+  network: "mainnet" as const,
+  epoch: 81889, // Post altair fork
+};
+
+export const capellaState = {
+  network: "mainnet" as const,
+  epoch: 217614, // Post capella fork
+};
+
+export const rangeSyncTest = {
+  network: "mainnet" as const,
+  startSlot: 3766816, // Post altair, first slot in epoch 117713
+  endSlot: 3766847, // 3766816 + 31, all blocks in same epoch
+};

package/src/testUtils/state.ts

@@ -0,0 +1,110 @@
+import {ChainForkConfig, createBeaconConfig} from "@lodestar/config";
+import {config, config as minimalConfig} from "@lodestar/config/default";
+import {
+  EPOCHS_PER_HISTORICAL_VECTOR,
+  EPOCHS_PER_SLASHINGS_VECTOR,
+  GENESIS_EPOCH,
+  GENESIS_SLOT,
+  SLOTS_PER_HISTORICAL_ROOT,
+} from "@lodestar/params";
+import {phase0, ssz} from "@lodestar/types";
+import {EpochCacheOpts} from "../cache/epochCache.js";
+import {BeaconStateCache} from "../cache/stateCache.js";
+import {ZERO_HASH} from "../constants/index.js";
+import {
+  BeaconStateAllForks,
+  BeaconStatePhase0,
+  CachedBeaconStateAllForks,
+  createCachedBeaconState,
+  createPubkeyCache,
+} from "../index.js";
+import {newZeroedArray} from "../util/index.js";
+
+/**
+ * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed.
+ */
+type TestBeaconState = Partial<phase0.BeaconState>;
+
+/**
+ * Generate beaconState, by default it will use the initial state defined when the `ChainStart` log is emitted.
+ * NOTE: All fields can be overridden through `opts`.
+ * @param {TestBeaconState} opts
+ * @returns {BeaconState}
+ */
+export function generateState(opts?: TestBeaconState): BeaconStatePhase0 {
+  return ssz.phase0.BeaconState.toViewDU({
+    genesisTime: Math.floor(Date.now() / 1000),
+    genesisValidatorsRoot: ZERO_HASH,
+    slot: GENESIS_SLOT,
+    fork: {
+      previousVersion: config.GENESIS_FORK_VERSION,
+      currentVersion: config.GENESIS_FORK_VERSION,
+      epoch: GENESIS_EPOCH,
+    },
+    latestBlockHeader: {
+      slot: 0,
+      proposerIndex: 0,
+      parentRoot: Buffer.alloc(32),
+      stateRoot: Buffer.alloc(32),
+      bodyRoot: ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()),
+    },
+    blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH),
+    stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH),
+    historicalRoots: [],
+    eth1Data: {
+      depositRoot: Buffer.alloc(32),
+      blockHash: Buffer.alloc(32),
+      depositCount: 0,
+    },
+    eth1DataVotes: [],
+    eth1DepositIndex: 0,
+    validators: [],
+    balances: [],
+    randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH),
+    slashings: newZeroedArray(EPOCHS_PER_SLASHINGS_VECTOR),
+    previousEpochAttestations: [],
+    currentEpochAttestations: [],
+    justificationBits: ssz.phase0.JustificationBits.defaultValue(),
+    previousJustifiedCheckpoint: {
+      epoch: GENESIS_EPOCH,
+      root: ZERO_HASH,
+    },
+    currentJustifiedCheckpoint: {
+      epoch: GENESIS_EPOCH,
+      root: ZERO_HASH,
+    },
+    finalizedCheckpoint: {
+      epoch: GENESIS_EPOCH,
+      root: ZERO_HASH,
+    },
+    ...opts,
+  });
+}
+
+export function generateCachedState(
+  config: ChainForkConfig = minimalConfig,
+  opts: TestBeaconState = {}
+): CachedBeaconStateAllForks {
+  const state = generateState(opts);
+  return createCachedBeaconState(state, {
+    config: createBeaconConfig(config, state.genesisValidatorsRoot),
+    // This is a test state, there's no need to have a global shared cache of keys
+    pubkeyCache: createPubkeyCache(),
+  });
+}
+
+export function createCachedBeaconStateTest<T extends BeaconStateAllForks>(
+  state: T,
+  configCustom: ChainForkConfig = config,
+  opts?: EpochCacheOpts
+): T & BeaconStateCache {
+  return createCachedBeaconState<T>(
+    state,
+    {
+      config: createBeaconConfig(configCustom, state.genesisValidatorsRoot),
+      // This is a test state, there's no need to have a global shared cache of keys
+      pubkeyCache: createPubkeyCache(),
+    },
+    opts
+  );
+}

package/src/testUtils/testFileCache.ts

@@ -0,0 +1,127 @@
+import fs from "node:fs";
+import path from "node:path";
+import {getClient} from "@lodestar/api";
+import {ChainForkConfig, createChainForkConfig} from "@lodestar/config";
+import {NetworkName, networksChainConfig} from "@lodestar/config/networks";
+import {SignedBeaconBlock} from "@lodestar/types";
+import {fetch} from "@lodestar/utils";
+import {CachedBeaconStateAllForks} from "../index.js";
+import {testCachePath} from "./cache.js";
+import {getInfuraBeaconUrl} from "./infura.js";
+import {createCachedBeaconStateTest} from "./state.js";
+
+/**
+ * Full link example:
+ * ```
+ * https://github.com/dapplion/ethereum-consensus-test-data/releases/download/v0.1.0/block_mainnet_3766821.ssz
+ * ``` */
+const TEST_FILES_BASE_URL = "https://github.com/dapplion/ethereum-consensus-test-data/releases/download/v0.1.0";
+
+/**
+ * Create a network config from known network params
+ */
+export function getNetworkConfig(network: NetworkName): ChainForkConfig {
+  const configNetwork = networksChainConfig[network];
+  return createChainForkConfig(configNetwork);
+}
+
+/**
+ * Download a state from Infura. Caches states in local fs by network and slot to only download once.
+ */
+export async function getNetworkCachedState(
+  network: NetworkName,
+  slot: number,
+  timeout?: number
+): Promise<CachedBeaconStateAllForks> {
+  const config = getNetworkConfig(network);
+  const fileId = `state_${network}_${slot}.ssz`;
+
+  const filepath = path.join(testCachePath, fileId);
+
+  if (fs.existsSync(filepath)) {
+    const stateSsz = fs.readFileSync(filepath);
+    return createCachedBeaconStateTest(config.getForkTypes(slot).BeaconState.deserializeToViewDU(stateSsz), config);
+  }
+
+  const stateSsz = await tryEach([
+    () => downloadTestFile(fileId),
+    () => {
+      const client = getClient(
+        {baseUrl: getInfuraBeaconUrl(network), globalInit: {timeoutMs: timeout ?? 300_000}},
+        {config}
+      );
+      return client.debug.getStateV2({stateId: slot}).then((r) => {
+        return r.ssz();
+      });
+    },
+  ]);
+
+  fs.writeFileSync(filepath, stateSsz);
+  return createCachedBeaconStateTest(config.getForkTypes(slot).BeaconState.deserializeToViewDU(stateSsz), config);
+}
+
+/**
+ * Download a state from Infura. Caches states in local fs by network and slot to only download once.
+ */
+export async function getNetworkCachedBlock(
+  network: NetworkName,
+  slot: number,
+  timeout?: number
+): Promise<SignedBeaconBlock> {
+  const config = getNetworkConfig(network);
+  const fileId = `block_${network}_${slot}.ssz`;
+
+  const filepath = path.join(testCachePath, fileId);
+
+  if (fs.existsSync(filepath)) {
+    const blockSsz = fs.readFileSync(filepath);
+    return config.getForkTypes(slot).SignedBeaconBlock.deserialize(blockSsz);
+  }
+
+  const blockSsz = await tryEach([
+    () => downloadTestFile(fileId),
+    async () => {
+      const client = getClient(
+        {baseUrl: getInfuraBeaconUrl(network), globalInit: {timeoutMs: timeout ?? 300_000}},
+        {config}
+      );
+
+      return (await client.beacon.getBlockV2({blockId: slot})).ssz();
+    },
+  ]);
+
+  fs.writeFileSync(filepath, blockSsz);
+  return config.getForkTypes(slot).SignedBeaconBlock.deserialize(blockSsz);
+}
+
+async function downloadTestFile(fileId: string): Promise<Uint8Array> {
+  const fileUrl = `${TEST_FILES_BASE_URL}/${fileId}`;
+  // biome-ignore lint/suspicious/noConsole: We explicity need to log to console
+  console.log(`Downloading file ${fileUrl}`);
+
+  try {
+    const res = await fetch(fileUrl);
+    if (!res.ok) {
+      throw new Error(`Error downloading ${fileUrl}: ${res.status} ${res.statusText}`);
+    }
+    return new Uint8Array(await res.arrayBuffer());
+  } catch (e) {
+    const error = e as Error;
+    error.message = `Error downloading ${fileUrl}: ${error.message}`;
+    throw error;
+  }
+}
+
+async function tryEach<T>(promises: (() => Promise<T>)[]): Promise<T> {
+  const errors: Error[] = [];
+
+  for (let i = 0; i < promises.length; i++) {
+    try {
+      return await promises[i]();
+    } catch (e) {
+      errors.push(e as Error);
+    }
+  }
+
+  throw Error(errors.map((e, i) => `Error[${i}] ${e.message}`).join("\n"));
+}