@aztec/p2p 0.0.1-commit.c7c42ec → 0.0.1-commit.cf93bcc56

package/src/services/tx_collection/file_store_tx_collection.ts

@@ -0,0 +1,198 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider } from '@aztec/foundation/timer';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { FileStoreTxSource } from './file_store_tx_source.js';
+import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
+
+/** Configuration for a FileStoreTxCollection instance. */
+export type FileStoreCollectionConfig = {
+  workerCount: number;
+  backoffBaseMs: number;
+  backoffMaxMs: number;
+};
+
+type FileStoreTxEntry = {
+  txHash: string;
+  context: TxAddContext;
+  deadline: Date;
+  attempts: number;
+  lastAttemptTime: number;
+  nextSourceIndex: number;
+};
+
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
+ * retries with round-robin across sources, and applies exponential backoff between
+ * full cycles through all sources.
+ */
+export class FileStoreTxCollection {
+  /** Map from tx hash string to entry for all pending downloads. */
+  private entries = new Map<string, FileStoreTxEntry>();
+
+  /** Worker promises for the shared worker pool. */
+  private workers: Promise<void>[] = [];
+
+  /** Whether the worker pool is running. */
+  private running = false;
+
+  /** Signal used to wake sleeping workers when new entries arrive or stop is called. */
+  private wakeSignal: PromiseWithResolvers<void>;
+
+  constructor(
+    private readonly sources: FileStoreTxSource[],
+    private readonly txCollectionSink: TxCollectionSink,
+    private readonly config: FileStoreCollectionConfig,
+    private readonly dateProvider: DateProvider = new DateProvider(),
+    private readonly log: Logger = createLogger('p2p:file_store_tx_collection'),
+  ) {
+    this.wakeSignal = promiseWithResolvers<void>();
+  }
+
+  /** Starts the shared worker pool. */
+  public start(): void {
+    if (this.sources.length === 0) {
+      this.log.debug('No file store sources configured');
+      return;
+    }
+    this.running = true;
+    for (let i = 0; i < this.config.workerCount; i++) {
+      this.workers.push(this.workerLoop());
+    }
+  }
+
+  /** Stops all workers and clears state. */
+  public async stop(): Promise<void> {
+    this.running = false;
+    this.wake();
+    await Promise.all(this.workers);
+    this.workers = [];
+    this.entries.clear();
+  }
+
+  /** Adds entries to the shared map and wakes workers. */
+  public startCollecting(txHashes: TxHash[], context: TxAddContext, deadline: Date): void {
+    if (this.sources.length === 0 || txHashes.length === 0) {
+      return;
+    }
+    if (+deadline <= this.dateProvider.now()) {
+      return;
+    }
+
+    for (const txHash of txHashes) {
+      const hashStr = txHash.toString();
+      if (!this.entries.has(hashStr)) {
+        this.entries.set(hashStr, {
+          txHash: hashStr,
+          context,
+          deadline,
+          attempts: 0,
+          lastAttemptTime: 0,
+          nextSourceIndex: Math.floor(Math.random() * this.sources.length),
+        });
+      }
+    }
+    this.wake();
+  }
+
+  /** Removes entries for txs that have been found elsewhere. */
+  public foundTxs(txs: Tx[]): void {
+    for (const tx of txs) {
+      this.entries.delete(tx.getTxHash().toString());
+    }
+  }
+
+  /** Clears all pending entries. */
+  public clearPending(): void {
+    this.entries.clear();
+  }
+
+  private async workerLoop(): Promise<void> {
+    while (this.running) {
+      const action = this.getNextAction();
+      if (action.type === 'sleep') {
+        await action.promise;
+        continue;
+      }
+
+      const entry = action.entry;
+      const source = this.sources[entry.nextSourceIndex % this.sources.length];
+      entry.nextSourceIndex++;
+      entry.attempts++;
+      entry.lastAttemptTime = this.dateProvider.now();
+
+      try {
+        const result = await this.txCollectionSink.collect(
+          hashes => source.getTxsByHash(hashes),
+          [TxHash.fromString(entry.txHash)],
+          { description: `file-store ${source.getInfo()}`, method: 'file-store', fileStore: source.getInfo() },
+          entry.context,
+        );
+        if (result.txs.length > 0) {
+          this.entries.delete(entry.txHash);
+        }
+      } catch (err) {
+        this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, { err });
+      }
+    }
+  }
+
+  /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */
+  private getNextAction(): { type: 'process'; entry: FileStoreTxEntry } | { type: 'sleep'; promise: Promise<void> } {
+    const now = this.dateProvider.now();
+    let best: FileStoreTxEntry | undefined;
+    let earliestReadyAt = Infinity;
+
+    for (const [key, entry] of this.entries) {
+      if (+entry.deadline <= now) {
+        this.entries.delete(key);
+        continue;
+      }
+      const backoffMs = this.getBackoffMs(entry);
+      const readyAt = entry.lastAttemptTime + backoffMs;
+      if (readyAt > now) {
+        earliestReadyAt = Math.min(earliestReadyAt, readyAt);
+        continue;
+      }
+      if (!best || entry.attempts < best.attempts) {
+        best = entry;
+      }
+    }
+
+    if (best) {
+      return { type: 'process', entry: best };
+    }
+    if (earliestReadyAt < Infinity) {
+      return { type: 'sleep', promise: this.sleepOrWake(earliestReadyAt - now) };
+    }
+    return { type: 'sleep', promise: this.waitForWake() };
+  }
+
+  /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */
+  private getBackoffMs(entry: FileStoreTxEntry): number {
+    const fullCycles = Math.floor(entry.attempts / this.sources.length);
+    if (fullCycles === 0) {
+      return 0;
+    }
+    return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs);
+  }
+
+  /** Resolves the current wake signal and creates a new one. */
+  private wake(): void {
+    this.wakeSignal.resolve();
+    this.wakeSignal = promiseWithResolvers<void>();
+  }
+
+  /** Waits until the wake signal is resolved. */
+  private async waitForWake(): Promise<void> {
+    await this.wakeSignal.promise;
+  }
+
+  /** Sleeps for the given duration or until the wake signal is resolved. */
+  private async sleepOrWake(ms: number): Promise<void> {
+    await Promise.race([sleep(ms), this.wakeSignal.promise]);
+  }
+}

package/src/services/tx_collection/file_store_tx_source.ts

@@ -0,0 +1,104 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { Timer } from '@aztec/foundation/timer';
+import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store';
+import { Tx, type TxHash } from '@aztec/stdlib/tx';
+import {
+  type Histogram,
+  Metrics,
+  type TelemetryClient,
+  type UpDownCounter,
+  getTelemetryClient,
+} from '@aztec/telemetry-client';
+
+import type { TxSource } from './tx_source.js';
+
+/** TxSource implementation that downloads txs from a file store. */
+export class FileStoreTxSource implements TxSource {
+  private downloadsSuccess: UpDownCounter;
+  private downloadsFailed: UpDownCounter;
+  private downloadDuration: Histogram;
+  private downloadSize: Histogram;
+
+  private constructor(
+    private readonly fileStore: ReadOnlyFileStore,
+    private readonly baseUrl: string,
+    private readonly basePath: string,
+    private readonly log: Logger,
+    telemetry: TelemetryClient,
+  ) {
+    const meter = telemetry.getMeter('file-store-tx-source');
+    this.downloadsSuccess = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_SUCCESS);
+    this.downloadsFailed = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_FAILED);
+    this.downloadDuration = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_DURATION);
+    this.downloadSize = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_SIZE);
+  }
+
+  /**
+   * Creates a FileStoreTxSource from a URL.
+   * @param url - The file store URL (s3://, gs://, file://, http://, https://).
+   * @param basePath - Base path for tx files within the store.
+   * @param log - Optional logger.
+   * @param telemetry - Optional telemetry client.
+   * @returns The FileStoreTxSource instance, or undefined if creation fails.
+   */
+  public static async create(
+    url: string,
+    basePath: string,
+    log: Logger = createLogger('p2p:file_store_tx_source'),
+    telemetry: TelemetryClient = getTelemetryClient(),
+  ): Promise<FileStoreTxSource | undefined> {
+    try {
+      const fileStore = await createReadOnlyFileStore(url, log);
+      if (!fileStore) {
+        log.warn(`Failed to create file store for URL: ${url}`);
+        return undefined;
+      }
+      return new FileStoreTxSource(fileStore, url, basePath, log, telemetry);
+    } catch (err) {
+      log.warn(`Error creating file store for URL: ${url}`, { error: err });
+      return undefined;
+    }
+  }
+
+  public getInfo(): string {
+    return `file-store:${this.baseUrl}`;
+  }
+
+  public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return Promise.all(
+      txHashes.map(async txHash => {
+        const path = `${this.basePath}/txs/${txHash.toString()}.bin`;
+        const timer = new Timer();
+        try {
+          const buffer = await this.fileStore.read(path);
+          this.downloadsSuccess.add(1);
+          this.downloadDuration.record(Math.ceil(timer.ms()));
+          this.downloadSize.record(buffer.length);
+          return Tx.fromBuffer(buffer);
+        } catch {
+          this.downloadsFailed.add(1);
+          // Tx not found or error reading - return undefined
+          return undefined;
+        }
+      }),
+    );
+  }
+}
+
+/**
+ * Creates FileStoreTxSource instances from URLs.
+ * @param urls - Array of file store URLs.
+ * @param basePath - Base path for tx files within each store.
+ * @param log - Optional logger.
+ * @param telemetry - Optional telemetry client.
+ * @returns Array of successfully created FileStoreTxSource instances.
+ */
+export async function createFileStoreTxSources(
+  urls: string[],
+  basePath: string,
+  log: Logger = createLogger('p2p:file_store_tx_source'),
+  telemetry: TelemetryClient = getTelemetryClient(),
+): Promise<FileStoreTxSource[]> {
+  const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log, telemetry)));
+  return sources.filter((s): s is FileStoreTxSource => s !== undefined);
+}

package/src/services/tx_collection/index.ts

@@ -1,2 +1,8 @@
 export { TxCollection, type FastCollectionRequestInput } from './tx_collection.js';
 export { type TxSource, createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js';
+export {
+  type MissingTxsCollector,
+  BatchTxRequesterCollector,
+  SendBatchRequestCollector,
+} from './proposal_tx_collector.js';
+export { FileStoreTxSource, createFileStoreTxSources } from './file_store_tx_source.js';

package/src/services/tx_collection/instrumentation.ts

@@ -4,7 +4,7 @@ import {
   Metrics,
   type TelemetryClient,
   type UpDownCounter,
-  ValueType,
+  createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
 import type { CollectionMethod } from './tx_collection.js';
@@ -17,21 +17,19 @@ export class TxCollectionInstrumentation {
   constructor(client: TelemetryClient, name: string) {
     const meter = client.getMeter(name);
 
-    this.txsCollected = meter.createUpDownCounter(Metrics.TX_COLLECTOR_COUNT, {
-      description: 'The number of txs collected',
+    this.txsCollected = createUpDownCounterWithDefault(meter, Metrics.TX_COLLECTOR_COUNT, {
+      [Attributes.TX_COLLECTION_METHOD]: [
+        'fast-req-resp',
+        'fast-node-rpc',
+        'slow-req-resp',
+        'slow-node-rpc',
+        'file-store',
+      ],
     });
 
-    this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX, {
-      unit: 'ms',
-      description: 'Average duration per tx of an individual tx collection request',
-      valueType: ValueType.INT,
-    });
+    this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX);
 
-    this.collectionDurationPerRequest = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_REQUEST, {
-      unit: 'ms',
-      description: 'Total duration of an individual tx collection request',
-      valueType: ValueType.INT,
-    });
+    this.collectionDurationPerRequest = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_REQUEST);
   }
 
   increaseTxsFor(what: CollectionMethod, count: number, duration: number) {

package/src/services/tx_collection/proposal_tx_collector.ts

@@ -0,0 +1,112 @@
+import type { Logger } from '@aztec/foundation/log';
+import type { DateProvider } from '@aztec/foundation/timer';
+import type { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { PeerId } from '@libp2p/interface';
+
+import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js';
+import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
+import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
+import type { IBatchRequestTxValidator } from '../reqresp/batch-tx-requester/tx_validator.js';
+import { type BlockTxsSource, ReqRespSubProtocol, chunkTxHashesRequest } from '../reqresp/index.js';
+
+/**
+ * Strategy interface for collecting missing transactions for a block or proposal.
+ * Allows swapping between different tx collection implementations for benchmarking.
+ */
+export interface MissingTxsCollector {
+  /**
+   * Collect missing transactions for a block or proposal.
+   * @param txHashes - The transaction hashes to collect
+   * @param blockTxsSource - The block or proposal containing the transactions
+   * @param pinnedPeer - Optional peer expected to have the transactions
+   * @param timeoutMs - Timeout in milliseconds
+   * @returns The collected transactions
+   */
+  collectTxs(
+    txHashes: TxHash[],
+    blockTxsSource: BlockTxsSource,
+    pinnedPeer: PeerId | undefined,
+    timeoutMs: number,
+  ): Promise<Tx[]>;
+}
+
+/**
+ * Collects transactions using the BatchTxRequester implementation.
+ * This uses a smart/dumb peer strategy with parallel workers.
+ */
+export class BatchTxRequesterCollector implements MissingTxsCollector {
+  constructor(
+    private p2pService: BatchTxRequesterLibP2PService,
+    private log: Logger,
+    private dateProvider: DateProvider,
+    private txValidator?: IBatchRequestTxValidator,
+    private batchTxRequesterConfig?: Partial<BatchTxRequesterConfig>,
+  ) {}
+
+  async collectTxs(
+    txHashes: TxHash[],
+    blockTxsSource: BlockTxsSource,
+    pinnedPeer: PeerId | undefined,
+    timeoutMs: number,
+  ): Promise<Tx[]> {
+    const {
+      batchTxRequesterSmartParallelWorkerCount: smartParallelWorkerCount,
+      batchTxRequesterDumbParallelWorkerCount: dumbParallelWorkerCount,
+      batchTxRequesterTxBatchSize: txBatchSize,
+      batchTxRequesterBadPeerThreshold: badPeerThreshold,
+    } = this.batchTxRequesterConfig ?? {};
+
+    const batchRequester = new BatchTxRequester(
+      txHashes,
+      blockTxsSource,
+      pinnedPeer,
+      timeoutMs,
+      this.p2pService,
+      this.log,
+      this.dateProvider,
+      {
+        smartParallelWorkerCount,
+        dumbParallelWorkerCount,
+        txBatchSize,
+        badPeerThreshold,
+        txValidator: this.txValidator,
+      },
+    );
+
+    return await BatchTxRequester.collectAllTxs(batchRequester.run());
+  }
+}
+
+const DEFAULT_MAX_PEERS = 10;
+const DEFAULT_MAX_RETRY_ATTEMPTS = 3;
+
+/**
+ * Collects transactions using the sendBatchRequest implementation from ReqResp.
+ * This is the original implementation that balances requests across peers.
+ */
+export class SendBatchRequestCollector implements MissingTxsCollector {
+  constructor(
+    private p2pService: BatchTxRequesterLibP2PService,
+    private maxPeers: number = DEFAULT_MAX_PEERS,
+    private maxRetryAttempts: number = DEFAULT_MAX_RETRY_ATTEMPTS,
+  ) {}
+
+  async collectTxs(
+    txHashes: TxHash[],
+    _blockTxsSource: BlockTxsSource,
+    pinnedPeer: PeerId | undefined,
+    timeoutMs: number,
+  ): Promise<Tx[]> {
+    const txs = await this.p2pService.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
+      ReqRespSubProtocol.TX,
+      chunkTxHashesRequest(txHashes),
+      pinnedPeer,
+      timeoutMs,
+      this.maxPeers,
+      this.maxRetryAttempts,
+    );
+
+    return txs.flat();
+  }
+}

package/src/services/tx_collection/slow_tx_collection.ts

@@ -4,7 +4,7 @@ import { type Logger, createLogger } from '@aztec/foundation/log';
 import { boundInclusive } from '@aztec/foundation/number';
 import { RunningPromise } from '@aztec/foundation/promise';
 import { DateProvider } from '@aztec/foundation/timer';
-import type { L2BlockNew } from '@aztec/stdlib/block';
+import type { L2Block } from '@aztec/stdlib/block';
 import { type L1RollupConstants, getEpochAtSlot, getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
 import { type Tx, TxHash } from '@aztec/stdlib/tx';
 
@@ -76,7 +76,7 @@ export class SlowTxCollection {
   }
 
   /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
-  public startCollecting(block: L2BlockNew, txHashes: TxHash[]) {
+  public startCollecting(block: L2Block, txHashes: TxHash[]) {
     const slot = block.header.getSlot();
     const deadline = this.getDeadlineForSlot(slot);
     if (+deadline < this.dateProvider.now()) {
@@ -91,6 +91,7 @@ export class SlowTxCollection {
 
     for (const txHash of txHashes) {
       this.missingTxs.set(txHash.toString(), {
+        block,
         blockNumber: block.number,
         deadline: this.getDeadlineForSlot(block.header.getSlot()),
         readyForReqResp: this.nodes.length === 0, // If we have no nodes, we can start reqresp immediately
@@ -109,18 +110,26 @@ export class SlowTxCollection {
 
     // Gather all missing txs that are not in fast collection and request them from the node
     const missingTxs = this.getMissingTxsForSlowCollection();
-    const missingTxHashes = missingTxs.map(([txHash]) => txHash).map(TxHash.fromString);
-    if (missingTxHashes.length === 0) {
+    if (missingTxs.length === 0) {
       return;
     }
 
-    // Request in chunks to avoid hitting RPC limits
-    for (const batch of chunk(missingTxHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
-      await this.txCollectionSink.collect(txHashes => node.getTxsByHash(txHashes), batch, {
-        description: `node ${node.getInfo()}`,
-        node: node.getInfo(),
-        method: 'slow-node-rpc',
-      });
+    // Group by block so we pass the correct mined context to the sink
+    for (const entries of this.groupByBlock(missingTxs)) {
+      const block = entries[0][1].block;
+      const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
+      for (const batch of chunk(txHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
+        await this.txCollectionSink.collect(
+          hashes => node.getTxsByHash(hashes),
+          batch,
+          {
+            description: `node ${node.getInfo()}`,
+            node: node.getInfo(),
+            method: 'slow-node-rpc',
+          },
+          { type: 'mined', block },
+        );
+      }
     }
 
     // Mark every tx that is still missing as ready for reqresp.
@@ -149,25 +158,30 @@ export class SlowTxCollection {
 
     const pinnedPeer = undefined;
     const timeoutMs = this.config.txCollectionSlowReqRespTimeoutMs;
-    const maxPeers = boundInclusive(Math.ceil(missingTxs.length / 3), 4, 16);
     const maxRetryAttempts = 3;
-    // Send a batch request via reqresp for the missing txs
-    await this.txCollectionSink.collect(
-      async txHashes => {
-        const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
-          ReqRespSubProtocol.TX,
-          chunkTxHashesRequest(txHashes),
-          pinnedPeer,
-          timeoutMs,
-          maxPeers,
-          maxRetryAttempts,
-        );
 
-        return txs.flat();
-      },
-      missingTxs.map(([txHash]) => TxHash.fromString(txHash)),
-      { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
-    );
+    // Group by block so we pass the correct mined context to the sink
+    for (const entries of this.groupByBlock(missingTxs)) {
+      const block = entries[0][1].block;
+      const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
+      const maxPeers = boundInclusive(Math.ceil(txHashes.length / 3), 4, 16);
+      await this.txCollectionSink.collect(
+        async hashes => {
+          const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
+            ReqRespSubProtocol.TX,
+            chunkTxHashesRequest(hashes),
+            pinnedPeer,
+            timeoutMs,
+            maxPeers,
+            maxRetryAttempts,
+          );
+          return txs.flat();
+        },
+        txHashes,
+        { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
+        { type: 'mined', block },
+      );
+    }
   }
 
   /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */
@@ -223,11 +237,31 @@ export class SlowTxCollection {
     }
   }
 
+  /** Groups missing tx entries by block number. */
+  private groupByBlock(entries: [string, MissingTxInfo][]): [string, MissingTxInfo][][] {
+    const groups = new Map<number, [string, MissingTxInfo][]>();
+    for (const entry of entries) {
+      const bn = +entry[1].blockNumber;
+      let group = groups.get(bn);
+      if (!group) {
+        group = [];
+        groups.set(bn, group);
+      }
+      group.push(entry);
+    }
+    return [...groups.values()];
+  }
+
   /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */
   private getDeadlineForSlot(slotNumber: SlotNumber): Date {
-    const epoch = getEpochAtSlot(slotNumber, this.constants);
-    const submissionEndEpoch = EpochNumber(epoch + this.constants.proofSubmissionEpochs);
-    const submissionEndTimestamp = getTimestampRangeForEpoch(submissionEndEpoch, this.constants)[1];
-    return new Date(Number(submissionEndTimestamp) * 1000);
+    return getProofDeadlineForSlot(slotNumber, this.constants);
   }
 }
+
+/** Computes the proof submission deadline for a given slot. A tx mined in this slot is no longer interesting after this deadline. */
+export function getProofDeadlineForSlot(slotNumber: SlotNumber, constants: L1RollupConstants): Date {
+  const epoch = getEpochAtSlot(slotNumber, constants);
+  const submissionEndEpoch = EpochNumber(epoch + constants.proofSubmissionEpochs);
+  const submissionEndTimestamp = getTimestampRangeForEpoch(submissionEndEpoch, constants)[1];
+  return new Date(Number(submissionEndTimestamp) * 1000);
+}