@0xbow/privacy-pools-core-sdk 1.0.4 → 1.1.0

package/src/core/data.service.ts

@@ -1,20 +1,12 @@
-import {
-  type PublicClient,
-  createPublicClient,
-  http,
-  parseAbiItem,
-} from "viem";
-import {
-  ChainConfig,
-  DepositEvent,
-  WithdrawalEvent,
-  RagequitEvent,
-} from "../types/events.js";
-import { PoolInfo } from "../types/account.js";
-import { Hash } from "../types/commitment.js";
-import { Logger } from "../utils/logger.js";
-import { DataError } from "../errors/data.error.js";
-import { ErrorCode } from "../errors/base.error.js";
+import {createPublicClient, type Hex, http, parseAbiItem, type PublicClient,} from "viem";
+import {mapLimit} from "async";
+import {ChainConfig, DepositEvent, RagequitEvent, WithdrawalEvent,} from "../types/events.js";
+import {PoolInfo} from "../types/account.js";
+import {Hash} from "../types/commitment.js";
+import {BlockRange, ChainLogFetchConfig, DEFAULT_LOG_FETCH_CONFIG, LogFetchConfig,} from "../types/rateLimit.js";
+import {Logger, LogLevel} from "../utils/logger.js";
+import {DataError} from "../errors/data.error.js";
+import {ErrorCode} from "../errors/base.error.js";
 
 // Event signatures from the contract
 const DEPOSIT_EVENT = parseAbiItem('event Deposited(address indexed _depositor, uint256 _commitment, uint256 _label, uint256 _value, uint256 _merkleRoot)');
@@ -33,15 +25,31 @@ const RAGEQUIT_EVENT = parseAbiItem('event Ragequit(address indexed _ragequitter
 export class DataService {
   private readonly clients: Map<number, PublicClient> = new Map();
   private readonly logger: Logger;
+  private readonly logFetchConfigs: Map<number, LogFetchConfig>;
 
   /**
    * Initialize the data service with chain configurations
    *
    * @param chainConfigs - Array of chain configurations containing chainId, RPC URL, and API key
+   * @param logFetchConfig - Per-chain configuration for rate-limited log fetching as a Map<chainId, config>.
+   *                         Each chain can have its own specific settings (e.g., different block chunk sizes).
    * @throws {DataError} If client initialization fails for any chain
    */
-  constructor(private readonly chainConfigs: ChainConfig[]) {
-    this.logger = new Logger({ prefix: "Data" });
+  constructor(
+    private readonly chainConfigs: ChainConfig[],
+    logFetchConfig: ChainLogFetchConfig = new Map()
+  ) {
+    this.logger = new Logger({ prefix: "Data", level: LogLevel.DEBUG });
+
+    // Initialize per-chain configs with defaults merged with chain-specific overrides
+    this.logFetchConfigs = new Map();
+    for (const config of chainConfigs) {
+      const chainSpecificConfig = logFetchConfig.get(config.chainId);
+      this.logFetchConfigs.set(
+        config.chainId,
+        { ...DEFAULT_LOG_FETCH_CONFIG, ...chainSpecificConfig }
+      );
+    }
 
     try {
       for (const config of chainConfigs) {
@@ -50,9 +58,7 @@ export class DataService {
         }
 
         const client = createPublicClient({
-          transport: http(config.rpcUrl, {
-            timeout: 20_000,
-          }),
+          transport: http(config.rpcUrl),
         });
         this.clients.set(config.chainId, client);
       }
@@ -68,33 +74,64 @@ export class DataService {
   /**
    * Get deposit events for a specific chain
    *
-   * @param chainId - Chain ID to fetch events from
-   * @param options - Event filter options including fromBlock, toBlock, and other filters
+   * @param pool - Pool info containing chainId, address, and deployment block
    * @returns Array of deposit events with properly typed fields (bigint for numbers, Hash for commitments)
    * @throws {DataError} If client is not configured, network error occurs, or event data is invalid
    */
-  async getDeposits(
-    pool: PoolInfo
-  ): Promise<DepositEvent[]> {
+  async getDeposits(pool: PoolInfo): Promise<DepositEvent[]> {
     try {
       const client = this.getClientForChain(pool.chainId);
-      const config = this.getConfigForChain(pool.chainId);
-
-      const logs = await client.getLogs({
-        address: pool.address,
-        event: DEPOSIT_EVENT,
-        fromBlock: pool.deploymentBlock ?? config.startBlock
-      }).catch(error => {
-        throw new DataError(
-          "Failed to fetch deposit logs",
-          ErrorCode.NETWORK_ERROR,
-          { error: error instanceof Error ? error.message : "Unknown error" },
-        );
-      });
+      const chainConfig = this.getConfigForChain(pool.chainId);
+      const logConfig = this.getLogFetchConfigForChain(pool.chainId);
+
+      const fromBlock = pool.deploymentBlock ?? chainConfig.startBlock;
+      const toBlock = await this.getCurrentBlock(pool.chainId);
+      const ranges = this.generateBlockRanges(
+        fromBlock,
+        toBlock,
+        logConfig.blockChunkSize
+      );
+
+      this.logger.info(
+        `Fetching deposits in ${ranges.length} chunks for pool ${pool.address}, chunk size is: ${logConfig.blockChunkSize}`
+      );
 
-      return logs.map((log) => {
+      // Use async.mapLimit for controlled concurrency
+      const allLogs = await mapLimit<BlockRange, unknown[]>(
+        ranges,
+        logConfig.concurrency,
+        async (range: BlockRange) => {
+          if (logConfig.chunkDelayMs > 0) {
+            await this.sleep(logConfig.chunkDelayMs);
+          }
+          return this.fetchLogsWithRetry(
+            client,
+            pool.address,
+            DEPOSIT_EVENT,
+            range,
+            logConfig
+          );
+        }
+      );
+
+      // Flatten and parse results
+      const flatLogs = allLogs.flat();
+
+      return flatLogs.map((log: unknown) => {
         try {
-          if (!log.args) {
+          const typedLog = log as {
+            args?: {
+              _depositor?: string;
+              _commitment?: bigint;
+              _label?: bigint;
+              _value?: bigint;
+              _merkleRoot?: bigint;
+            };
+            blockNumber?: bigint;
+            transactionHash?: Hex;
+          };
+
+          if (!typedLog.args) {
             throw DataError.invalidLog("deposit", "missing args");
           }
 
@@ -104,9 +141,16 @@ export class DataService {
             _label: label,
             _value: value,
             _merkleRoot: precommitment,
-          } = log.args;
+          } = typedLog.args;
 
-          if (!depositor || !commitment || !label || !precommitment || !log.blockNumber || !log.transactionHash) {
+          if (
+            !depositor ||
+            !commitment ||
+            !label ||
+            !precommitment ||
+            !typedLog.blockNumber ||
+            !typedLog.transactionHash
+          ) {
             throw DataError.invalidLog("deposit", "missing required fields");
           }
 
@@ -116,25 +160,31 @@ export class DataService {
             label: label as Hash,
             value: value || BigInt(0),
             precommitment: precommitment as Hash,
-            blockNumber: BigInt(log.blockNumber),
-            transactionHash: log.transactionHash,
+            blockNumber: BigInt(typedLog.blockNumber),
+            transactionHash: typedLog.transactionHash,
           };
         } catch (error) {
           if (error instanceof DataError) throw error;
-          throw DataError.invalidLog("deposit", error instanceof Error ? error.message : "Unknown error");
+          throw DataError.invalidLog(
+            "deposit",
+            error instanceof Error ? error.message : "Unknown error"
+          );
         }
       });
     } catch (error) {
       if (error instanceof DataError) throw error;
-      throw DataError.networkError(pool.chainId, error instanceof Error ? error : new Error(String(error)));
+      throw DataError.networkError(
+        pool.chainId,
+        error instanceof Error ? error : new Error(String(error))
+      );
     }
   }
 
   /**
    * Get withdrawal events for a specific chain
    *
-   * @param chainId - Chain ID to fetch events from
-   * @param options - Event filter options including fromBlock, toBlock, and other filters
+   * @param pool - Pool info containing chainId, address, and deployment block
+   * @param fromBlock - Optional starting block (defaults to pool deployment block)
    * @returns Array of withdrawal events with properly typed fields (bigint for numbers, Hash for commitments)
    * @throws {DataError} If client is not configured, network error occurs, or event data is invalid
    */
@@ -144,23 +194,55 @@ export class DataService {
   ): Promise<WithdrawalEvent[]> {
     try {
       const client = this.getClientForChain(pool.chainId);
-      const config = this.getConfigForChain(pool.chainId);
-
-      const logs = await client.getLogs({
-        address: pool.address,
-        event: WITHDRAWAL_EVENT,
-        fromBlock: fromBlock ?? config.startBlock,
-      }).catch(error => {
-        throw new DataError(
-          "Failed to fetch withdrawal logs",
-          ErrorCode.NETWORK_ERROR,
-          { error: error instanceof Error ? error.message : "Unknown error" },
-        );
-      });
+      const chainConfig = this.getConfigForChain(pool.chainId);
+      const logConfig = this.getLogFetchConfigForChain(pool.chainId);
+
+      const startBlock = fromBlock ?? chainConfig.startBlock;
+      const toBlock = await this.getCurrentBlock(pool.chainId);
+      const ranges = this.generateBlockRanges(
+        startBlock,
+        toBlock,
+        logConfig.blockChunkSize
+      );
 
-      return logs.map((log) => {
+      this.logger.debug(
+        `Fetching withdrawals in ${ranges.length} chunks for pool ${pool.address}`
+      );
+
+      // Use async.mapLimit for controlled concurrency
+      const allLogs = await mapLimit<BlockRange, unknown[]>(
+        ranges,
+        logConfig.concurrency,
+        async (range: BlockRange) => {
+          if (logConfig.chunkDelayMs > 0) {
+            await this.sleep(logConfig.chunkDelayMs);
+          }
+          return this.fetchLogsWithRetry(
+            client,
+            pool.address,
+            WITHDRAWAL_EVENT,
+            range,
+            logConfig
+          );
+        }
+      );
+
+      // Flatten and parse results
+      const flatLogs = allLogs.flat();
+
+      return flatLogs.map((log: unknown) => {
         try {
-          if (!log.args) {
+          const typedLog = log as {
+            args?: {
+              _value?: bigint;
+              _spentNullifier?: bigint;
+              _newCommitment?: bigint;
+            };
+            blockNumber?: bigint;
+            transactionHash?: Hex;
+          };
+
+          if (!typedLog.args) {
             throw DataError.invalidLog("withdrawal", "missing args");
           }
 
@@ -168,9 +250,15 @@ export class DataService {
             _value: value,
             _spentNullifier: spentNullifier,
             _newCommitment: newCommitment,
-          } = log.args;
+          } = typedLog.args;
 
-          if (!value || !spentNullifier || !newCommitment || !log.blockNumber || !log.transactionHash) {
+          if (
+            !value ||
+            !spentNullifier ||
+            !newCommitment ||
+            !typedLog.blockNumber ||
+            !typedLog.transactionHash
+          ) {
             throw DataError.invalidLog("withdrawal", "missing required fields");
           }
 
@@ -178,25 +266,31 @@ export class DataService {
             withdrawn: value,
             spentNullifier: spentNullifier as Hash,
             newCommitment: newCommitment as Hash,
-            blockNumber: BigInt(log.blockNumber),
-            transactionHash: log.transactionHash,
+            blockNumber: BigInt(typedLog.blockNumber),
+            transactionHash: typedLog.transactionHash,
           };
         } catch (error) {
           if (error instanceof DataError) throw error;
-          throw DataError.invalidLog("withdrawal", error instanceof Error ? error.message : "Unknown error");
+          throw DataError.invalidLog(
+            "withdrawal",
+            error instanceof Error ? error.message : "Unknown error"
+          );
         }
       });
     } catch (error) {
       if (error instanceof DataError) throw error;
-      throw DataError.networkError(pool.chainId, error instanceof Error ? error : new Error(String(error)));
+      throw DataError.networkError(
+        pool.chainId,
+        error instanceof Error ? error : new Error(String(error))
+      );
     }
   }
 
   /**
    * Get ragequit events for a specific chain
    *
-   * @param chainId - Chain ID to fetch events from
-   * @param options - Event filter options including fromBlock, toBlock, and other filters
+   * @param pool - Pool info containing chainId, address, and deployment block
+   * @param fromBlock - Optional starting block (defaults to pool deployment block)
    * @returns Array of ragequit events with properly typed fields (bigint for numbers, Hash for commitments)
    * @throws {DataError} If client is not configured, network error occurs, or event data is invalid
    */
@@ -206,23 +300,56 @@ export class DataService {
   ): Promise<RagequitEvent[]> {
     try {
       const client = this.getClientForChain(pool.chainId);
-      const config = this.getConfigForChain(pool.chainId);
-
-      const logs = await client.getLogs({
-        address: pool.address,
-        event: RAGEQUIT_EVENT,
-        fromBlock: fromBlock ?? config.startBlock,
-      }).catch(error => {
-        throw new DataError(
-          "Failed to fetch ragequit logs",
-          ErrorCode.NETWORK_ERROR,
-          { error: error instanceof Error ? error.message : "Unknown error" },
-        );
-      });
+      const chainConfig = this.getConfigForChain(pool.chainId);
+      const logConfig = this.getLogFetchConfigForChain(pool.chainId);
+
+      const startBlock = fromBlock ?? chainConfig.startBlock;
+      const toBlock = await this.getCurrentBlock(pool.chainId);
+      const ranges = this.generateBlockRanges(
+        startBlock,
+        toBlock,
+        logConfig.blockChunkSize
+      );
+
+      this.logger.debug(
+        `Fetching ragequits in ${ranges.length} chunks for pool ${pool.address}`
+      );
+
+      // Use async.mapLimit for controlled concurrency
+      const allLogs = await mapLimit<BlockRange, unknown[]>(
+        ranges,
+        logConfig.concurrency,
+        async (range: BlockRange) => {
+          if (logConfig.chunkDelayMs > 0) {
+            await this.sleep(logConfig.chunkDelayMs);
+          }
+          return this.fetchLogsWithRetry(
+            client,
+            pool.address,
+            RAGEQUIT_EVENT,
+            range,
+            logConfig
+          );
+        }
+      );
+
+      // Flatten and parse results
+      const flatLogs = allLogs.flat();
 
-      return logs.map((log) => {
+      return flatLogs.map((log: unknown) => {
         try {
-          if (!log.args) {
+          const typedLog = log as {
+            args?: {
+              _ragequitter?: string;
+              _commitment?: bigint;
+              _label?: bigint;
+              _value?: bigint;
+            };
+            blockNumber?: bigint;
+            transactionHash?: Hex;
+          };
+
+          if (!typedLog.args) {
             throw DataError.invalidLog("ragequit", "missing args");
           }
 
@@ -231,9 +358,15 @@ export class DataService {
             _commitment: commitment,
             _label: label,
             _value: value,
-          } = log.args;
+          } = typedLog.args;
 
-          if (!ragequitter || !commitment || !label || !log.blockNumber || !log.transactionHash) {
+          if (
+            !ragequitter ||
+            !commitment ||
+            !label ||
+            !typedLog.blockNumber ||
+            !typedLog.transactionHash
+          ) {
             throw DataError.invalidLog("ragequit", "missing required fields");
           }
 
@@ -242,20 +375,107 @@ export class DataService {
             commitment: commitment as Hash,
             label: label as Hash,
             value: value || BigInt(0),
-            blockNumber: BigInt(log.blockNumber),
-            transactionHash: log.transactionHash,
+            blockNumber: BigInt(typedLog.blockNumber),
+            transactionHash: typedLog.transactionHash,
           };
         } catch (error) {
           if (error instanceof DataError) throw error;
-          throw DataError.invalidLog("ragequit", error instanceof Error ? error.message : "Unknown error");
+          throw DataError.invalidLog(
+            "ragequit",
+            error instanceof Error ? error.message : "Unknown error"
+          );
         }
       });
     } catch (error) {
       if (error instanceof DataError) throw error;
-      throw DataError.networkError(pool.chainId, error instanceof Error ? error : new Error(String(error)));
+      throw DataError.networkError(
+        pool.chainId,
+        error instanceof Error ? error : new Error(String(error))
+      );
     }
   }
 
+  /**
+   * Gets the current block number for a chain
+   */
+  private async getCurrentBlock(chainId: number): Promise<bigint> {
+    const client = this.getClientForChain(chainId);
+    return client.getBlockNumber();
+  }
+
+  /**
+   * Generates block ranges for chunked fetching
+   */
+  private generateBlockRanges(
+    fromBlock: bigint,
+    toBlock: bigint,
+    chunkSize: number
+  ): BlockRange[] {
+    const ranges: BlockRange[] = [];
+    let current = fromBlock;
+
+    while (current <= toBlock) {
+      const end = current + BigInt(chunkSize) - 1n;
+      ranges.push({
+        fromBlock: current,
+        toBlock: end > toBlock ? toBlock : end,
+      });
+      current = end + 1n;
+    }
+
+    return ranges;
+  }
+
+  /**
+   * Fetches logs for a single block range with retry logic
+   */
+  private async fetchLogsWithRetry<T>(
+    client: PublicClient,
+    address: string,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    event: any,
+    range: BlockRange,
+    logConfig: LogFetchConfig
+  ): Promise<T[]> {
+    const maxRetries = logConfig.retryOnFailure
+      ? logConfig.maxRetries
+      : 0;
+    let lastError: Error | undefined;
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        const logs = await client.getLogs({
+          address: address as `0x${string}`,
+          event,
+          fromBlock: range.fromBlock,
+          toBlock: range.toBlock,
+        });
+        return logs as T[];
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+
+        if (attempt < maxRetries) {
+          const delay =
+            logConfig.retryBaseDelayMs * Math.pow(2, attempt);
+          this.logger.warn(
+            `Log fetch failed, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})`,
+            { error: lastError.message, range }
+          );
+          await this.sleep(delay);
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
+  /**
+   * Helper to add delay between requests
+   */
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
   private getClientForChain(chainId: number): PublicClient {
     const client = this.clients.get(chainId);
     if (!client) {
@@ -265,10 +485,19 @@ export class DataService {
   }
 
   private getConfigForChain(chainId: number): ChainConfig {
-    const config = this.chainConfigs.find(c => c.chainId === chainId);
+    const config = this.chainConfigs.find((c) => c.chainId === chainId);
     if (!config) {
       throw DataError.chainNotConfigured(chainId);
     }
     return config;
   }
+
+  private getLogFetchConfigForChain(chainId: number): LogFetchConfig {
+    const config = this.logFetchConfigs.get(chainId);
+    if (!config) {
+      // Fallback to default if not found (shouldn't happen if constructor is correct)
+      return DEFAULT_LOG_FETCH_CONFIG;
+    }
+    return config;
+  }
 }

package/src/types/index.ts

@@ -1,3 +1,4 @@
 export * from "./commitment.js";
 export * from "./withdrawal.js";
 export * from "./keys.js";
+export * from "./rateLimit.js";

package/src/types/rateLimit.ts

@@ -0,0 +1,66 @@
+/**
+ * Configuration options for rate-limited log fetching
+ */
+export interface LogFetchConfig {
+  /**
+   * Maximum number of blocks to fetch in a single RPC call.
+   * Default: 10000 (typical limit for most RPC providers)
+   */
+  blockChunkSize: number;
+
+  /**
+   * Maximum number of concurrent log fetch operations.
+   * Default: 3
+   */
+  concurrency: number;
+
+  /**
+   * Delay in milliseconds between chunk requests (for additional throttling).
+   * Default: 0 (no delay)
+   */
+  chunkDelayMs: number;
+
+  /**
+   * Whether to retry failed chunk fetches.
+   * Default: true
+   */
+  retryOnFailure: boolean;
+
+  /**
+   * Maximum number of retries for a failed chunk.
+   * Default: 3
+   */
+  maxRetries: number;
+
+  /**
+   * Base delay for exponential backoff on retries (ms).
+   * Default: 1000
+   */
+  retryBaseDelayMs: number;
+}
+
+/**
+ * Default log fetch configuration
+ */
+export const DEFAULT_LOG_FETCH_CONFIG: LogFetchConfig = {
+  blockChunkSize: 10000,
+  concurrency: 3,
+  chunkDelayMs: 0,
+  retryOnFailure: true,
+  maxRetries: 3,
+  retryBaseDelayMs: 1000,
+};
+
+/**
+ * Per-chain log fetch configuration map
+ * Maps chainId to its specific LogFetchConfig
+ */
+export type ChainLogFetchConfig = Map<number, Partial<LogFetchConfig>>;
+
+/**
+ * Block range for chunked fetching
+ */
+export interface BlockRange {
+  fromBlock: bigint;
+  toBlock: bigint;
+}