curvance 4.0.3 → 4.1.0

package/dist/retry-provider.js

@@ -1,16 +1,43 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RetryableProvider = exports.DEFAULT_RETRY_CONFIG = void 0;
+exports.getRpcDebugSnapshot = getRpcDebugSnapshot;
+exports.subscribeToRpcDebug = subscribeToRpcDebug;
+exports.resetRpcDebugState = resetRpcDebugState;
 exports.configureRetries = configureRetries;
 exports.createRetryableProvider = createRetryableProvider;
 exports.wrapProviderWithRetries = wrapProviderWithRetries;
 exports.isRetryableProvider = isRetryableProvider;
+exports.isRetryableReadProvider = isRetryableReadProvider;
 exports.classifyError = classifyError;
+const rpc_1 = require("./chains/rpc");
+/** Named ethers provider methods that make RPC calls on the read transport. */
+const RPC_PROVIDER_METHODS = new Set([
+    'getBalance',
+    'getCode',
+    'getStorageAt',
+    'getTransactionCount',
+    'getBlock',
+    'getBlockNumber',
+    'getGasPrice',
+    'getFeeData',
+    'getTransaction',
+    'getTransactionReceipt',
+    'getLogs',
+    'getNetwork',
+    'detectNetwork',
+    'call',
+    'estimateGas',
+]);
 exports.DEFAULT_RETRY_CONFIG = {
-    maxRetries: 3,
-    baseDelay: 1000, // 1 second
-    maxDelay: 10000, // 10 seconds
+    maxRetries: rpc_1.DEFAULT_CHAIN_RPC_POLICY.retryCount,
+    baseDelay: rpc_1.DEFAULT_CHAIN_RPC_POLICY.retryDelayMs,
+    maxDelay: 1000,
     backoffMultiplier: 2,
+    timeoutMs: rpc_1.DEFAULT_CHAIN_RPC_POLICY.timeoutMs,
+    fallbackCooldownMs: rpc_1.DEFAULT_CHAIN_RPC_POLICY.fallbackCooldownMs,
+    rankSampleCount: rpc_1.DEFAULT_CHAIN_RPC_POLICY.rankSampleCount,
+    rankWeights: rpc_1.DEFAULT_CHAIN_RPC_POLICY.rankWeights,
     retryableErrors: [
         // Rate limiting
         'rate limit',
@@ -53,10 +80,71 @@ exports.DEFAULT_RETRY_CONFIG = {
         'try again'
     ]
 };
+const rpcDebugListeners = new Set();
+const rpcDebugStates = new Map();
+function normalizeRpcUrl(url) {
+    if (!url) {
+        return null;
+    }
+    return url.replace(/\/+$/, '');
+}
+function getProviderUrl(provider) {
+    const connection = provider._getConnection?.() ?? provider.connection ?? null;
+    return normalizeRpcUrl(connection?.url);
+}
+function getEndpointId(url, label) {
+    return url ?? label;
+}
+function cloneRpcDebugState(state) {
+    return { ...state };
+}
+function emitRpcDebugSnapshot() {
+    if (rpcDebugListeners.size === 0) {
+        return;
+    }
+    const snapshot = getRpcDebugSnapshot();
+    for (const listener of rpcDebugListeners) {
+        listener(snapshot);
+    }
+}
+function getRpcDebugSnapshot() {
+    return {
+        updatedAt: Date.now(),
+        endpoints: [...rpcDebugStates.values()]
+            .map(cloneRpcDebugState)
+            .sort((a, b) => {
+            if (a.role !== b.role) {
+                return a.role === 'primary' ? -1 : 1;
+            }
+            return a.label.localeCompare(b.label);
+        }),
+    };
+}
+function subscribeToRpcDebug(listener) {
+    rpcDebugListeners.add(listener);
+    listener(getRpcDebugSnapshot());
+    return () => {
+        rpcDebugListeners.delete(listener);
+    };
+}
+function resetRpcDebugState() {
+    rpcDebugStates.clear();
+    emitRpcDebugSnapshot();
+}
 class RetryableProvider {
     config;
-    constructor(config = {}) {
+    fallbackProviders;
+    fallbackProviderStates;
+    _fallbackActivated = false;
+    primaryReadCooldownUntil = 0;
+    constructor(config = {}, fallbackProviders = null) {
         this.config = { ...exports.DEFAULT_RETRY_CONFIG, ...config };
+        this.fallbackProviders = Array.isArray(fallbackProviders)
+            ? fallbackProviders
+            : fallbackProviders
+                ? [fallbackProviders]
+                : [];
+        this.fallbackProviderStates = this.fallbackProviders.map((provider, index) => this.createProviderState(provider, 'fallback', `fallback-${index + 1}`, index));
     }
     async sleep(ms) {
         return new Promise(resolve => setTimeout(resolve, ms));
@@ -67,6 +155,166 @@ class RetryableProvider {
         // Add jitter to prevent thundering herd when multiple clients retry simultaneously
         return capped * (0.5 + Math.random() * 0.5);
     }
+    createProviderState(provider, role, label, index) {
+        const url = getProviderUrl(provider);
+        const state = {
+            provider,
+            label,
+            role,
+            endpointId: getEndpointId(url, label),
+            url,
+            index,
+            cooldownUntil: 0,
+            lastFailureAt: 0,
+            lastSuccessAt: 0,
+            attempts: 0,
+            successes: 0,
+            retryableFailures: 0,
+            nonRetryableFailures: 0,
+            timeoutFailures: 0,
+            fallbackSelections: 0,
+            lastLatencyMs: null,
+            lastError: null,
+            lastAttemptAt: 0,
+            recentSamples: [],
+        };
+        this.publishDebugState(state);
+        return state;
+    }
+    getRecentSamples(state) {
+        return state.recentSamples.slice(-this.config.rankSampleCount);
+    }
+    getProviderSuccessRate(state) {
+        const samples = this.getRecentSamples(state);
+        if (samples.length === 0) {
+            return 0.5;
+        }
+        const successes = samples.filter((sample) => sample.success).length;
+        return successes / samples.length;
+    }
+    getProviderAverageLatency(state) {
+        const latencies = this.getRecentSamples(state)
+            .filter((sample) => sample.success && sample.latencyMs != null)
+            .map((sample) => sample.latencyMs);
+        if (latencies.length === 0) {
+            return null;
+        }
+        return latencies.reduce((sum, latency) => sum + latency, 0) / latencies.length;
+    }
+    getProviderRankScore(state) {
+        const averageLatencyMs = this.getProviderAverageLatency(state);
+        const latencyScore = averageLatencyMs == null
+            ? 0.5
+            : Math.max(0, 1 - Math.min(averageLatencyMs, this.config.timeoutMs) / this.config.timeoutMs);
+        const stabilityScore = this.getProviderSuccessRate(state);
+        return ((latencyScore * this.config.rankWeights.latency) +
+            (stabilityScore * this.config.rankWeights.stability));
+    }
+    publishDebugState(state) {
+        const current = rpcDebugStates.get(state.endpointId) ?? {
+            endpointId: state.endpointId,
+            label: state.label,
+            role: state.role,
+            url: state.url,
+            attempts: 0,
+            successes: 0,
+            retryableFailures: 0,
+            nonRetryableFailures: 0,
+            timeoutFailures: 0,
+            fallbackSelections: 0,
+            recentSampleCount: 0,
+            recentSuccessRate: null,
+            averageLatencyMs: null,
+            lastLatencyMs: null,
+            rankScore: null,
+            lastError: null,
+            lastAttemptAt: null,
+            lastSuccessAt: null,
+            lastFailureAt: null,
+            cooldownUntil: null,
+        };
+        current.label = state.label;
+        current.role = state.role;
+        current.url = state.url;
+        current.recentSampleCount = this.getRecentSamples(state).length;
+        current.recentSuccessRate = current.recentSampleCount > 0 ? this.getProviderSuccessRate(state) : null;
+        current.averageLatencyMs = this.getProviderAverageLatency(state);
+        current.lastLatencyMs = state.lastLatencyMs;
+        current.rankScore = this.getProviderRankScore(state);
+        current.lastError = state.lastError;
+        current.lastAttemptAt = state.lastAttemptAt || current.lastAttemptAt;
+        current.lastSuccessAt = state.lastSuccessAt || current.lastSuccessAt;
+        current.lastFailureAt = state.lastFailureAt || current.lastFailureAt;
+        current.cooldownUntil = state.cooldownUntil || null;
+        rpcDebugStates.set(state.endpointId, current);
+        emitRpcDebugSnapshot();
+    }
+    recordProviderSelection(state) {
+        state.fallbackSelections += 1;
+        const current = rpcDebugStates.get(state.endpointId);
+        if (current) {
+            current.fallbackSelections += 1;
+        }
+        this.publishDebugState(state);
+    }
+    recordProviderAttempt(state, latencyMs, error) {
+        const recordedAt = Date.now();
+        state.attempts += 1;
+        state.lastAttemptAt = recordedAt;
+        state.lastLatencyMs = latencyMs;
+        state.recentSamples.push({
+            success: error == null,
+            latencyMs,
+        });
+        if (state.recentSamples.length > this.config.rankSampleCount) {
+            state.recentSamples.shift();
+        }
+        const current = rpcDebugStates.get(state.endpointId);
+        if (current) {
+            current.attempts += 1;
+            current.lastAttemptAt = recordedAt;
+            current.lastLatencyMs = latencyMs;
+        }
+        if (error == null) {
+            state.successes += 1;
+            state.lastSuccessAt = recordedAt;
+            state.lastError = null;
+            if (current) {
+                current.successes += 1;
+                current.lastSuccessAt = recordedAt;
+                current.lastError = null;
+            }
+            this.publishDebugState(state);
+            return;
+        }
+        const isTimeout = error?.code === 'timeout' || String(error?.message ?? '').toLowerCase().includes('timeout');
+        const isRetryable = this.isRetryableError(error);
+        state.lastFailureAt = recordedAt;
+        state.lastError = error?.message ?? String(error);
+        if (isRetryable) {
+            state.retryableFailures += 1;
+        }
+        else {
+            state.nonRetryableFailures += 1;
+        }
+        if (isTimeout) {
+            state.timeoutFailures += 1;
+        }
+        if (current) {
+            current.lastFailureAt = recordedAt;
+            current.lastError = state.lastError;
+            if (isRetryable) {
+                current.retryableFailures += 1;
+            }
+            else {
+                current.nonRetryableFailures += 1;
+            }
+            if (isTimeout) {
+                current.timeoutFailures += 1;
+            }
+        }
+        this.publishDebugState(state);
+    }
     isRetryableError(error) {
         // First check for non-retryable smart contract errors
         const errorMessage = error?.message?.toLowerCase() || '';
@@ -109,20 +357,141 @@ class RetryableProvider {
             errorCode.includes(retryableError) ||
             errorStatus.includes(retryableError));
     }
-    async executeWithRetry(operation, context = 'RPC call') {
+    isReadFallbackActive(primaryState) {
+        if (this.primaryReadCooldownUntil === 0) {
+            return false;
+        }
+        if (Date.now() >= this.primaryReadCooldownUntil) {
+            this.primaryReadCooldownUntil = 0;
+            this._fallbackActivated = false;
+            if (primaryState) {
+                primaryState.cooldownUntil = 0;
+                this.publishDebugState(primaryState);
+            }
+            return false;
+        }
+        return true;
+    }
+    getFallbackProviderLabel(state) {
+        return state.label;
+    }
+    isProviderCoolingDown(cooldownUntil) {
+        return cooldownUntil > Date.now();
+    }
+    compareFallbackProviders(a, b) {
+        const aCooling = this.isProviderCoolingDown(a.cooldownUntil);
+        const bCooling = this.isProviderCoolingDown(b.cooldownUntil);
+        if (aCooling !== bCooling) {
+            return aCooling ? 1 : -1;
+        }
+        if (aCooling && bCooling) {
+            return a.cooldownUntil - b.cooldownUntil || a.index - b.index;
+        }
+        const aRankScore = this.getProviderRankScore(a);
+        const bRankScore = this.getProviderRankScore(b);
+        if (aRankScore !== bRankScore) {
+            return bRankScore - aRankScore;
+        }
+        const aSuccessRate = this.getProviderSuccessRate(a);
+        const bSuccessRate = this.getProviderSuccessRate(b);
+        if (aSuccessRate !== bSuccessRate) {
+            return bSuccessRate - aSuccessRate;
+        }
+        const aLatency = this.getProviderAverageLatency(a);
+        const bLatency = this.getProviderAverageLatency(b);
+        if (aLatency != null || bLatency != null) {
+            if (aLatency == null) {
+                return 1;
+            }
+            if (bLatency == null) {
+                return -1;
+            }
+            if (aLatency !== bLatency) {
+                return aLatency - bLatency;
+            }
+        }
+        if (a.lastSuccessAt !== b.lastSuccessAt) {
+            return b.lastSuccessAt - a.lastSuccessAt;
+        }
+        if (a.lastFailureAt !== b.lastFailureAt) {
+            return a.lastFailureAt - b.lastFailureAt;
+        }
+        return a.index - b.index;
+    }
+    getOrderedFallbackProviders() {
+        return [...this.fallbackProviderStates].sort((a, b) => this.compareFallbackProviders(a, b));
+    }
+    markFallbackFailure(state) {
+        state.lastFailureAt = Date.now();
+        state.cooldownUntil = state.lastFailureAt + this.config.fallbackCooldownMs;
+        this.publishDebugState(state);
+    }
+    markFallbackSuccess(state) {
+        state.lastSuccessAt = Date.now();
+        state.lastFailureAt = 0;
+        state.cooldownUntil = 0;
+        this.publishDebugState(state);
+    }
+    getOrderedFallbackOps(fallbackOps) {
+        const fallbackOpMap = new Map(fallbackOps.map((entry) => [entry.state, entry.operation]));
+        return this.getOrderedFallbackProviders()
+            .map((state) => {
+            const operation = fallbackOpMap.get(state);
+            if (!operation) {
+                return null;
+            }
+            return { state, operation };
+        })
+            .filter((entry) => entry != null);
+    }
+    async executeWithTimeout(operation, timeoutMs, context) {
+        if (timeoutMs <= 0) {
+            return operation();
+        }
+        let timeoutHandle = null;
+        try {
+            return await Promise.race([
+                operation(),
+                new Promise((_, reject) => {
+                    timeoutHandle = setTimeout(() => {
+                        const error = new Error(`[rpc] ${context}: timeout after ${timeoutMs}ms`);
+                        error.code = "timeout";
+                        reject(error);
+                    }, timeoutMs);
+                }),
+            ]);
+        }
+        finally {
+            if (timeoutHandle != null) {
+                clearTimeout(timeoutHandle);
+            }
+        }
+    }
+    withReadTimeout(operation, context) {
+        return () => this.executeWithTimeout(operation, this.config.timeoutMs, context);
+    }
+    async executeWithRetry(operation, context = 'RPC call', providerState = null) {
         let lastError;
         for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) {
+            const startedAt = Date.now();
             try {
-                return await operation();
+                const result = await operation();
+                if (providerState) {
+                    this.recordProviderAttempt(providerState, Date.now() - startedAt);
+                }
+                return result;
             }
             catch (error) {
+                if (providerState) {
+                    this.recordProviderAttempt(providerState, Date.now() - startedAt, error);
+                }
                 lastError = error;
                 const isRetryable = this.isRetryableError(error);
                 // Don't retry on the last attempt or if error is not retryable
                 if (attempt === this.config.maxRetries || !isRetryable) {
                     if (!isRetryable && attempt === 0) {
                         // Log that this error is not retryable for debugging
-                        console.debug(`${context} failed with non-retryable error: ${error.message}`);
+                        console.debug(`[rpc] ${context}: non-retryable`, error.message);
                     }
                     throw error;
                 }
@@ -131,17 +500,72 @@ class RetryableProvider {
                 if (this.config.onRetry) {
                     this.config.onRetry(attempt + 1, error, delay);
                 }
-                console.warn(`${context} failed (attempt ${attempt + 1}/${this.config.maxRetries + 1}): ${error.message}. Retrying in ${delay}ms...`);
+                console.debug(`[rpc] ${context}: attempt ${attempt + 1}/${this.config.maxRetries + 1} failed, retry in ${delay}ms`, error.message);
                 await this.sleep(delay);
             }
         }
         throw lastError;
     }
+    /**
+     * Execute a read operation against the primary provider, falling back to
+     * the dedicated RPC providers if the primary exhausts all retries.
+     *
+     * Write operations (signing, sending transactions) never use the fallback
+     * because the fallback providers cannot sign.
+     */
+    async executeFallbackChain(fallbackOps, context, originalError) {
+        let lastError = originalError;
+        for (const { state, operation } of fallbackOps) {
+            const fallbackContext = `${context} [${this.getFallbackProviderLabel(state)}]`;
+            try {
+                this.recordProviderSelection(state);
+                const result = await this.executeWithRetry(this.withReadTimeout(operation, fallbackContext), fallbackContext, state);
+                this.markFallbackSuccess(state);
+                return result;
+            }
+            catch (fallbackError) {
+                if (!this.isRetryableError(fallbackError)) {
+                    throw fallbackError;
+                }
+                this.markFallbackFailure(state);
+                lastError = fallbackError;
+                console.warn(`[rpc] ${fallbackContext} failed after ${this.config.maxRetries + 1} attempts. ` +
+                    `Trying the next configured read RPC.`);
+            }
+        }
+        throw lastError;
+    }
+    async executeWithReadFallback(primaryOp, fallbackOps, context, primaryState) {
+        const timedPrimaryOp = this.withReadTimeout(primaryOp, context);
+        const orderedFallbackOps = this.getOrderedFallbackOps(fallbackOps);
+        if (this.isReadFallbackActive(primaryState)) {
+            return this.executeFallbackChain(orderedFallbackOps, context);
+        }
+        try {
+            return await this.executeWithRetry(timedPrimaryOp, context, primaryState);
+        }
+        catch (primaryError) {
+            if (!this.isRetryableError(primaryError)) {
+                throw primaryError;
+            }
+            this.primaryReadCooldownUntil = Date.now() + this.config.fallbackCooldownMs;
+            primaryState.cooldownUntil = this.primaryReadCooldownUntil;
+            this.publishDebugState(primaryState);
+            if (!this._fallbackActivated) {
+                this._fallbackActivated = true;
+                console.warn(`[rpc] Primary provider failed for ${context} after ${this.config.maxRetries + 1} attempts. ` +
+                    `Falling back to dedicated RPCs for read operations for ${this.config.fallbackCooldownMs}ms.`);
+            }
+            return this.executeFallbackChain(orderedFallbackOps, context, primaryError);
+        }
+    }
     wrapProvider(provider) {
         // If it's already wrapped, return as-is
         if (provider._isRetryable) {
             return provider;
         }
+        const hasFallback = this.fallbackProviders.length > 0;
+        const primaryState = this.createProviderState(provider, 'primary', 'primary', -1);
         const retryableProvider = new Proxy(provider, {
             get: (target, prop, receiver) => {
                 const original = Reflect.get(target, prop, receiver);
@@ -152,19 +576,54 @@ class RetryableProvider {
                 // Wrap the main RPC send method
                 if (prop === 'send' && typeof original === 'function') {
                     return async (method, params) => {
-                        return this.executeWithRetry(() => original.apply(target, [method, params]), `RPC ${method}`);
+                        const primaryOp = () => original.apply(target, [method, params]);
+                        if (hasFallback) {
+                            const fallbackOps = this.fallbackProviderStates.map((state) => ({
+                                state,
+                                operation: () => state.provider.send(method, params),
+                            }));
+                            return this.executeWithReadFallback(primaryOp, fallbackOps, `RPC ${method}`, primaryState);
+                        }
+                        return this.executeWithRetry(primaryOp, `RPC ${method}`, primaryState);
                     };
                 }
                 // For JsonRpcProvider, also wrap _send if it exists
                 if (prop === '_send' && typeof original === 'function') {
                     return async (payload, callback) => {
-                        return this.executeWithRetry(() => original.apply(target, arguments), `RPC ${payload.method || 'unknown'}`);
+                        const method = payload.method || 'unknown';
+                        const primaryOp = () => original.apply(target, [payload, callback]);
+                        if (hasFallback) {
+                            const fallbackOps = this.fallbackProviderStates.map((state) => ({
+                                state,
+                                operation: () => state.provider._send(payload, callback),
+                            }));
+                            return this.executeWithReadFallback(primaryOp, fallbackOps, `RPC ${method}`, primaryState);
+                        }
+                        return this.executeWithRetry(primaryOp, `RPC ${method}`, primaryState);
                     };
                 }
                 // Wrap other async methods that might make RPC calls
                 if (typeof original === 'function' && this.isRpcMethod(prop)) {
                     return async (...args) => {
-                        return this.executeWithRetry(() => original.apply(target, args), `Provider method ${String(prop)}`);
+                        const primaryOp = () => original.apply(target, args);
+                        if (hasFallback) {
+                            const fallbackOps = this.fallbackProviderStates
+                                .map((state) => {
+                                const fbMethod = state.provider[prop];
+                                if (typeof fbMethod !== 'function') {
+                                    return null;
+                                }
+                                return {
+                                    state,
+                                    operation: () => fbMethod.apply(state.provider, args),
+                                };
+                            })
+                                .filter((entry) => entry != null);
+                            if (fallbackOps.length > 0) {
+                                return this.executeWithReadFallback(primaryOp, fallbackOps, `Provider method ${String(prop)}`, primaryState);
+                            }
+                        }
+                        return this.executeWithRetry(primaryOp, `Provider method ${String(prop)}`, primaryState);
                     };
                 }
                 // If it's a function, bind it to the target
@@ -180,26 +639,7 @@ class RetryableProvider {
         if (typeof prop !== 'string')
             return false;
         // Common ethers.js methods that make RPC calls
-        const rpcMethods = [
-            'getBalance',
-            'getCode',
-            'getStorageAt',
-            'getTransactionCount',
-            'getBlock',
-            'getBlockNumber',
-            'getGasPrice',
-            'getFeeData',
-            'getTransaction',
-            'getTransactionReceipt',
-            'call',
-            'estimateGas',
-            'sendTransaction',
-            'waitForTransaction',
-            'getLogs',
-            'getNetwork',
-            'detectNetwork'
-        ];
-        return rpcMethods.includes(prop);
+        return RPC_PROVIDER_METHODS.has(prop);
     }
     updateConfig(newConfig) {
         this.config = { ...this.config, ...newConfig };
@@ -225,8 +665,8 @@ function configureRetries(config = {}) {
 /**
  * Create a provider with retry capabilities
  */
-function createRetryableProvider(provider, config = {}) {
-    const retryProvider = new RetryableProvider(config);
+function createRetryableProvider(provider, config = {}, readFallback = null) {
+    const retryProvider = new RetryableProvider(config, readFallback);
     return retryProvider.wrapProvider(provider);
 }
 /**
@@ -239,11 +679,22 @@ function getGlobalRetryProvider() {
     return globalRetryProvider;
 }
 /**
- * Wrap a provider with the global retry configuration
+ * Wrap a provider with the global retry configuration.
+ *
+ * When `readFallback` is supplied, read-only RPC methods (eth_call,
+ * eth_getBalance, etc.) will fall through to the fallback providers after
+ * exhausting retries on the primary.  Write/signing methods never use
+ * the fallback because only the primary (wallet) provider can sign.
  */
-function wrapProviderWithRetries(provider) {
-    const retryProvider = getGlobalRetryProvider();
-    return retryProvider.wrapProvider(provider);
+function wrapProviderWithRetries(provider, readFallback = null) {
+    const hasFallback = Array.isArray(readFallback) ? readFallback.length > 0 : readFallback != null;
+    if (hasFallback) {
+        // Fallback is per-invocation — create a dedicated instance so the
+        // fallback providers aren't shared across setupChain calls.
+        const retryProvider = new RetryableProvider(getGlobalRetryProvider().getConfig(), readFallback);
+        return retryProvider.wrapProvider(provider);
+    }
+    return getGlobalRetryProvider().wrapProvider(provider);
 }
 /**
  * Utility function to check if a provider is already wrapped with retries
@@ -251,6 +702,9 @@ function wrapProviderWithRetries(provider) {
 function isRetryableProvider(provider) {
     return provider._isRetryable === true;
 }
+function isRetryableReadProvider(provider) {
+    return provider._isRetryable === true;
+}
 /**
  * Utility function to classify error types for debugging
  */