@monygroupcorp/micro-web3 0.1.1 → 1.2.0

package/src/indexer/SyncEngine.js

@@ -0,0 +1,494 @@
+// src/indexer/SyncEngine.js
+
+/**
+ * Sync engine for EventIndexer.
+ * Handles historical sync, real-time updates, and reorg recovery.
+ */
+class SyncEngine {
+  constructor(storage, queryEngine, eventBus, config = {}) {
+    this.storage = storage;
+    this.queryEngine = queryEngine;
+    this.eventBus = eventBus;
+
+    // Configuration with defaults
+    this.batchSize = config.batchSize || 2000;
+    this.confirmations = config.confirmations || 2;
+    this.realTimeEnabled = config.realTimeEnabled !== false;
+    this.retryAttempts = config.retryAttempts || 3;
+    this.retryDelay = config.retryDelay || 1000;
+    this.pollInterval = config.pollInterval || 12000;
+    this.reorgTracking = config.reorgTracking || false; // Full reorg tracking (optional)
+    this.reorgDepth = config.reorgDepth || 100; // How many blocks to track
+
+    // State
+    this.provider = null;
+    this.contract = null;
+    this.eventTypes = [];
+    this.deployBlock = 0;
+    this.chainId = null;
+
+    this.state = 'initializing'; // initializing | syncing | synced | paused | error
+    this.currentBlock = 0;
+    this.latestBlock = 0;
+    this.eventsIndexed = 0;
+    this.lastSyncTime = null;
+    this.error = null;
+
+    // Real-time sync
+    this.eventListeners = [];
+    this.pollTimer = null;
+    this.isPaused = false;
+
+    // Reorg tracking (optional)
+    this.recentBlocks = new Map(); // blockNumber -> blockHash
+  }
+
+  /**
+   * Initialize sync engine.
+   * @param {Object} params
+   * @param {ethers.Provider} params.provider - Ethereum provider
+   * @param {ethers.Contract} params.contract - Contract instance
+   * @param {Object[]} params.eventTypes - Event types from ABI
+   * @param {number} params.deployBlock - Starting block
+   * @param {number} params.chainId - Chain ID
+   */
+  async initialize({ provider, contract, eventTypes, deployBlock, chainId }) {
+    this.provider = provider;
+    this.contract = contract;
+    this.eventTypes = eventTypes;
+    this.deployBlock = deployBlock;
+    this.chainId = chainId;
+
+    // Validate chain ID matches stored data
+    await this._validateChainId();
+
+    // Load sync state
+    const syncState = await this.storage.getSyncState();
+    if (syncState) {
+      this.currentBlock = syncState.lastSyncedBlock;
+      this.eventsIndexed = Object.values(syncState.eventCounts || {}).reduce((a, b) => a + b, 0);
+      this.lastSyncTime = syncState.lastSyncTime;
+    } else {
+      this.currentBlock = this.deployBlock - 1;
+    }
+  }
+
+  async _validateChainId() {
+    const syncState = await this.storage.getSyncState();
+    if (syncState && syncState.chainId !== this.chainId) {
+      console.warn('[SyncEngine] Chain ID mismatch, clearing data');
+      await this.storage.clear();
+    }
+  }
+
+  /**
+   * Start syncing.
+   */
+  async start() {
+    if (this.state === 'synced' || this.state === 'syncing') return;
+
+    try {
+      // Historical sync
+      await this._syncHistorical();
+
+      // Start real-time sync
+      if (this.realTimeEnabled && !this.isPaused) {
+        await this._startRealTimeSync();
+      }
+    } catch (error) {
+      this.state = 'error';
+      this.error = error;
+      this.eventBus.emit('indexer:error', {
+        code: 'SYNC_ERROR',
+        message: error.message,
+        cause: error,
+        recoverable: true
+      });
+    }
+  }
+
+  async _syncHistorical() {
+    this.state = 'syncing';
+    this.latestBlock = await this.provider.getBlockNumber();
+
+    // Account for confirmations
+    const targetBlock = this.latestBlock - this.confirmations;
+
+    if (this.currentBlock >= targetBlock) {
+      this.state = 'synced';
+      this.eventBus.emit('indexer:syncComplete', {
+        eventsIndexed: this.eventsIndexed,
+        duration: 0
+      });
+      return;
+    }
+
+    const startTime = Date.now();
+    const startBlock = this.currentBlock + 1;
+
+    this.eventBus.emit('indexer:syncStarted', {
+      fromBlock: startBlock,
+      toBlock: targetBlock
+    });
+
+    // Process in batches
+    for (let fromBlock = startBlock; fromBlock <= targetBlock; fromBlock += this.batchSize) {
+      if (this.isPaused) break;
+
+      const toBlock = Math.min(fromBlock + this.batchSize - 1, targetBlock);
+
+      await this._syncBatch(fromBlock, toBlock);
+
+      // Emit progress
+      const progress = (toBlock - startBlock + 1) / (targetBlock - startBlock + 1);
+      this.eventBus.emit('indexer:syncProgress', {
+        progress,
+        currentBlock: toBlock,
+        latestBlock: this.latestBlock,
+        eventsIndexed: this.eventsIndexed
+      });
+    }
+
+    if (!this.isPaused) {
+      this.state = 'synced';
+      this.lastSyncTime = Date.now();
+
+      this.eventBus.emit('indexer:syncComplete', {
+        eventsIndexed: this.eventsIndexed,
+        duration: Date.now() - startTime
+      });
+    }
+  }
+
+  async _syncBatch(fromBlock, toBlock) {
+    const events = await this._fetchEventsWithRetry(fromBlock, toBlock);
+
+    if (events.length > 0) {
+      // Parse and index events
+      const indexedEvents = events.map(e => this._parseEvent(e));
+      await this.storage.putEvents(indexedEvents);
+      this.eventsIndexed += indexedEvents.length;
+
+      // Notify query engine
+      const eventsByType = this._groupByType(indexedEvents);
+      for (const [type, typeEvents] of Object.entries(eventsByType)) {
+        this.queryEngine.notifyNewEvents(type, typeEvents);
+      }
+    }
+
+    // Update sync state
+    this.currentBlock = toBlock;
+    await this._saveSyncState();
+  }
+
+  async _fetchEventsWithRetry(fromBlock, toBlock) {
+    for (let attempt = 0; attempt < this.retryAttempts; attempt++) {
+      try {
+        // Query all event types
+        const allEvents = [];
+        for (const eventType of this.eventTypes) {
+          const filter = this.contract.filters[eventType.name]();
+          const events = await this.contract.queryFilter(filter, fromBlock, toBlock);
+          allEvents.push(...events);
+        }
+        return allEvents;
+      } catch (error) {
+        if (attempt === this.retryAttempts - 1) throw error;
+
+        // Exponential backoff
+        const delay = this.retryDelay * Math.pow(2, attempt);
+        await this._sleep(delay);
+      }
+    }
+    return [];
+  }
+
+  _parseEvent(rawEvent) {
+    // Extract indexed and non-indexed params
+    const indexed = {};
+    const data = {};
+
+    if (rawEvent.args) {
+      const eventFragment = rawEvent.eventFragment ||
+        this.contract.interface.getEvent(rawEvent.event);
+
+      if (eventFragment) {
+        for (let i = 0; i < eventFragment.inputs.length; i++) {
+          const input = eventFragment.inputs[i];
+          const value = rawEvent.args[i];
+          const serialized = this._serializeValue(value);
+
+          if (input.indexed) {
+            indexed[input.name] = serialized;
+          }
+          data[input.name] = serialized;
+        }
+      }
+    }
+
+    return {
+      id: `${rawEvent.transactionHash}-${rawEvent.logIndex}`,
+      type: rawEvent.event,
+      blockNumber: rawEvent.blockNumber,
+      blockHash: rawEvent.blockHash,
+      transactionHash: rawEvent.transactionHash,
+      logIndex: rawEvent.logIndex,
+      timestamp: 0, // Will be filled by block data if available
+      data,
+      indexed
+    };
+  }
+
+  _serializeValue(value) {
+    if (value === null || value === undefined) return null;
+    if (typeof value === 'bigint' || value._isBigNumber) {
+      return value.toString();
+    }
+    if (Array.isArray(value)) {
+      return value.map(v => this._serializeValue(v));
+    }
+    return value;
+  }
+
+  _groupByType(events) {
+    const grouped = {};
+    for (const event of events) {
+      if (!grouped[event.type]) {
+        grouped[event.type] = [];
+      }
+      grouped[event.type].push(event);
+    }
+    return grouped;
+  }
+
+  async _saveSyncState() {
+    await this.storage.setSyncState({
+      lastSyncedBlock: this.currentBlock,
+      lastSyncTime: Date.now(),
+      eventCounts: {}, // Could track per-type counts
+      chainId: this.chainId
+    });
+  }
+
+  async _startRealTimeSync() {
+    // Try WebSocket subscription first
+    try {
+      for (const eventType of this.eventTypes) {
+        const filter = this.contract.filters[eventType.name]();
+        const listener = async (...args) => {
+          const event = args[args.length - 1]; // Last arg is the event object
+          await this._handleNewEvent(event);
+        };
+
+        this.contract.on(filter, listener);
+        this.eventListeners.push({ filter, listener });
+      }
+    } catch (error) {
+      // Fall back to polling
+      console.warn('[SyncEngine] WebSocket subscription failed, falling back to polling');
+      this._startPolling();
+    }
+
+    // Also start polling as backup (handles missed events)
+    this._startPolling();
+  }
+
+  _startPolling() {
+    if (this.pollTimer) return;
+
+    this.pollTimer = setInterval(async () => {
+      if (this.isPaused) return;
+
+      try {
+        const latestBlock = await this.provider.getBlockNumber();
+        const targetBlock = latestBlock - this.confirmations;
+
+        if (targetBlock > this.currentBlock) {
+          await this._syncBatch(this.currentBlock + 1, targetBlock);
+        }
+
+        this.latestBlock = latestBlock;
+
+        // Optional: reorg tracking
+        if (this.reorgTracking) {
+          await this._checkForReorg(latestBlock);
+        }
+      } catch (error) {
+        console.error('[SyncEngine] Polling error:', error);
+      }
+    }, this.pollInterval);
+  }
+
+  async _handleNewEvent(rawEvent) {
+    // Wait for confirmations
+    if (this.confirmations > 0) {
+      await this._waitForConfirmations(rawEvent.transactionHash);
+    }
+
+    const indexedEvent = this._parseEvent(rawEvent);
+    await this.storage.putEvents([indexedEvent]);
+    this.eventsIndexed++;
+
+    // Update current block if this event is ahead
+    if (indexedEvent.blockNumber > this.currentBlock) {
+      this.currentBlock = indexedEvent.blockNumber;
+      await this._saveSyncState();
+    }
+
+    // Notify
+    this.queryEngine.notifyNewEvents(indexedEvent.type, [indexedEvent]);
+    this.eventBus.emit('indexer:newEvents', {
+      eventType: indexedEvent.type,
+      events: [indexedEvent]
+    });
+  }
+
+  async _waitForConfirmations(txHash) {
+    let confirmations = 0;
+    while (confirmations < this.confirmations) {
+      const receipt = await this.provider.getTransactionReceipt(txHash);
+      if (!receipt) {
+        await this._sleep(1000);
+        continue;
+      }
+
+      const currentBlock = await this.provider.getBlockNumber();
+      confirmations = currentBlock - receipt.blockNumber + 1;
+
+      if (confirmations < this.confirmations) {
+        await this._sleep(1000);
+      }
+    }
+  }
+
+  async _checkForReorg(currentBlock) {
+    const block = await this.provider.getBlock(currentBlock);
+    if (!block) return;
+
+    // Check if parent hash matches what we stored
+    const storedParentHash = this.recentBlocks.get(currentBlock - 1);
+    if (storedParentHash && storedParentHash !== block.parentHash) {
+      // Reorg detected!
+      await this._handleReorg(currentBlock - 1);
+    }
+
+    // Track this block
+    this.recentBlocks.set(currentBlock, block.hash);
+
+    // Prune old entries
+    if (this.recentBlocks.size > this.reorgDepth) {
+      const oldest = Math.min(...this.recentBlocks.keys());
+      this.recentBlocks.delete(oldest);
+    }
+  }
+
+  async _handleReorg(forkBlock) {
+    // Find common ancestor
+    let checkBlock = forkBlock;
+    while (checkBlock > this.deployBlock) {
+      const storedHash = this.recentBlocks.get(checkBlock);
+      const chainBlock = await this.provider.getBlock(checkBlock);
+
+      if (storedHash === chainBlock?.hash) break;
+      checkBlock--;
+    }
+
+    // Delete events from orphaned blocks
+    await this.storage.deleteEventsAfterBlock(checkBlock);
+
+    // Update sync state
+    this.currentBlock = checkBlock;
+    await this._saveSyncState();
+
+    // Emit reorg event
+    this.eventBus.emit('indexer:reorg', {
+      forkBlock,
+      commonAncestor: checkBlock,
+      depth: forkBlock - checkBlock
+    });
+
+    // Re-sync from common ancestor
+    await this._syncHistorical();
+  }
+
+  /**
+   * Pause real-time sync.
+   */
+  pause() {
+    this.isPaused = true;
+    this.state = 'paused';
+    this.eventBus.emit('indexer:paused', {});
+  }
+
+  /**
+   * Resume real-time sync.
+   */
+  resume() {
+    this.isPaused = false;
+    this.start();
+    this.eventBus.emit('indexer:resumed', {});
+  }
+
+  /**
+   * Force re-sync from block.
+   * @param {number} fromBlock - Block to start from (default: deployBlock)
+   */
+  async resync(fromBlock) {
+    this.pause();
+
+    if (fromBlock !== undefined) {
+      this.currentBlock = fromBlock - 1;
+    } else {
+      this.currentBlock = this.deployBlock - 1;
+    }
+
+    await this.storage.clear();
+    await this._saveSyncState();
+
+    this.eventsIndexed = 0;
+    this.resume();
+  }
+
+  /**
+   * Get current sync status.
+   * @returns {SyncStatus}
+   */
+  getStatus() {
+    return {
+      state: this.state,
+      currentBlock: this.currentBlock,
+      latestBlock: this.latestBlock,
+      progress: this.latestBlock > 0
+        ? Math.min(1, (this.currentBlock - this.deployBlock) / (this.latestBlock - this.deployBlock))
+        : 0,
+      eventsIndexed: this.eventsIndexed,
+      lastSyncTime: this.lastSyncTime,
+      error: this.error
+    };
+  }
+
+  /**
+   * Stop sync and cleanup.
+   */
+  async stop() {
+    this.isPaused = true;
+
+    // Remove event listeners
+    for (const { filter, listener } of this.eventListeners) {
+      this.contract.off(filter, listener);
+    }
+    this.eventListeners = [];
+
+    // Stop polling
+    if (this.pollTimer) {
+      clearInterval(this.pollTimer);
+      this.pollTimer = null;
+    }
+  }
+
+  _sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+export default SyncEngine;

package/src/indexer/index.js

@@ -0,0 +1,13 @@
+// src/indexer/index.js
+
+import QueryEngine from './QueryEngine.js';
+import SyncEngine from './SyncEngine.js';
+import EntityResolver from './EntityResolver.js';
+import Patterns from './Patterns.js';
+
+export {
+  QueryEngine,
+  SyncEngine,
+  EntityResolver,
+  Patterns
+};

package/src/services/BlockchainService.js

@@ -533,6 +533,36 @@ class BlockchainService {
         return this.connectionState === 'connected';
     }
 
+    // NEW: Get provider instance (for EventIndexer)
+    getProvider() {
+        return this.provider;
+    }
+
+    // NEW: Create contract instance (for EventIndexer)
+    getContract(address, abi) {
+        if (!this.provider) {
+            throw new Error('Provider not initialized');
+        }
+        return new ethers.Contract(address, abi, this.provider);
+    }
+
+    // NEW: Get current chain ID (for EventIndexer)
+    async getChainId() {
+        if (!this.provider) {
+            throw new Error('Provider not initialized');
+        }
+        const network = await this.provider.getNetwork();
+        return Number(network.chainId);
+    }
+
+    // NEW: Get block by number (for EventIndexer)
+    async getBlock(blockNumber) {
+        if (!this.provider) {
+            throw new Error('Provider not initialized');
+        }
+        return this.provider.getBlock(blockNumber);
+    }
+
     async getCurrentTier() {
         try {
             // Check cache first

package/src/services/ContractCache.js

@@ -1,5 +1,5 @@
 class ContractCache {
-    constructor(eventBus) {
+    constructor(eventBus, options = {}) {
         if (!eventBus) {
             throw new Error('ContractCache requires an eventBus instance.');
         }
@@ -45,7 +45,12 @@ class ContractCache {
         
         // Setup event listeners for cache invalidation
         this.setupEventListeners();
-        
+
+        // NEW: Optional event-driven invalidation from EventIndexer
+        if (options.eventInvalidation) {
+            this.setupEventInvalidation(options.eventInvalidation);
+        }
+
         // Debug mode
         this.debug = false;
     }
@@ -87,6 +92,19 @@ class ContractCache {
         });
     }
 
+    /**
+     * Configure which events invalidate which cache patterns
+     */
+    setupEventInvalidation(config) {
+        // config: { 'Transfer': ['balance'], 'Approval': ['allowance'] }
+        this.eventBus.on('indexer:newEvents', ({ eventType }) => {
+            const patterns = config[eventType];
+            if (patterns && Array.isArray(patterns)) {
+                this.invalidateByPattern(...patterns);
+            }
+        });
+    }
+
     /**
      * Generate cache key from method name and arguments
      * @param {string} method - Method name