@onlineapps/conn-orch-registry 1.1.4

package/src/registryEventConsumer.js

@@ -0,0 +1,422 @@
+/**
+ * registryEventConsumer.js
+ *
+ * Extension for ServiceRegistryClient to consume registry change events.
+ * Implements opt-in subscription to registry.changes exchange.
+ * Maintains local service index and enables lazy loading of specs.
+ *
+ * @module @onlineapps/connector-registry-client/src/registryEventConsumer
+ */
+
+const EventEmitter = require('events');
+
+// Default StorageConnector - can be overridden for testing
+let DefaultStorageConnector;
+try {
+  DefaultStorageConnector = require('@onlineapps/connector-storage');
+} catch (e) {
+  // StorageConnector not available, must be injected
+  DefaultStorageConnector = null;
+}
+
+class RegistryEventConsumer extends EventEmitter {
+  /**
+   * @param {Object} opts
+   * @param {Object} opts.queueManager - QueueManager instance from ServiceRegistryClient
+   * @param {string} opts.serviceName - Name of the consuming service
+   * @param {Object} [opts.redis] - Optional Redis client for index persistence
+   * @param {Object} [opts.storageConfig] - Configuration for StorageConnector
+   * @param {Object} [opts.StorageConnector] - Injectable StorageConnector class for testing
+   * @param {boolean} [opts.cacheEnabled=true] - Enable local spec caching
+   * @param {number} [opts.maxCacheSize=50] - Maximum cached specs
+   */
+  constructor({ queueManager, serviceName, redis = null, storageConfig = {}, StorageConnector = null, cacheEnabled = true, maxCacheSize = 50 }) {
+    super();
+
+    this.queueManager = queueManager;
+    this.serviceName = serviceName;
+    this.redis = redis;
+
+    // Use injected StorageConnector or default one
+    this.StorageConnectorClass = StorageConnector || DefaultStorageConnector;
+
+    // Initialize storage connector for MinIO access (only if available)
+    this.storage = this.StorageConnectorClass ? new this.StorageConnectorClass({
+      ...storageConfig,
+      cacheEnabled,
+      maxCacheSize
+    }) : null;
+
+    // Local service index: serviceName -> { fingerprint, version, status, path }
+    this.serviceIndex = new Map();
+
+    // Local spec cache: fingerprint -> spec content
+    this.specCache = new Map();
+
+    // Registry events queue name
+    this.eventsQueueName = `${serviceName}.registry`;
+
+    // Exchange name for registry events
+    this.exchangeName = 'registry.changes';
+  }
+
+  /**
+   * Initialize storage connector
+   */
+  async initStorage() {
+    if (this.storage) {
+      await this.storage.initialize();
+      this.emit('storageReady');
+    } else {
+      console.warn('Storage connector not available - spec downloading disabled');
+    }
+  }
+
+  /**
+   * Subscribe to registry change events (opt-in)
+   * Creates service-specific queue bound to registry.changes exchange
+   */
+  async subscribeToChanges() {
+    try {
+      const channel = this.queueManager.channel;
+
+      // Assert exchange exists (fanout type for broadcasting)
+      await channel.assertExchange(this.exchangeName, 'fanout', {
+        durable: true
+      });
+
+      // Create service-specific queue for registry events
+      // Try to assert queue with arguments, fall back to simple assert if conflict
+      try {
+        await channel.assertQueue(this.eventsQueueName, {
+          durable: true,
+          arguments: {
+            'x-message-ttl': 60000, // Messages expire after 1 minute
+            'x-max-length': 1000     // Keep max 1000 messages
+          }
+        });
+      } catch (error) {
+        if (error.code === 406) {
+          // Queue exists with different arguments, use it as-is
+          await channel.assertQueue(this.eventsQueueName, {
+            durable: true
+          });
+        } else {
+          throw error;
+        }
+      }
+
+      // Bind queue to exchange (receive all registry events)
+      await channel.bindQueue(this.eventsQueueName, this.exchangeName, '');
+
+      // Load initial snapshot from Redis if available
+      await this.loadIndexFromCache();
+
+      // Start consuming events
+      await channel.consume(
+        this.eventsQueueName,
+        msg => this._handleRegistryEvent(msg),
+        { noAck: false }
+      );
+
+      this.emit('subscribed', { queue: this.eventsQueueName, exchange: this.exchangeName });
+      console.log(`[${this.serviceName}] Subscribed to registry changes on ${this.eventsQueueName}`);
+
+      // Request initial snapshot
+      await this.requestSnapshot();
+
+    } catch (error) {
+      this.emit('error', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Handle incoming registry event
+   * @private
+   */
+  async _handleRegistryEvent(msg) {
+    try {
+      const event = JSON.parse(msg.content.toString());
+
+      switch (event.type) {
+        case 'SERVICE_SPEC_PUBLISHED':
+          await this.handleSpecPublished(event);
+          break;
+
+        case 'SERVICE_STATUS_CHANGED':
+          await this.handleStatusChanged(event);
+          break;
+
+        case 'REGISTRY_SNAPSHOT':
+          await this.handleSnapshot(event);
+          break;
+
+        case 'SERVICE_REMOVED':
+          await this.handleServiceRemoved(event);
+          break;
+
+        default:
+          console.log(`[${this.serviceName}] Unknown event type: ${event.type}`);
+      }
+
+      // Persist index after update
+      await this.persistIndex();
+
+      // Acknowledge message
+      this.queueManager.channel.ack(msg);
+
+      // Emit event for service to react
+      this.emit('registryEvent', event);
+
+    } catch (error) {
+      console.error(`[${this.serviceName}] Failed to handle registry event:`, error);
+      this.emit('error', error);
+
+      // Reject message without requeue
+      this.queueManager.channel.nack(msg, false, false);
+    }
+  }
+
+  /**
+   * Handle SERVICE_SPEC_PUBLISHED event
+   */
+  async handleSpecPublished(event) {
+    const { service, fingerprint, version, status, bucket, path } = event;
+
+    // Update local index (not content!)
+    this.serviceIndex.set(service, {
+      fingerprint,
+      version,
+      status: status || 'ACTIVE',
+      bucket: bucket || 'registry',
+      path,
+      updatedAt: event.timestamp
+    });
+
+    this.emit('serviceUpdated', { service, fingerprint, version });
+    console.log(`[${this.serviceName}] Service ${service} updated with fingerprint ${fingerprint}`);
+  }
+
+  /**
+   * Handle SERVICE_STATUS_CHANGED event
+   */
+  async handleStatusChanged(event) {
+    const { service, status } = event;
+
+    const entry = this.serviceIndex.get(service);
+    if (entry) {
+      entry.status = status;
+      entry.updatedAt = event.timestamp;
+
+      this.emit('statusChanged', { service, status });
+      console.log(`[${this.serviceName}] Service ${service} status changed to ${status}`);
+    }
+  }
+
+  /**
+   * Handle REGISTRY_SNAPSHOT event
+   */
+  async handleSnapshot(event) {
+    const { services } = event;
+
+    // Clear and rebuild index
+    this.serviceIndex.clear();
+
+    for (const service of services) {
+      this.serviceIndex.set(service.name, {
+        fingerprint: service.fingerprint,
+        version: service.version,
+        status: service.status,
+        bucket: service.bucket || 'registry',
+        path: service.path,
+        updatedAt: event.timestamp
+      });
+    }
+
+    this.emit('snapshotReceived', { count: services.length });
+    console.log(`[${this.serviceName}] Received snapshot with ${services.length} services`);
+  }
+
+  /**
+   * Handle SERVICE_REMOVED event
+   */
+  async handleServiceRemoved(event) {
+    const { service } = event;
+
+    if (this.serviceIndex.has(service)) {
+      this.serviceIndex.delete(service);
+
+      // Clear cached specs for this service
+      const fingerprintsToRemove = [];
+      for (const [fingerprint, spec] of this.specCache.entries()) {
+        if (spec.serviceName === service) {
+          fingerprintsToRemove.push(fingerprint);
+        }
+      }
+      fingerprintsToRemove.forEach(fp => this.specCache.delete(fp));
+
+      this.emit('serviceRemoved', { service });
+      console.log(`[${this.serviceName}] Service ${service} removed from index`);
+    }
+  }
+
+  /**
+   * Request initial snapshot from registry
+   */
+  async requestSnapshot() {
+    // Publish request for snapshot
+    const request = {
+      type: 'SNAPSHOT_REQUEST',
+      requester: this.serviceName,
+      timestamp: new Date().toISOString()
+    };
+
+    await this.queueManager.channel.publish(
+      this.exchangeName,
+      '',
+      Buffer.from(JSON.stringify(request))
+    );
+  }
+
+  /**
+   * Get service spec with lazy loading from MinIO
+   * @param {string} serviceName - Name of the service
+   * @returns {Promise<Object>} - Service specification
+   */
+  async getServiceSpec(serviceName) {
+    // Get entry from index
+    const entry = this.serviceIndex.get(serviceName);
+    if (!entry) {
+      throw new Error(`Service ${serviceName} not found in index`);
+    }
+
+    if (entry.status !== 'ACTIVE') {
+      throw new Error(`Service ${serviceName} is not active (status: ${entry.status})`);
+    }
+
+    const { fingerprint, bucket, path } = entry;
+
+    // Check local cache first
+    if (this.specCache.has(fingerprint)) {
+      this.emit('cacheHit', { service: serviceName, fingerprint });
+      return this.specCache.get(fingerprint);
+    }
+
+    // Check if storage is available
+    if (!this.storage) {
+      throw new Error('Storage connector not available for downloading specs');
+    }
+
+    // Download from MinIO with verification
+    console.log(`[${this.serviceName}] Downloading spec for ${serviceName} from ${bucket}/${path}`);
+
+    const content = await this.storage.downloadWithVerification(
+      bucket,
+      path,
+      fingerprint
+    );
+
+    const spec = JSON.parse(content);
+
+    // Cache forever (immutable content)
+    this.specCache.set(fingerprint, spec);
+
+    this.emit('specLoaded', { service: serviceName, fingerprint });
+
+    return spec;
+  }
+
+  /**
+   * Check if service is active (synchronous, from local index)
+   * @param {string} serviceName - Name of the service
+   * @returns {boolean}
+   */
+  isServiceActive(serviceName) {
+    const entry = this.serviceIndex.get(serviceName);
+    return entry && entry.status === 'ACTIVE';
+  }
+
+  /**
+   * Get list of active services (synchronous, from local index)
+   * @returns {Array<string>}
+   */
+  getActiveServices() {
+    return Array.from(this.serviceIndex.entries())
+      .filter(([, entry]) => entry.status === 'ACTIVE')
+      .map(([name]) => name);
+  }
+
+  /**
+   * Get service info from local index
+   * @param {string} serviceName
+   * @returns {Object|null}
+   */
+  getServiceInfo(serviceName) {
+    return this.serviceIndex.get(serviceName) || null;
+  }
+
+  /**
+   * Persist index to Redis (if configured)
+   */
+  async persistIndex() {
+    if (!this.redis) return;
+
+    try {
+      const indexData = JSON.stringify(Array.from(this.serviceIndex.entries()));
+      await this.redis.set(
+        `registry:index:${this.serviceName}`,
+        indexData,
+        'EX',
+        3600 // Expire after 1 hour
+      );
+    } catch (error) {
+      console.error(`[${this.serviceName}] Failed to persist index:`, error);
+    }
+  }
+
+  /**
+   * Load index from Redis cache
+   */
+  async loadIndexFromCache() {
+    if (!this.redis) return;
+
+    try {
+      const indexData = await this.redis.get(`registry:index:${this.serviceName}`);
+      if (indexData) {
+        const entries = JSON.parse(indexData);
+        this.serviceIndex = new Map(entries);
+
+        this.emit('indexLoaded', { count: this.serviceIndex.size });
+        console.log(`[${this.serviceName}] Loaded ${this.serviceIndex.size} services from cache`);
+      }
+    } catch (error) {
+      console.error(`[${this.serviceName}] Failed to load index from cache:`, error);
+    }
+  }
+
+  /**
+   * Clear all caches
+   */
+  clearCache() {
+    this.specCache.clear();
+    if (this.storage && this.storage.clearCache) {
+      this.storage.clearCache();
+    }
+    this.emit('cacheCleared');
+  }
+
+  /**
+   * Get stats about current state
+   */
+  getStats() {
+    return {
+      indexSize: this.serviceIndex.size,
+      activeServices: this.getActiveServices().length,
+      cachedSpecs: this.specCache.size,
+      queueName: this.eventsQueueName
+    };
+  }
+}
+
+module.exports = RegistryEventConsumer;