@gokiteam/goki-dev 0.2.0

package/src/singletons/FirestoreBroadcaster.js

@@ -0,0 +1,367 @@
+// FirestoreBroadcaster singleton for managing SSE connections to Firestore change streams
+// Handles client registration, Firebase snapshot listeners with reference counting, and change broadcasting
+
+import { Logger } from './Logger.js'
+import { FunctionTriggerDispatcher } from './FunctionTriggerDispatcher.js'
+import { initializeApp, getApps } from 'firebase-admin/app'
+import { getFirestore } from 'firebase-admin/firestore'
+import { Application } from '../configs/Application.js'
+
+const DEFAULT_PROJECT_ID = Application.firestore.projectId
+
+// Set emulator host environment variable for Firebase Admin SDK
+process.env.FIRESTORE_EMULATOR_HOST = Application.firestore.emulatorHost
+
+// Cache Firebase app instances by projectId
+const appCache = new Map()
+
+function getFirestoreDb (projectId) {
+  const pid = projectId || DEFAULT_PROJECT_ID
+  if (appCache.has(pid)) {
+    return appCache.get(pid)
+  }
+  const appName = `firestore-${pid}`
+  const existingApp = getApps().find(a => a.name === appName)
+  const app = existingApp || initializeApp({ projectId: pid }, appName)
+  const db = getFirestore(app)
+  appCache.set(pid, db)
+  return db
+}
+
+class FirestoreBroadcasterClass {
+  constructor () {
+    this.clients = new Map() // connectionId → { ctx, filters: { projectId, collectionPath } }
+    this.listeners = new Map() // listenerKey (projectId:collectionPath) → { unsubscribe, refCount }
+    this.heartbeatId = null
+    this.heartbeatIntervalMs = 30000
+  }
+
+  /**
+   * Register a new SSE client and start collection listener if needed
+   * @param {string} connectionId - Unique identifier for the connection (traceId)
+   * @param {Object} ctx - Koa context with res.write for SSE
+   * @param {Object} filters - { projectId, collectionPath }
+   */
+  addClient (connectionId, ctx, filters = {}) {
+    this.clients.set(connectionId, { ctx, filters })
+    this.startHeartbeat()
+
+    // Start listening to the collection if filters provided
+    if (filters.projectId && filters.collectionPath) {
+      this.startCollectionListener(filters.projectId, filters.collectionPath)
+    }
+
+    Logger.log({
+      level: 'info',
+      message: 'Firestore SSE client connected',
+      data: { connectionId, filters, totalClients: this.clients.size }
+    })
+  }
+
+  /**
+   * Unregister an SSE client and stop listener if no more clients watching
+   * @param {string} connectionId - Unique identifier for the connection
+   */
+  removeClient (connectionId) {
+    const client = this.clients.get(connectionId)
+    if (!client) return
+
+    const removed = this.clients.delete(connectionId)
+
+    if (removed) {
+      // Decrement listener ref count
+      if (client.filters.projectId && client.filters.collectionPath) {
+        this.decrementListenerRefCount(client.filters.projectId, client.filters.collectionPath)
+      }
+
+      Logger.log({
+        level: 'info',
+        message: 'Firestore SSE client disconnected',
+        data: { connectionId, totalClients: this.clients.size }
+      })
+
+      if (this.clients.size === 0) {
+        this.stopHeartbeat()
+      }
+    }
+  }
+
+  /**
+   * Start a Firebase snapshot listener for a collection
+   * Uses reference counting to share one listener across multiple clients
+   * @param {string} projectId - Firestore project ID
+   * @param {string} collectionPath - Collection path
+   */
+  startCollectionListener (projectId, collectionPath) {
+    const listenerKey = `${projectId}:${collectionPath}`
+
+    // Increment ref count if listener already exists
+    if (this.listeners.has(listenerKey)) {
+      const listener = this.listeners.get(listenerKey)
+      listener.refCount++
+      Logger.log({
+        level: 'debug',
+        message: 'Firestore listener ref count incremented',
+        data: { projectId, collectionPath, refCount: listener.refCount }
+      })
+      return
+    }
+
+    // Create new snapshot listener
+    try {
+      const db = getFirestoreDb(projectId)
+      Logger.log({
+        level: 'info',
+        message: 'Creating Firestore snapshot listener',
+        data: { projectId, collectionPath, listenerKey }
+      })
+      const unsubscribe = db.collection(collectionPath).onSnapshot(
+        (snapshot) => {
+          const changeCount = snapshot.docChanges().length
+          Logger.log({
+            level: 'info',
+            message: 'Firestore snapshot received',
+            data: { projectId, collectionPath, changeCount }
+          })
+
+          snapshot.docChanges().forEach((change) => {
+            const changeEvent = {
+              eventId: `evt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+              timestamp: new Date().toISOString(),
+              projectId,
+              collectionPath,
+              documentId: change.doc.id,
+              changeType: change.type, // 'added', 'modified', 'removed'
+              document: change.type !== 'removed' ? {
+                name: change.doc.ref.path,
+                fields: change.doc.data(),
+                createTime: change.doc.createTime?.toDate().toISOString(),
+                updateTime: change.doc.updateTime?.toDate().toISOString()
+              } : null
+            }
+
+            Logger.log({
+              level: 'info',
+              message: 'Broadcasting Firestore change event',
+              data: { eventId: changeEvent.eventId, changeType: changeEvent.changeType, documentId: changeEvent.documentId, clientCount: this.clients.size }
+            })
+
+            this.broadcast(changeEvent)
+          })
+        },
+        (error) => {
+          Logger.log({
+            level: 'error',
+            message: 'Firestore snapshot listener error',
+            data: { projectId, collectionPath, error: error.message }
+          })
+          // Remove listener on error
+          this.stopCollectionListener(projectId, collectionPath)
+        }
+      )
+
+      this.listeners.set(listenerKey, { unsubscribe, refCount: 1 })
+
+      Logger.log({
+        level: 'info',
+        message: 'Firestore snapshot listener started',
+        data: { projectId, collectionPath }
+      })
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to start Firestore snapshot listener',
+        data: { projectId, collectionPath, error: error.message }
+      })
+    }
+  }
+
+  /**
+   * Decrement listener reference count and stop if reaches zero
+   * @param {string} projectId - Firestore project ID
+   * @param {string} collectionPath - Collection path
+   */
+  decrementListenerRefCount (projectId, collectionPath) {
+    const listenerKey = `${projectId}:${collectionPath}`
+    const listener = this.listeners.get(listenerKey)
+
+    if (!listener) return
+
+    listener.refCount--
+
+    Logger.log({
+      level: 'debug',
+      message: 'Firestore listener ref count decremented',
+      data: { projectId, collectionPath, refCount: listener.refCount }
+    })
+
+    if (listener.refCount <= 0) {
+      this.stopCollectionListener(projectId, collectionPath)
+    }
+  }
+
+  /**
+   * Stop a Firebase snapshot listener
+   * @param {string} projectId - Firestore project ID
+   * @param {string} collectionPath - Collection path
+   */
+  stopCollectionListener (projectId, collectionPath) {
+    const listenerKey = `${projectId}:${collectionPath}`
+    const listener = this.listeners.get(listenerKey)
+
+    if (listener && listener.unsubscribe) {
+      listener.unsubscribe()
+      this.listeners.delete(listenerKey)
+
+      Logger.log({
+        level: 'info',
+        message: 'Firestore snapshot listener stopped',
+        data: { projectId, collectionPath }
+      })
+    }
+  }
+
+  /**
+   * Broadcast a change event to all connected clients matching filters
+   * @param {Object} change - { eventId, timestamp, projectId, collectionPath, documentId, changeType, document }
+   */
+  broadcast (change) {
+    let sentCount = 0
+    for (const [connectionId, { ctx, filters }] of this.clients) {
+      if (this.matchesFilters(change, filters)) {
+        try {
+          this.sendSSE(ctx, 'change', change)
+          sentCount++
+        } catch (error) {
+          Logger.log({
+            level: 'error',
+            message: 'Failed to send SSE change event to client',
+            data: { connectionId, error: error.message }
+          })
+          this.removeClient(connectionId)
+        }
+      }
+    }
+    Logger.log({
+      level: 'debug',
+      message: 'Firestore change broadcasted to SSE clients',
+      data: {
+        eventId: change.eventId,
+        changeType: change.changeType,
+        documentId: change.documentId,
+        totalClients: this.clients.size,
+        sentToClients: sentCount
+      }
+    })
+    // Dispatch to Cloud Functions (regardless of SSE client count)
+    try {
+      if (FunctionTriggerDispatcher && typeof FunctionTriggerDispatcher.onFirestoreChange === 'function') {
+        FunctionTriggerDispatcher.onFirestoreChange(change).catch(err => {
+          Logger.log({ level: 'error', message: 'Cloud Function Firestore dispatch error', data: { error: err.message } })
+        })
+      }
+    } catch (err) {
+      Logger.log({ level: 'error', message: 'Error calling FunctionTriggerDispatcher', data: { error: err.message } })
+    }
+  }
+
+  /**
+   * Check if change matches client filters
+   * @param {Object} change - The change event to check
+   * @param {Object} filters - { projectId?, collectionPath? }
+   * @returns {boolean}
+   */
+  matchesFilters (change, filters) {
+    // No filters = receive all changes
+    if (!filters || Object.keys(filters).length === 0) {
+      return true
+    }
+
+    // Project filter
+    if (filters.projectId && change.projectId !== filters.projectId) {
+      return false
+    }
+
+    // Collection filter
+    if (filters.collectionPath && change.collectionPath !== filters.collectionPath) {
+      return false
+    }
+
+    return true
+  }
+
+  /**
+   * Send SSE formatted message to client
+   * @param {Object} ctx - Koa context
+   * @param {string} event - Event name
+   * @param {Object} data - Data to send
+   */
+  sendSSE (ctx, event, data) {
+    const message = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`
+    ctx.res.write(message)
+  }
+
+  /**
+   * Start the shared heartbeat interval (one for all clients)
+   */
+  startHeartbeat () {
+    if (this.heartbeatId) return
+    this.heartbeatId = setInterval(() => this.sendHeartbeat(), this.heartbeatIntervalMs)
+  }
+
+  /**
+   * Stop the shared heartbeat interval when no clients remain
+   */
+  stopHeartbeat () {
+    if (this.heartbeatId) {
+      clearInterval(this.heartbeatId)
+      this.heartbeatId = null
+    }
+  }
+
+  /**
+   * Send heartbeat to all connected clients
+   * Used to keep connections alive
+   */
+  sendHeartbeat () {
+    for (const [connectionId, { ctx }] of this.clients) {
+      try {
+        ctx.res.write(': heartbeat\n\n')
+      } catch (error) {
+        Logger.log({
+          level: 'error',
+          message: 'Failed to send heartbeat to Firestore SSE client',
+          data: { connectionId, error: error.message }
+        })
+        this.removeClient(connectionId)
+      }
+    }
+  }
+
+  /**
+   * Get current client and listener statistics
+   * @returns {Object}
+   */
+  getStats () {
+    const stats = {
+      totalClients: this.clients.size,
+      totalListeners: this.listeners.size,
+      clientsByCollection: {},
+      listenerRefCounts: {}
+    }
+
+    for (const [, { filters }] of this.clients) {
+      const key = `${filters.projectId}:${filters.collectionPath}`
+      stats.clientsByCollection[key] = (stats.clientsByCollection[key] || 0) + 1
+    }
+
+    for (const [listenerKey, { refCount }] of this.listeners) {
+      stats.listenerRefCounts[listenerKey] = refCount
+    }
+
+    return stats
+  }
+}
+
+// Export singleton instance
+export const FirestoreBroadcaster = new FirestoreBroadcasterClass()