@gokiteam/goki-dev 0.2.0

package/src/singletons/PostgresBroadcaster.js

@@ -0,0 +1,367 @@
+// PostgresBroadcaster singleton for managing SSE connections to PostgreSQL change detection via polling
+// Handles client registration, pg_stat_user_tables polling with reference counting, and change broadcasting
+
+import { Logger } from './Logger.js'
+import { PostgresClient } from './PostgresClient.js'
+
+class PostgresBroadcasterClass {
+  constructor () {
+    this.clients = new Map() // connectionId -> { ctx, filters: { database, schema, table } }
+    this.watchers = new Map() // watcherKey (database:schema:table) -> { intervalId, lastStats: { inserts, updates, deletes, liveRows }, refCount }
+    this.heartbeatId = null
+    this.heartbeatIntervalMs = 30000
+    this.pollIntervalMs = 2000
+  }
+
+  /**
+   * Register a new SSE client and start watcher if needed
+   * @param {string} connectionId - Unique identifier for the connection (traceId)
+   * @param {Object} ctx - Koa context with res.write for SSE
+   * @param {Object} filters - { database, schema, table }
+   */
+  addClient (connectionId, ctx, filters = {}) {
+    this.clients.set(connectionId, { ctx, filters })
+    this.startHeartbeat()
+    if (filters.database && filters.schema && filters.table) {
+      this.startWatcher(filters.database, filters.schema, filters.table)
+    }
+    Logger.log({
+      level: 'info',
+      message: 'PostgreSQL SSE client connected',
+      data: { connectionId, filters, totalClients: this.clients.size }
+    })
+  }
+
+  /**
+   * Unregister an SSE client and stop watcher if no more clients watching
+   * @param {string} connectionId - Unique identifier for the connection
+   */
+  removeClient (connectionId) {
+    const client = this.clients.get(connectionId)
+    if (!client) return
+    const removed = this.clients.delete(connectionId)
+    if (removed) {
+      if (client.filters.database && client.filters.schema && client.filters.table) {
+        this.decrementWatcherRefCount(client.filters.database, client.filters.schema, client.filters.table)
+      }
+      Logger.log({
+        level: 'info',
+        message: 'PostgreSQL SSE client disconnected',
+        data: { connectionId, totalClients: this.clients.size }
+      })
+      if (this.clients.size === 0) {
+        this.stopHeartbeat()
+      }
+    }
+  }
+
+  /**
+   * Start a polling watcher for a specific table
+   * Uses reference counting to share one watcher across multiple clients
+   * @param {string} database - PostgreSQL database name
+   * @param {string} schema - Schema name
+   * @param {string} table - Table name
+   */
+  async startWatcher (database, schema, table) {
+    const watcherKey = `${database}:${schema}:${table}`
+    if (this.watchers.has(watcherKey)) {
+      const watcher = this.watchers.get(watcherKey)
+      watcher.refCount++
+      Logger.log({
+        level: 'debug',
+        message: 'PostgreSQL watcher ref count incremented',
+        data: { database, schema, table, refCount: watcher.refCount }
+      })
+      return
+    }
+    try {
+      const result = await PostgresClient.query(
+        'SELECT n_tup_ins, n_tup_upd, n_tup_del, n_live_tup FROM pg_stat_user_tables WHERE schemaname = $1 AND relname = $2',
+        [schema, table],
+        database
+      )
+      const lastStats = result.rows.length > 0
+        ? {
+            inserts: parseInt(result.rows[0].n_tup_ins) || 0,
+            updates: parseInt(result.rows[0].n_tup_upd) || 0,
+            deletes: parseInt(result.rows[0].n_tup_del) || 0,
+            liveRows: parseInt(result.rows[0].n_live_tup) || 0
+          }
+        : { inserts: 0, updates: 0, deletes: 0, liveRows: 0 }
+      const intervalId = setInterval(() => this.pollForChanges(watcherKey, database, schema, table), this.pollIntervalMs)
+      this.watchers.set(watcherKey, { intervalId, lastStats, refCount: 1 })
+      Logger.log({
+        level: 'info',
+        message: 'PostgreSQL watcher started',
+        data: { database, schema, table, lastStats }
+      })
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to start PostgreSQL watcher',
+        data: { database, schema, table, error: error.message }
+      })
+    }
+  }
+
+  /**
+   * Decrement watcher reference count and stop if reaches zero
+   * @param {string} database - PostgreSQL database name
+   * @param {string} schema - Schema name
+   * @param {string} table - Table name
+   */
+  decrementWatcherRefCount (database, schema, table) {
+    const watcherKey = `${database}:${schema}:${table}`
+    const watcher = this.watchers.get(watcherKey)
+    if (!watcher) return
+    watcher.refCount--
+    Logger.log({
+      level: 'debug',
+      message: 'PostgreSQL watcher ref count decremented',
+      data: { database, schema, table, refCount: watcher.refCount }
+    })
+    if (watcher.refCount <= 0) {
+      this.stopWatcher(watcherKey)
+    }
+  }
+
+  /**
+   * Stop a polling watcher
+   * @param {string} watcherKey - Key in format "database:schema:table"
+   */
+  stopWatcher (watcherKey) {
+    const watcher = this.watchers.get(watcherKey)
+    if (watcher) {
+      clearInterval(watcher.intervalId)
+      this.watchers.delete(watcherKey)
+      Logger.log({
+        level: 'info',
+        message: 'PostgreSQL watcher stopped',
+        data: { watcherKey }
+      })
+    }
+  }
+
+  /**
+   * Poll pg_stat_user_tables for changes and broadcast if detected
+   * @param {string} watcherKey - Key in format "database:schema:table"
+   * @param {string} database - PostgreSQL database name
+   * @param {string} schema - Schema name
+   * @param {string} table - Table name
+   */
+  async pollForChanges (watcherKey, database, schema, table) {
+    const watcher = this.watchers.get(watcherKey)
+    if (!watcher) return
+    try {
+      const result = await PostgresClient.query(
+        'SELECT n_tup_ins, n_tup_upd, n_tup_del, n_live_tup FROM pg_stat_user_tables WHERE schemaname = $1 AND relname = $2',
+        [schema, table],
+        database
+      )
+      if (result.rows.length === 0) return
+      const currentStats = {
+        inserts: parseInt(result.rows[0].n_tup_ins) || 0,
+        updates: parseInt(result.rows[0].n_tup_upd) || 0,
+        deletes: parseInt(result.rows[0].n_tup_del) || 0,
+        liveRows: parseInt(result.rows[0].n_live_tup) || 0
+      }
+      const hasChanged =
+        currentStats.inserts !== watcher.lastStats.inserts ||
+        currentStats.updates !== watcher.lastStats.updates ||
+        currentStats.deletes !== watcher.lastStats.deletes ||
+        currentStats.liveRows !== watcher.lastStats.liveRows
+      if (hasChanged) {
+        const change = {
+          eventId: `evt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+          timestamp: new Date().toISOString(),
+          database,
+          schema,
+          table,
+          changeType: 'data_changed',
+          stats: currentStats
+        }
+        Logger.log({
+          level: 'info',
+          message: 'PostgreSQL change detected',
+          data: { watcherKey, previousStats: watcher.lastStats, currentStats }
+        })
+        watcher.lastStats = currentStats
+        this.broadcast(change)
+      }
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'PostgreSQL poll error',
+        data: { watcherKey, error: error.message }
+      })
+    }
+  }
+
+  /**
+   * Broadcast a change event to all connected clients matching filters
+   * @param {Object} change - { eventId, timestamp, database, schema, table, changeType, stats }
+   */
+  broadcast (change) {
+    if (this.clients.size === 0) return
+    let sentCount = 0
+    for (const [connectionId, { ctx, filters }] of this.clients) {
+      if (this.matchesFilters(change, filters)) {
+        try {
+          this.sendSSE(ctx, 'change', change)
+          sentCount++
+        } catch (error) {
+          Logger.log({
+            level: 'error',
+            message: 'Failed to send SSE change event to client',
+            data: { connectionId, error: error.message }
+          })
+          this.removeClient(connectionId)
+        }
+      }
+    }
+    Logger.log({
+      level: 'debug',
+      message: 'PostgreSQL change broadcasted to SSE clients',
+      data: {
+        eventId: change.eventId,
+        changeType: change.changeType,
+        table: change.table,
+        totalClients: this.clients.size,
+        sentToClients: sentCount
+      }
+    })
+  }
+
+  /**
+   * Check if change matches client filters
+   * @param {Object} change - The change event to check
+   * @param {Object} filters - { database?, schema?, table? }
+   * @returns {boolean}
+   */
+  matchesFilters (change, filters) {
+    if (!filters || Object.keys(filters).length === 0) {
+      return true
+    }
+    if (filters.database && change.database !== filters.database) {
+      return false
+    }
+    if (filters.schema && change.schema !== filters.schema) {
+      return false
+    }
+    if (filters.table && change.table !== filters.table) {
+      return false
+    }
+    return true
+  }
+
+  /**
+   * Send SSE formatted message to client
+   * @param {Object} ctx - Koa context
+   * @param {string} event - Event name
+   * @param {Object} data - Data to send
+   */
+  sendSSE (ctx, event, data) {
+    ctx.res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`)
+  }
+
+  /**
+   * Start the shared heartbeat interval (one for all clients)
+   */
+  startHeartbeat () {
+    if (this.heartbeatId) return
+    this.heartbeatId = setInterval(() => this.sendHeartbeat(), this.heartbeatIntervalMs)
+  }
+
+  /**
+   * Stop the shared heartbeat interval when no clients remain
+   */
+  stopHeartbeat () {
+    if (this.heartbeatId) {
+      clearInterval(this.heartbeatId)
+      this.heartbeatId = null
+    }
+  }
+
+  /**
+   * Send heartbeat to all connected clients
+   * Used to keep connections alive
+   */
+  sendHeartbeat () {
+    for (const [connectionId, { ctx }] of this.clients) {
+      try {
+        ctx.res.write(': heartbeat\n\n')
+      } catch (error) {
+        Logger.log({
+          level: 'error',
+          message: 'Failed to send heartbeat to PostgreSQL SSE client',
+          data: { connectionId, error: error.message }
+        })
+        this.removeClient(connectionId)
+      }
+    }
+  }
+
+  /**
+   * Notify all clients watching a specific database that data has changed
+   * Called after write queries (INSERT/UPDATE/DELETE) to trigger immediate refresh
+   * @param {string} database - PostgreSQL database name
+   * @param {string} schema - Schema name (optional, broadcasts to all schemas if omitted)
+   * @param {string} table - Table name (optional, broadcasts to all tables if omitted)
+   */
+  notifyChange (database, schema, table) {
+    if (this.clients.size === 0) return
+    const change = {
+      eventId: `evt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+      timestamp: new Date().toISOString(),
+      database,
+      schema: schema || '*',
+      table: table || '*',
+      changeType: 'data_changed',
+      stats: null
+    }
+    // Broadcast to all clients watching this database
+    let sentCount = 0
+    for (const [connectionId, { ctx, filters }] of this.clients) {
+      if (filters.database && filters.database !== database) continue
+      if (schema && filters.schema && filters.schema !== schema) continue
+      if (table && filters.table && filters.table !== table) continue
+      try {
+        this.sendSSE(ctx, 'change', change)
+        sentCount++
+      } catch (error) {
+        this.removeClient(connectionId)
+      }
+    }
+    if (sentCount > 0) {
+      Logger.log({
+        level: 'info',
+        message: 'PostgreSQL change notification sent',
+        data: { database, schema, table, sentToClients: sentCount }
+      })
+    }
+  }
+
+  /**
+   * Get current client and watcher statistics
+   * @returns {Object}
+   */
+  getStats () {
+    const stats = {
+      totalClients: this.clients.size,
+      totalWatchers: this.watchers.size,
+      clientsByTable: {},
+      watcherRefCounts: {}
+    }
+    for (const [, { filters }] of this.clients) {
+      const key = `${filters.database}:${filters.schema}:${filters.table}`
+      stats.clientsByTable[key] = (stats.clientsByTable[key] || 0) + 1
+    }
+    for (const [watcherKey, { refCount }] of this.watchers) {
+      stats.watcherRefCounts[watcherKey] = refCount
+    }
+    return stats
+  }
+}
+
+// Export singleton instance
+export const PostgresBroadcaster = new PostgresBroadcasterClass()

package/src/singletons/PostgresClient.js

@@ -0,0 +1,180 @@
+import pg from 'pg'
+import { Application } from '../configs/Application.js'
+
+const { Pool } = pg
+
+class PostgresClientClass {
+  constructor () {
+    this.pools = new Map() // Map of database -> pool
+    this.defaultDatabase = null
+  }
+
+  initialize () {
+    // Pools are created lazily on first use
+    // This method exists for API compatibility with Server.js
+    const config = Application.postgres
+    this.defaultDatabase = config.database
+  }
+
+  getPool (database = null) {
+    const config = Application.postgres
+    const dbName = database || config.database
+    if (!this.pools.has(dbName)) {
+      const pool = new Pool({
+        host: config.host,
+        port: config.port,
+        user: config.user,
+        password: config.password,
+        database: dbName,
+        max: 5,
+        idleTimeoutMillis: 30000,
+        connectionTimeoutMillis: 5000
+      })
+      pool.on('error', (err) => {
+        console.error(`PostgreSQL pool error (${dbName}):`, err.message)
+      })
+      this.pools.set(dbName, pool)
+    }
+    return this.pools.get(dbName)
+  }
+
+  async query (text, params = [], database = null) {
+    const pool = this.getPool(database)
+    const client = await pool.connect()
+    try {
+      const result = await client.query(text, params)
+      return result
+    } finally {
+      client.release()
+    }
+  }
+
+  async testConnection (database = null) {
+    try {
+      const result = await this.query('SELECT NOW() as time, current_database() as database', [], database)
+      return { connected: true, time: result.rows[0].time, database: result.rows[0].database }
+    } catch (error) {
+      return { connected: false, error: error.message }
+    }
+  }
+
+  async getSchemas (database = null) {
+    const result = await this.query(`
+      SELECT schema_name
+      FROM information_schema.schemata
+      WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
+      ORDER BY schema_name
+    `, [], database)
+    return result.rows.map(row => row.schema_name)
+  }
+
+  async getTables (schema = 'public', database = null) {
+    const result = await this.query(`
+      SELECT
+        t.table_name,
+        (SELECT COUNT(*) FROM information_schema.columns c
+         WHERE c.table_schema = t.table_schema AND c.table_name = t.table_name) as column_count
+      FROM information_schema.tables t
+      WHERE t.table_schema = $1
+        AND t.table_type = 'BASE TABLE'
+      ORDER BY t.table_name
+    `, [schema], database)
+    const tables = result.rows.map(row => ({
+      name: row.table_name,
+      columnCount: parseInt(row.column_count) || 0,
+      rowCount: 0
+    }))
+    // Fetch exact row counts for each table (dev environment, tables are small)
+    const countPromises = tables.map(async (table) => {
+      try {
+        const countResult = await this.query(
+          `SELECT COUNT(*) as count FROM "${schema}"."${table.name}"`,
+          [], database
+        )
+        table.rowCount = parseInt(countResult.rows[0].count) || 0
+      } catch (error) {
+        table.rowCount = 0
+      }
+    })
+    await Promise.all(countPromises)
+    return tables
+  }
+
+  async getColumns (schema, table, database = null) {
+    const result = await this.query(`
+      SELECT
+        column_name,
+        data_type,
+        is_nullable,
+        column_default,
+        character_maximum_length,
+        numeric_precision,
+        numeric_scale
+      FROM information_schema.columns
+      WHERE table_schema = $1 AND table_name = $2
+      ORDER BY ordinal_position
+    `, [schema, table], database)
+    return result.rows.map(row => ({
+      name: row.column_name,
+      type: row.data_type,
+      nullable: row.is_nullable === 'YES',
+      default: row.column_default,
+      maxLength: row.character_maximum_length,
+      precision: row.numeric_precision,
+      scale: row.numeric_scale
+    }))
+  }
+
+  async getRows (schema, table, limit = 50, offset = 0, database = null) {
+    const countResult = await this.query(
+      `SELECT COUNT(*) as total FROM "${schema}"."${table}"`,
+      [],
+      database
+    )
+    const total = parseInt(countResult.rows[0].total)
+    const result = await this.query(
+      `SELECT * FROM "${schema}"."${table}" LIMIT $1 OFFSET $2`,
+      [limit, offset],
+      database
+    )
+    return {
+      rows: result.rows,
+      total,
+      fields: result.fields.map(f => ({ name: f.name, dataTypeID: f.dataTypeID }))
+    }
+  }
+
+  async executeQuery (sql, database = null) {
+    const trimmedSql = sql.trim().toUpperCase()
+    const allowedStatements = ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'WITH']
+    const firstWord = trimmedSql.split(/\s/)[0]
+    if (!allowedStatements.includes(firstWord)) {
+      throw new Error('Only SELECT, INSERT, UPDATE, and DELETE queries are allowed')
+    }
+    const result = await this.query(sql, [], database)
+    return {
+      rows: result.rows || [],
+      rowCount: result.rowCount,
+      fields: result.fields ? result.fields.map(f => ({ name: f.name, dataTypeID: f.dataTypeID })) : []
+    }
+  }
+
+  async getDatabases () {
+    const result = await this.query(`
+      SELECT datname as name
+      FROM pg_database
+      WHERE datistemplate = false
+      ORDER BY datname
+    `)
+    return result.rows.map(row => row.name)
+  }
+
+  async shutdown () {
+    for (const [dbName, pool] of this.pools) {
+      await pool.end()
+    }
+    this.pools.clear()
+  }
+}
+
+export const PostgresClient = new PostgresClientClass()