@gokiteam/goki-dev 0.2.0

package/src/singletons/LogBroadcaster.js

@@ -0,0 +1,159 @@
+import { Logger } from './Logger.js'
+
+const SEVERITY_ORDER = ['DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL', 'ALERT', 'EMERGENCY']
+
+/**
+ * Check if a log entry matches a MongoDB-style filter object.
+ * Supports: exact match, $in, $nin, $regex, $gte/$lte/$gt/$lt, $ne, $eq,
+ *           $or, $nor, $not, $search
+ */
+const matchesFilter = (entry, filter) => {
+  if (!filter || Object.keys(filter).length === 0) return true
+  for (const [key, condition] of Object.entries(filter)) {
+    if (key === '$or') {
+      if (!condition.some(sub => matchesFilter(entry, sub))) return false
+      continue
+    }
+    if (key === '$nor') {
+      if (condition.some(sub => matchesFilter(entry, sub))) return false
+      continue
+    }
+    if (key === '$search') {
+      const { term, fields } = condition
+      const regex = new RegExp(term, 'i')
+      if (!fields.some(f => {
+        const val = getField(entry, f)
+        return val != null && regex.test(String(val))
+      })) return false
+      continue
+    }
+    if (!matchCondition(entry, key, condition)) return false
+  }
+  return true
+}
+
+const getField = (entry, field) => {
+  const parts = field.split('.')
+  let value = entry
+  for (const part of parts) {
+    if (value == null) return undefined
+    if (typeof value[part] === 'string' && (part === 'labels' || part === 'jsonPayload')) {
+      try { value = JSON.parse(value[part]) } catch { value = value[part] }
+    } else {
+      value = value[part]
+    }
+  }
+  return value
+}
+
+const severityRank = (s) => {
+  const i = SEVERITY_ORDER.indexOf(s)
+  return i === -1 ? 0 : i
+}
+
+const matchCondition = (entry, field, condition) => {
+  const val = getField(entry, field)
+  if (typeof condition !== 'object' || condition === null) {
+    return String(val) === String(condition)
+  }
+  if (condition.$eq !== undefined) return String(val) === String(condition.$eq)
+  if (condition.$ne !== undefined) return String(val) !== String(condition.$ne)
+  if (condition.$gt !== undefined) {
+    return field === 'severity' ? severityRank(val) > severityRank(condition.$gt) : val > condition.$gt
+  }
+  if (condition.$gte !== undefined) {
+    return field === 'severity' ? severityRank(val) >= severityRank(condition.$gte) : val >= condition.$gte
+  }
+  if (condition.$lt !== undefined) {
+    return field === 'severity' ? severityRank(val) < severityRank(condition.$lt) : val < condition.$lt
+  }
+  if (condition.$lte !== undefined) {
+    return field === 'severity' ? severityRank(val) <= severityRank(condition.$lte) : val <= condition.$lte
+  }
+  if (condition.$in !== undefined) return condition.$in.includes(val)
+  if (condition.$nin !== undefined) return !condition.$nin.includes(val)
+  if (condition.$regex !== undefined) {
+    if (val == null) return false
+    try { return new RegExp(condition.$regex, condition.$options || '').test(String(val)) } catch { return false }
+  }
+  if (condition.$not !== undefined) return !matchCondition(entry, field, condition.$not)
+  return true
+}
+
+class LogBroadcasterClass {
+  constructor () {
+    this.clients = new Map()
+    this.heartbeatId = null
+    this.heartbeatIntervalMs = 30000
+  }
+
+  addClient (connectionId, ctx, filter = null) {
+    this.clients.set(connectionId, { ctx, filter })
+    this.startHeartbeat()
+    Logger.log({
+      level: 'info',
+      message: 'Log SSE client connected',
+      data: { connectionId, totalClients: this.clients.size, hasFilter: !!filter }
+    })
+  }
+
+  removeClient (connectionId) {
+    const removed = this.clients.delete(connectionId)
+    if (removed) {
+      Logger.log({
+        level: 'info',
+        message: 'Log SSE client disconnected',
+        data: { connectionId, totalClients: this.clients.size }
+      })
+      if (this.clients.size === 0) {
+        this.stopHeartbeat()
+      }
+    }
+  }
+
+  broadcast (entry) {
+    if (this.clients.size === 0) return
+    const data = JSON.stringify(entry)
+    for (const [connectionId, { ctx, filter }] of this.clients) {
+      try {
+        if (filter && !matchesFilter(entry, filter)) continue
+        ctx.res.write(`event: log\ndata: ${data}\n\n`)
+      } catch (error) {
+        Logger.log({
+          level: 'error',
+          message: 'Failed to send log SSE to client',
+          data: { connectionId, error: error.message }
+        })
+        this.removeClient(connectionId)
+      }
+    }
+  }
+
+  startHeartbeat () {
+    if (this.heartbeatId) return
+    this.heartbeatId = setInterval(() => this.sendHeartbeat(), this.heartbeatIntervalMs)
+  }
+
+  stopHeartbeat () {
+    if (this.heartbeatId) {
+      clearInterval(this.heartbeatId)
+      this.heartbeatId = null
+    }
+  }
+
+  sendHeartbeat () {
+    for (const [connectionId, { ctx }] of this.clients) {
+      try {
+        ctx.res.write(': heartbeat\n\n')
+      } catch (error) {
+        this.removeClient(connectionId)
+      }
+    }
+  }
+
+  getClientCount () {
+    return this.clients.size
+  }
+}
+
+export const LogBroadcaster = new LogBroadcasterClass()

package/src/singletons/Logger.js

@@ -0,0 +1,49 @@
+import { Application } from '../configs/Application.js'
+
+const { logLevel, environment } = Application
+
+const logLevels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  debug: 3
+}
+
+const currentLogLevel = logLevels[logLevel] || logLevels.info
+
+class LoggerClass {
+  log ({ level, message, data = {} }) {
+    const levelValue = logLevels[level] || logLevels.info
+    if (levelValue > currentLogLevel) return
+    const timestamp = new Date().toISOString()
+    const logEntry = {
+      timestamp,
+      level,
+      message,
+      environment,
+      ...data
+    }
+    const output = environment === 'development'
+      ? JSON.stringify(logEntry, null, 2)
+      : JSON.stringify(logEntry)
+    console.log(output)
+  }
+
+  error (message, data) {
+    this.log({ level: 'error', message, data })
+  }
+
+  warn (message, data) {
+    this.log({ level: 'warn', message, data })
+  }
+
+  info (message, data) {
+    this.log({ level: 'info', message, data })
+  }
+
+  debug (message, data) {
+    this.log({ level: 'debug', message, data })
+  }
+}
+
+export const Logger = new LoggerClass()

package/src/singletons/MemoryJsonStore.js

@@ -0,0 +1,175 @@
+import fs from 'fs'
+import path from 'path'
+import { Application } from '../configs/Application.js'
+import { Default } from '../configs/Default.js'
+
+class MemoryJsonStoreClass {
+  constructor () {
+    this.collections = new Map()
+    this.flushTimers = new Map()
+    this.dirtyCollections = new Set()
+    this.isInitialized = false
+  }
+
+  initialize () {
+    if (this.isInitialized) return
+    const dataDir = Application.storage.dataDir
+    if (!fs.existsSync(dataDir)) {
+      fs.mkdirSync(dataDir, { recursive: true })
+    }
+    this.isInitialized = true
+  }
+
+  registerCollection (collectionName, filePath) {
+    if (this.collections.has(collectionName)) {
+      throw new Error(`Collection ${collectionName} already registered`)
+    }
+    const fullPath = path.join(Application.storage.dataDir, filePath)
+    const dir = path.dirname(fullPath)
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true })
+    }
+    let data = []
+    if (fs.existsSync(fullPath)) {
+      const content = fs.readFileSync(fullPath, 'utf8')
+      data = content ? JSON.parse(content) : []
+    }
+    this.collections.set(collectionName, {
+      data,
+      filePath: fullPath
+    })
+    this.startAutoFlush(collectionName)
+  }
+
+  startAutoFlush (collectionName) {
+    const intervalMs = Default.storage.autoFlushIntervalMs
+    const timer = setInterval(() => {
+      if (this.dirtyCollections.has(collectionName)) {
+        this.flush(collectionName)
+      }
+    }, intervalMs)
+    this.flushTimers.set(collectionName, timer)
+  }
+
+  markDirty (collectionName) {
+    this.dirtyCollections.add(collectionName)
+  }
+
+  flush (collectionName) {
+    const collection = this.collections.get(collectionName)
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`)
+    }
+    const jsonData = JSON.stringify(collection.data, null, 2)
+    fs.writeFileSync(collection.filePath, jsonData, 'utf8')
+    this.dirtyCollections.delete(collectionName)
+  }
+
+  flushAll () {
+    for (const collectionName of this.dirtyCollections) {
+      this.flush(collectionName)
+    }
+  }
+
+  get (collectionName, id, idField = 'id') {
+    const collection = this.collections.get(collectionName)
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`)
+    }
+    return collection.data.find(item => item[idField] === id)
+  }
+
+  find (collectionName, predicate) {
+    const collection = this.collections.get(collectionName)
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`)
+    }
+    return collection.data.filter(predicate)
+  }
+
+  list (collectionName, options = {}) {
+    const collection = this.collections.get(collectionName)
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`)
+    }
+    let data = [...collection.data]
+    if (options.filter) {
+      data = data.filter(options.filter)
+    }
+    if (options.sort) {
+      data = data.sort(options.sort)
+    }
+    const total = data.length
+    if (options.offset !== undefined) {
+      data = data.slice(options.offset)
+    }
+    if (options.limit !== undefined) {
+      data = data.slice(0, options.limit)
+    }
+    return { data, total }
+  }
+
+  create (collectionName, item) {
+    const collection = this.collections.get(collectionName)
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`)
+    }
+    collection.data.push(item)
+    this.markDirty(collectionName)
+    return item
+  }
+
+  update (collectionName, id, updates, idField = 'id') {
+    const collection = this.collections.get(collectionName)
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`)
+    }
+    const index = collection.data.findIndex(item => item[idField] === id)
+    if (index === -1) {
+      throw new Error(`Item with ${idField}=${id} not found in ${collectionName}`)
+    }
+    collection.data[index] = { ...collection.data[index], ...updates }
+    this.markDirty(collectionName)
+    return collection.data[index]
+  }
+
+  delete (collectionName, id, idField = 'id') {
+    const collection = this.collections.get(collectionName)
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`)
+    }
+    const index = collection.data.findIndex(item => item[idField] === id)
+    if (index === -1) {
+      throw new Error(`Item with ${idField}=${id} not found in ${collectionName}`)
+    }
+    const deleted = collection.data.splice(index, 1)[0]
+    this.markDirty(collectionName)
+    return deleted
+  }
+
+  clear (collectionName) {
+    const collection = this.collections.get(collectionName)
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`)
+    }
+    collection.data = []
+    this.markDirty(collectionName)
+    this.flush(collectionName)
+  }
+
+  clearAll () {
+    for (const collectionName of this.collections.keys()) {
+      this.clear(collectionName)
+    }
+  }
+
+  shutdown () {
+    this.flushAll()
+    for (const timer of this.flushTimers.values()) {
+      clearInterval(timer)
+    }
+    this.flushTimers.clear()
+  }
+}
+
+export const MemoryJsonStore = new MemoryJsonStoreClass()

package/src/singletons/MessageBroadcaster.js

@@ -0,0 +1,190 @@
+// MessageBroadcaster singleton for managing SSE connections to pub/sub message streams
+// Handles client registration, filtering, and broadcasting new messages
+
+import { Logger } from './Logger.js'
+
+class MessageBroadcasterClass {
+  constructor () {
+    this.clients = new Map() // connectionId → { ctx, filters }
+    this.heartbeatId = null
+    this.heartbeatIntervalMs = 30000
+  }
+
+  /**
+   * Register a new SSE client
+   * @param {string} connectionId - Unique identifier for the connection (traceId)
+   * @param {Object} ctx - Koa context with res.write for SSE
+   * @param {Object} filters - { topic?, sender? }
+   */
+  addClient (connectionId, ctx, filters = {}) {
+    this.clients.set(connectionId, { ctx, filters })
+    this.startHeartbeat()
+
+    Logger.log({
+      level: 'info',
+      message: 'SSE client connected',
+      data: { connectionId, filters, totalClients: this.clients.size }
+    })
+  }
+
+  /**
+   * Unregister an SSE client
+   * @param {string} connectionId - Unique identifier for the connection
+   */
+  removeClient (connectionId) {
+    const removed = this.clients.delete(connectionId)
+
+    if (removed) {
+      Logger.log({
+        level: 'info',
+        message: 'SSE client disconnected',
+        data: { connectionId, totalClients: this.clients.size }
+      })
+      if (this.clients.size === 0) {
+        this.stopHeartbeat()
+      }
+    }
+  }
+
+  /**
+   * Broadcast a message to all connected clients matching filters
+   * @param {Object} message - { messageId, topic, data, attributes, publishTime }
+   */
+  broadcast (message) {
+    if (this.clients.size === 0) return
+    let sentCount = 0
+
+    for (const [connectionId, { ctx, filters }] of this.clients) {
+      if (this.matchesFilters(message, filters)) {
+        try {
+          this.sendSSE(ctx, 'message', message)
+          sentCount++
+        } catch (error) {
+          Logger.log({
+            level: 'error',
+            message: 'Failed to send SSE message to client',
+            data: { connectionId, error: error.message }
+          })
+          // Remove dead connection
+          this.removeClient(connectionId)
+        }
+      }
+    }
+
+    Logger.log({
+      level: 'debug',
+      message: 'Message broadcasted to SSE clients',
+      data: {
+        messageId: message.messageId,
+        topic: message.topic,
+        totalClients: this.clients.size,
+        sentToClients: sentCount
+      }
+    })
+  }
+
+  /**
+   * Check if message matches client filters
+   * @param {Object} message - The message to check
+   * @param {Object} filters - { topic?, sender? }
+   * @returns {boolean}
+   */
+  matchesFilters (message, filters) {
+    // No filters = receive all messages
+    if (!filters || Object.keys(filters).length === 0) {
+      return true
+    }
+
+    // Topic filter
+    if (filters.topic && message.topic !== filters.topic) {
+      return false
+    }
+
+    // Sender filter (extracted from attributes)
+    if (filters.sender) {
+      const messageSender = message.attributes?.sender || message.sender
+      if (messageSender !== filters.sender) {
+        return false
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Send SSE formatted message to client
+   * @param {Object} ctx - Koa context
+   * @param {string} event - Event name
+   * @param {Object} data - Data to send
+   */
+  sendSSE (ctx, event, data) {
+    const message = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`
+    ctx.res.write(message)
+  }
+
+  /**
+   * Start the shared heartbeat interval (one for all clients)
+   */
+  startHeartbeat () {
+    if (this.heartbeatId) return
+    this.heartbeatId = setInterval(() => this.sendHeartbeat(), this.heartbeatIntervalMs)
+  }
+
+  /**
+   * Stop the shared heartbeat interval when no clients remain
+   */
+  stopHeartbeat () {
+    if (this.heartbeatId) {
+      clearInterval(this.heartbeatId)
+      this.heartbeatId = null
+    }
+  }
+
+  /**
+   * Send heartbeat to all connected clients
+   * Used to keep connections alive
+   */
+  sendHeartbeat () {
+    for (const [connectionId, { ctx }] of this.clients) {
+      try {
+        ctx.res.write(': heartbeat\n\n')
+      } catch (error) {
+        Logger.log({
+          level: 'error',
+          message: 'Failed to send heartbeat to client',
+          data: { connectionId, error: error.message }
+        })
+        this.removeClient(connectionId)
+      }
+    }
+  }
+
+  /**
+   * Get current client count
+   * @returns {number}
+   */
+  getClientCount () {
+    return this.clients.size
+  }
+
+  /**
+   * Get client statistics
+   * @returns {Object}
+   */
+  getStats () {
+    const stats = {
+      totalClients: this.clients.size,
+      clientsByFilter: {}
+    }
+
+    for (const [, { filters }] of this.clients) {
+      const filterKey = JSON.stringify(filters)
+      stats.clientsByFilter[filterKey] = (stats.clientsByFilter[filterKey] || 0) + 1
+    }
+
+    return stats
+  }
+}
+
+// Export singleton instance
+export const MessageBroadcaster = new MessageBroadcasterClass()