@gokiteam/goki-dev 0.2.0

package/src/middleware/Reply.js

@@ -0,0 +1,24 @@
+export const Reply = () => {
+  return async (ctx, next) => {
+    ctx.reply = (data, status = 200) => {
+      ctx.status = status
+      ctx.body = {
+        success: true,
+        status,
+        message: getDefaultMessage(status),
+        data
+      }
+    }
+    await next()
+  }
+}
+
+const getDefaultMessage = (status) => {
+  const messages = {
+    200: 'The request succeeded.',
+    201: 'The resource was created successfully.',
+    202: 'The request was accepted for processing.',
+    204: 'The request succeeded with no content to return.'
+  }
+  return messages[status] || 'Request completed.'
+}

package/src/middleware/TraceId.js

@@ -0,0 +1,9 @@
+import { randomUUID } from 'crypto'
+
+export const TraceId = () => {
+  return async (ctx, next) => {
+    const traceId = ctx.request.body?.traceId || ctx.headers['x-trace-id'] || randomUUID()
+    ctx.state.traceId = traceId
+    await next()
+  }
+}

package/src/middleware/WebhookProxy.js

@@ -0,0 +1,234 @@
+import zlib from 'zlib'
+import { WebhookProxy } from '../singletons/WebhookProxy.js'
+import { Logger } from '../singletons/Logger.js'
+
+const MAX_BODY_LOG_SIZE = 8192
+
+function tryParseJson (str) {
+  if (!str || typeof str !== 'string') return null
+  const trimmed = str.trim()
+  if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
+    try { return JSON.parse(trimmed) } catch { return null }
+  }
+  return null
+}
+
+function truncateBody (body, maxSize = MAX_BODY_LOG_SIZE) {
+  if (!body) return null
+  const str = typeof body === 'string' ? body : JSON.stringify(body)
+  if (str.length > maxSize) {
+    return str.substring(0, maxSize) + `... [truncated, total ${str.length} bytes]`
+  }
+  return tryParseJson(str) || str
+}
+
+function decompressBuffer (buffer, encoding) {
+  if (!encoding) return buffer
+  try {
+    if (encoding === 'gzip' || encoding === 'x-gzip') {
+      return zlib.gunzipSync(buffer)
+    }
+    if (encoding === 'deflate') {
+      return zlib.inflateSync(buffer)
+    }
+    if (encoding === 'br') {
+      return zlib.brotliDecompressSync(buffer)
+    }
+  } catch {
+    return buffer
+  }
+  return buffer
+}
+
+function collectStreamBody (stream, contentEncoding) {
+  return new Promise((resolve) => {
+    const chunks = []
+    let size = 0
+    stream.on('data', (chunk) => {
+      if (size < MAX_BODY_LOG_SIZE) {
+        chunks.push(chunk)
+      }
+      size += chunk.length
+    })
+    stream.on('end', () => {
+      try {
+        const raw = decompressBuffer(Buffer.concat(chunks), contentEncoding).toString('utf-8')
+        resolve({ raw, totalSize: size })
+      } catch {
+        resolve({ raw: null, totalSize: size })
+      }
+    })
+    stream.on('error', () => {
+      resolve({ raw: null, totalSize: size })
+    })
+  })
+}
+
+function bufferRequestBody (req) {
+  return new Promise((resolve) => {
+    const chunks = []
+    let size = 0
+    req.on('data', (chunk) => {
+      chunks.push(chunk)
+      size += chunk.length
+    })
+    req.on('end', () => {
+      try {
+        resolve(Buffer.concat(chunks))
+      } catch {
+        resolve(null)
+      }
+    })
+    req.on('error', () => {
+      resolve(null)
+    })
+    // If already consumed or no body, resolve immediately
+    if (req.readableEnded || req.complete) {
+      resolve(null)
+    }
+  })
+}
+
+export const WebhookProxyMiddleware = () => {
+  return async (ctx, next) => {
+    // Handle both /v1/webhooks/proxy/* (explicit routes) and /v1/webhooks/* (auto-routing)
+    if (!ctx.path.startsWith('/v1/webhooks/')) {
+      return next()
+    }
+    let resolved = WebhookProxy.resolveTarget(ctx.path)
+    if (!resolved) {
+      // Auto-route to gateway-webhook if path matches /v1/webhooks/{endpoint}
+      const webhookMatch = ctx.path.match(/^\/v1\/webhooks\/([^/?]+)/)
+      if (webhookMatch) {
+        const endpoint = webhookMatch[1]
+        // Skip 'proxy' prefix - those use explicit routing
+        if (endpoint !== 'proxy') {
+          resolved = {
+            target: 'http://gateway-webhook-app:3000',
+            path: ctx.path,
+            fullUrl: `http://gateway-webhook-app:3000${ctx.path}`,
+            prefix: endpoint,
+            isAutoRouted: true
+          }
+          Logger.log({
+            level: 'info',
+            message: 'Webhook proxy: auto-routing to gateway-webhook',
+            data: { method: ctx.method, path: ctx.path, endpoint, target: resolved.target }
+          })
+        }
+      }
+      if (!resolved) {
+        Logger.log({
+          level: 'warn',
+          message: 'Webhook proxy: no route configured',
+          data: { method: ctx.method, path: ctx.path }
+        })
+        ctx.status = 404
+        ctx.body = {
+          success: false,
+          status: 404,
+          message: 'No webhook route configured for this path',
+          data: { path: ctx.path }
+        }
+        return
+      }
+    }
+    const startTime = Date.now()
+    const queryParams = ctx.querystring ? Object.fromEntries(new URLSearchParams(ctx.querystring)) : null
+    // Buffer the raw request body so we can log it AND pass it to the proxy
+    const rawBodyBuffer = await bufferRequestBody(ctx.req)
+    const requestBodyStr = rawBodyBuffer ? rawBodyBuffer.toString('utf-8') : null
+    Logger.log({
+      level: 'info',
+      message: `Webhook IN: ${ctx.method} ${ctx.path} → ${resolved.fullUrl}`,
+      data: {
+        method: ctx.method,
+        path: ctx.path,
+        target: resolved.fullUrl,
+        queryParams,
+        requestBody: truncateBody(requestBodyStr)
+      }
+    })
+    // Bypass Koa response handling
+    ctx.respond = false
+    // Rewrite the URL before proxying
+    ctx.req.url = resolved.path + (ctx.querystring ? '?' + ctx.querystring : '')
+    // Create a readable stream from the buffered body so http-proxy can forward it
+    const { Readable } = await import('stream')
+    const bodyStream = rawBodyBuffer
+      ? Readable.from(rawBodyBuffer)
+      : Readable.from(Buffer.alloc(0))
+    return new Promise((resolve) => {
+      WebhookProxy.proxy.web(ctx.req, ctx.res, {
+        target: resolved.target,
+        buffer: bodyStream
+      }, (err) => {
+        const elapsed = Date.now() - startTime
+        if (err) {
+          Logger.log({
+            level: 'error',
+            message: `Webhook ERR: ${ctx.method} ${ctx.path} → 502 (${elapsed}ms) ${err.message}`,
+            data: {
+              method: ctx.method,
+              path: ctx.path,
+              target: resolved.fullUrl,
+              error: err.message,
+              responseTimeMs: elapsed,
+              requestBody: truncateBody(requestBodyStr)
+            }
+          })
+          WebhookProxy.logRequest({
+            prefix: resolved.prefix,
+            method: ctx.method,
+            originalPath: ctx.path,
+            targetUrl: resolved.fullUrl,
+            statusCode: 502,
+            responseTimeMs: elapsed,
+            error: err.message,
+            queryParams,
+            requestBody: truncateBody(requestBodyStr)
+          })
+          if (!ctx.res.headersSent) {
+            ctx.res.writeHead(502, { 'Content-Type': 'application/json' })
+          }
+          ctx.res.end(JSON.stringify({
+            success: false,
+            message: `Webhook proxy error: ${err.message}`,
+            target: resolved.fullUrl
+          }))
+        }
+        resolve()
+      })
+      // Log successful responses via proxyRes event
+      WebhookProxy.proxy.once('proxyRes', async (proxyRes) => {
+        const elapsed = Date.now() - startTime
+        const contentEncoding = (proxyRes.headers['content-encoding'] || '').trim().toLowerCase()
+        const { raw: responseBody, totalSize: responseSize } = await collectStreamBody(proxyRes, contentEncoding || null)
+        Logger.log({
+          level: proxyRes.statusCode >= 400 ? 'warn' : 'info',
+          message: `Webhook OUT: ${ctx.method} ${ctx.path} → ${proxyRes.statusCode} (${elapsed}ms)`,
+          data: {
+            method: ctx.method,
+            path: ctx.path,
+            target: resolved.fullUrl,
+            statusCode: proxyRes.statusCode,
+            responseTimeMs: elapsed,
+            responseBody: truncateBody(responseBody),
+            responseSize
+          }
+        })
+        WebhookProxy.logRequest({
+          prefix: resolved.prefix,
+          method: ctx.method,
+          originalPath: ctx.path,
+          targetUrl: resolved.fullUrl,
+          statusCode: proxyRes.statusCode,
+          responseTimeMs: elapsed,
+          queryParams,
+          requestBody: truncateBody(requestBodyStr),
+          responseBody: truncateBody(responseBody)
+        })
+      })
+    })
+  }
+}

package/src/protocols/mqtt/Broker.js

@@ -0,0 +1,92 @@
+import aedes from 'aedes'
+import { Logger } from '../../singletons/Logger.js'
+import { Handlers } from './Handlers.js'
+
+class BrokerClass {
+  constructor () {
+    this.broker = null
+    this.isInitialized = false
+  }
+
+  initialize () {
+    if (this.isInitialized) {
+      Logger.log({
+        level: 'warn',
+        message: 'MQTT Broker already initialized'
+      })
+      return this.broker
+    }
+
+    // Create Aedes broker instance
+    this.broker = aedes()
+
+    // Register event handlers
+    this.setupEventHandlers()
+
+    this.isInitialized = true
+
+    Logger.log({
+      level: 'info',
+      message: 'MQTT Broker initialized'
+    })
+
+    return this.broker
+  }
+
+  setupEventHandlers () {
+    // Client connected
+    this.broker.on('client', (client) => {
+      Handlers.handleClientConnect(client)
+    })
+
+    // Client disconnected
+    this.broker.on('clientDisconnect', (client) => {
+      Handlers.handleClientDisconnect(client)
+    })
+
+    // Message published
+    this.broker.on('publish', (packet, client) => {
+      Handlers.handlePublish(packet, client)
+    })
+
+    // Client subscribed to topic
+    this.broker.on('subscribe', (subscriptions, client) => {
+      Handlers.handleSubscribe(subscriptions, client)
+    })
+
+    // Client unsubscribed from topic
+    this.broker.on('unsubscribe', (unsubscriptions, client) => {
+      Handlers.handleUnsubscribe(unsubscriptions, client)
+    })
+
+    // Client error
+    this.broker.on('clientError', (client, error) => {
+      Handlers.handleClientError(client, error)
+    })
+
+    // Connection error
+    this.broker.on('connectionError', (client, error) => {
+      Handlers.handleConnectionError(client, error)
+    })
+  }
+
+  getInstance () {
+    if (!this.isInitialized) {
+      throw new Error('MQTT Broker not initialized. Call initialize() first.')
+    }
+    return this.broker
+  }
+
+  shutdown () {
+    if (this.broker) {
+      this.broker.close(() => {
+        Logger.log({
+          level: 'info',
+          message: 'MQTT Broker closed'
+        })
+      })
+    }
+  }
+}
+
+export const Broker = new BrokerClass()

package/src/protocols/mqtt/Handlers.js

@@ -0,0 +1,175 @@
+import Moment from 'moment'
+import { Logger } from '../../singletons/Logger.js'
+import { SqliteStore } from '../../singletons/SqliteStore.js'
+import { PubSubBridge } from './PubSubBridge.js'
+
+class HandlersClass {
+  handleClientConnect (client) {
+    const clientData = {
+      clientId: client.id,
+      connectedAt: Moment().toISOString(),
+      disconnectedAt: null,
+      will: client.will || null,
+      clean: client.clean || false,
+      version: client.version || null
+    }
+
+    // Store client connection
+    SqliteStore.create('mqtt_clients', clientData)
+
+    Logger.log({
+      level: 'info',
+      message: 'MQTT client connected',
+      data: {
+        clientId: client.id,
+        clean: client.clean,
+        will: !!client.will
+      }
+    })
+
+    // Publish lifecycle event to Pub/Sub
+    PubSubBridge.publishLifecycleEvent({
+      clientId: client.id,
+      eventType: 'connected'
+    })
+  }
+
+  handleClientDisconnect (client) {
+    try {
+      // Update client disconnection time
+      const existingClient = SqliteStore.get('mqtt_clients', client.id, 'clientId')
+      if (existingClient) {
+        SqliteStore.update(
+          'mqtt_clients',
+          client.id,
+          { disconnectedAt: Moment().toISOString() },
+          'clientId'
+        )
+      }
+
+      Logger.log({
+        level: 'info',
+        message: 'MQTT client disconnected',
+        data: { clientId: client.id }
+      })
+
+      // Publish lifecycle event to Pub/Sub
+      PubSubBridge.publishLifecycleEvent({
+        clientId: client.id,
+        eventType: 'disconnected'
+      })
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Error handling client disconnect',
+        data: { clientId: client.id, error: error.message }
+      })
+    }
+  }
+
+  handlePublish (packet, client) {
+    // Skip internal $SYS topics
+    if (packet.topic.startsWith('$SYS/')) {
+      return
+    }
+
+    const messageData = {
+      id: `msg-${Moment().valueOf()}-${Math.random().toString(36).substr(2, 9)}`,
+      topic: packet.topic,
+      payload: packet.payload.toString(),
+      qos: packet.qos || 0,
+      retain: packet.retain || false,
+      timestamp: Moment().toISOString(),
+      clientId: client ? client.id : 'broker'
+    }
+
+    // Store message
+    SqliteStore.create('mqtt_messages', messageData)
+
+    Logger.log({
+      level: 'debug',
+      message: 'MQTT message published',
+      data: {
+        topic: packet.topic,
+        clientId: messageData.clientId,
+        qos: messageData.qos,
+        retain: messageData.retain,
+        payloadSize: packet.payload.length
+      }
+    })
+
+    // Bridge message to Pub/Sub (mimics AWS IoT Core behavior)
+    // Only bridge messages from actual clients, not broker-published messages
+    if (client) {
+      PubSubBridge.publishToPubSub({
+        clientId: client.id,
+        topicName: packet.topic,
+        payload: packet.payload
+      })
+    }
+  }
+
+  handleSubscribe (subscriptions, client) {
+    subscriptions.forEach(sub => {
+      const subscriptionData = {
+        id: `sub-${Moment().valueOf()}-${Math.random().toString(36).substr(2, 9)}`,
+        clientId: client.id,
+        topic: sub.topic,
+        qos: sub.qos || 0,
+        timestamp: Moment().toISOString()
+      }
+
+      // Store subscription
+      SqliteStore.create('mqtt_subscriptions', subscriptionData)
+
+      Logger.log({
+        level: 'debug',
+        message: 'MQTT client subscribed',
+        data: {
+          clientId: client.id,
+          topic: sub.topic,
+          qos: sub.qos
+        }
+      })
+    })
+  }
+
+  handleUnsubscribe (unsubscriptions, client) {
+    unsubscriptions.forEach(topic => {
+      Logger.log({
+        level: 'debug',
+        message: 'MQTT client unsubscribed',
+        data: {
+          clientId: client.id,
+          topic
+        }
+      })
+    })
+  }
+
+  handleClientError (client, error) {
+    Logger.log({
+      level: 'error',
+      message: 'MQTT client error',
+      data: {
+        clientId: client.id,
+        error: error.message,
+        stack: error.stack
+      }
+    })
+  }
+
+  handleConnectionError (client, error) {
+    Logger.log({
+      level: 'error',
+      message: 'MQTT connection error',
+      data: {
+        clientId: client ? client.id : 'unknown',
+        error: error.message,
+        stack: error.stack
+      }
+    })
+  }
+}
+
+export const Handlers = new HandlersClass()

package/src/protocols/mqtt/PubSubBridge.js

@@ -0,0 +1,162 @@
+import { Application } from '../../configs/Application.js'
+import { Logger } from '../../singletons/Logger.js'
+
+const { pubsub } = Application
+const PUBSUB_API = `http://${pubsub.emulatorHost}:${pubsub.emulatorPort}`
+const PROJECT_ID = pubsub.projectId
+
+/**
+ * MQTT → Pub/Sub Bridge
+ *
+ * Routes MQTT messages to Google Cloud Pub/Sub emulator
+ * Mimics AWS IoT Core's behavior of bridging MQTT to Pub/Sub
+ *
+ * Message format matches AWS IoT Core:
+ * {
+ *   clientId: string,
+ *   topicName: string,
+ *   packet: string (base64-encoded binary),
+ *   receivedAtMilliseconds: number
+ * }
+ */
+class PubSubBridgeClass {
+  constructor () {
+    this.enabled = true
+    this.pubsubTopic = 'mqttMessageReceived'
+  }
+
+  /**
+   * Publish MQTT message to Pub/Sub
+   *
+   * @param {Object} params
+   * @param {string} params.clientId - MQTT client ID
+   * @param {string} params.topicName - MQTT topic name
+   * @param {Buffer} params.payload - Binary message payload
+   * @returns {Promise<void>}
+   */
+  async publishToPubSub ({ clientId, topicName, payload }) {
+    if (!this.enabled) {
+      return
+    }
+
+    try {
+      // Format message to match AWS IoT Core format
+      const message = {
+        clientId: clientId || 'N/A',
+        topicName,
+        packet: payload.toString('base64'),
+        receivedAtMilliseconds: Date.now()
+      }
+
+      // Convert message to base64-encoded JSON
+      const messageData = Buffer.from(JSON.stringify(message)).toString('base64')
+
+      // Publish to Pub/Sub emulator
+      const response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics/${this.pubsubTopic}:publish`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          messages: [{ data: messageData }]
+        })
+      })
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        throw new Error(`Pub/Sub publish failed: ${response.status} - ${errorText}`)
+      }
+
+      Logger.log({
+        level: 'debug',
+        message: 'MQTT → Pub/Sub: Message bridged',
+        data: {
+          clientId,
+          topic: topicName,
+          pubsubTopic: this.pubsubTopic,
+          payloadSize: payload.length
+        }
+      })
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'MQTT → Pub/Sub: Bridge failed',
+        data: {
+          clientId,
+          topic: topicName,
+          error: error.message,
+          stack: error.stack
+        }
+      })
+    }
+  }
+
+  /**
+   * Publish lifecycle event (connected/disconnected)
+   *
+   * @param {Object} params
+   * @param {string} params.clientId - MQTT client ID
+   * @param {string} params.eventType - 'connected' or 'disconnected'
+   * @returns {Promise<void>}
+   */
+  async publishLifecycleEvent ({ clientId, eventType }) {
+    if (!this.enabled) {
+      return
+    }
+
+    try {
+      const topicName = `simulator/${eventType}`
+      const eventPayload = JSON.stringify({
+        clientId,
+        eventType,
+        timestamp: Date.now()
+      })
+
+      // Format as MQTT message for consistency
+      const message = {
+        clientId,
+        topicName,
+        packet: Buffer.from(eventPayload).toString('base64'),
+        receivedAtMilliseconds: Date.now()
+      }
+
+      // Convert message to base64-encoded JSON
+      const messageData = Buffer.from(JSON.stringify(message)).toString('base64')
+
+      // Publish to Pub/Sub emulator
+      const response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics/${this.pubsubTopic}:publish`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          messages: [{ data: messageData }]
+        })
+      })
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        throw new Error(`Lifecycle event publish failed: ${response.status} - ${errorText}`)
+      }
+
+      Logger.log({
+        level: 'debug',
+        message: 'MQTT → Pub/Sub: Lifecycle event published',
+        data: {
+          clientId,
+          eventType,
+          topic: topicName
+        }
+      })
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'MQTT → Pub/Sub: Lifecycle event failed',
+        data: {
+          clientId,
+          eventType,
+          error: error.message,
+          stack: error.stack
+        }
+      })
+    }
+  }
+}
+
+export const PubSubBridge = new PubSubBridgeClass()