@gokiteam/goki-dev 0.2.0

package/patterns/testing/Patterns.md

@@ -0,0 +1,232 @@
+# Testing Patterns
+
+## Test structure
+
+Coding pattern:
+```javascript
+import { describe, it, beforeEach } from 'mocha'
+import { expect } from 'chai'
+
+describe('Feature Name', () => {
+  beforeEach(async function () {
+    const { id } = this.currentTest.ctx
+    this.testId = id
+  })
+
+  describe('Happy paths', () => {
+    it('should work with valid input', async function () {
+      // Test implementation
+    })
+  })
+
+  describe('Extended positive tests', () => {
+    // Additional success cases
+  })
+
+  describe('Edge cases', () => {
+    // Boundary conditions
+  })
+
+  describe('Validation', () => {
+    // Invalid input handling
+  })
+})
+```
+
+## API endpoint testing
+
+Coding pattern:
+```javascript
+import { describe, it, beforeEach } from 'mocha'
+import Chai, { expect } from 'chai'
+import ChaiHttp from 'chai-http'
+import { Application } from '../../../configs/Application.js'
+
+Chai.use(ChaiHttp)
+
+const endpoint = {
+  method: 'post',
+  url: '/v1/devices/create'
+}
+
+describe(`Testing API endpoint: ${endpoint.method.toUpperCase()} ${endpoint.url}`, () => {
+  beforeEach(async function () {
+    const { id } = this.currentTest.ctx
+    this.request = () => Chai.request(Application.server.base)[endpoint.method](endpoint.url)
+      .set('test-id', id)
+  })
+
+  it('should create device successfully', async function () {
+    const response = await this.request().send({
+      propertyId: 'prop-123',
+      model: 1
+    })
+    expect(response).to.have.status(200)
+    expect(response.body.device).to.exist
+  })
+
+  it('should reject missing required field', async function () {
+    const response = await this.request().send({
+      model: 1
+      // Missing propertyId
+    })
+    expect(response).to.have.status(400)
+  })
+})
+```
+
+## Pub/Sub subscriber testing
+
+Coding pattern:
+```javascript
+import { describe, it, beforeEach } from 'mocha'
+import { expect } from 'chai'
+import { PubSubSubscriberProbe } from '@gokiteam/lab'
+import { Application } from '../../../configs/Application.js'
+
+describe('EventReceived Subscriber', () => {
+  beforeEach(async function () {
+    const { id } = this.currentTest.ctx
+    this.testId = id
+    this.probe = new PubSubSubscriberProbe({
+      projectId: Application.pubSub.projectId,
+      keyFilename: Application.pubSub.keyFilename
+    })
+  })
+
+  it('should process event successfully', async function () {
+    const topicName = Application.pubSub.subscribers.eventReceived.subscriptionName
+    await this.probe.publishMessage({
+      topicName,
+      message: {
+        deviceId: 'test-device-id',
+        action: 'update'
+      },
+      attributes: { traceId: this.testId }
+    })
+    const result = await this.probe.waitForResult({
+      topicName,
+      traceId: this.testId,
+      timeoutMilliseconds: 5000
+    })
+    expect(result.state).to.equal('processedSuccessfully')
+  })
+})
+```
+
+## Logic function testing
+
+### Direct testing
+```javascript
+import { describe, it } from 'mocha'
+import { expect } from 'chai'
+import { PrepareDeviceStatusRecord } from '../../../src/logic/PrepareDeviceStatusRecord.js'
+
+describe('PrepareDeviceStatusRecord', () => {
+  it('should prepare device status record', () => {
+    const device = { doc: { propertyId: 'prop-123' } }
+    const deviceStatus = { doc: { batteryStatus: { percentage: 75 } } }
+    const deviceCurrentTimestamp = 1234567890
+    const result = PrepareDeviceStatusRecord({
+      device,
+      deviceStatus,
+      deviceCurrentTimestamp
+    })
+    expect(result).to.be.instanceOf(Buffer)
+  })
+})
+```
+
+### Via HTTP endpoint
+```javascript
+describe('ValidateDeviceControlOperation', () => {
+  beforeEach(async function () {
+    const { id } = this.currentTest.ctx
+    this.request = () => Chai.request(Application.server.base)
+      .post('/logic/ValidateDeviceControlOperation')
+      .set('test-id', id)
+  })
+
+  it('should validate unlock operation', async function () {
+    const response = await this.request().send({
+      key: { /* key data */ },
+      device: { /* device data */ },
+      deviceStatus: { /* status data */ },
+      deviceCurrentTimestamp: 1234567890,
+      isLockOperation: false
+    })
+    expect(response).to.have.status(200)
+    expect(response.body.errorCode).to.be.null
+  })
+})
+```
+
+## Common assertions
+
+### Status codes
+```javascript
+expect(response).to.have.status(200)
+expect(response).to.have.status(400)
+expect(response).to.have.status(404)
+```
+
+### Response body
+```javascript
+expect(response.body).to.exist
+expect(response.body.device).to.have.property('macAddress')
+expect(response.body.device.macAddress).to.be.a('string')
+```
+
+### Arrays
+```javascript
+expect(response.body.devices).to.be.an('array')
+expect(response.body.devices).to.have.lengthOf(5)
+```
+
+### Pattern matching
+```javascript
+expect(response.body.device.macAddress).to.match(/^([0-9A-F]{2}:){5}[0-9A-F]{2}$/)
+```
+
+## Test data management
+
+### Using collections
+```javascript
+beforeEach(async function () {
+  const { id } = this.currentTest.ctx
+  const { collections } = this
+  this.device = await collections.devices.create({
+    propertyId: 'prop-123',
+    macAddress: '00:11:22:33:44:55',
+    model: 1
+  })
+  this.deviceStatus = await collections.deviceStatuses.create({
+    deviceId: this.device.id
+  })
+})
+```
+
+## Running tests
+
+```bash
+# All tests
+npm test
+
+# Parallel execution
+npm run test:parallel
+
+# API tests only
+npm test:api
+
+# Logic tests only
+npm test:logic
+
+# Pub/Sub tests only
+npm test:pubsub
+
+# Specific test file
+npm test -- tests/component/app/devices/create.js
+
+# Specific test by name
+npm test -- --grep "should create device"
+```

package/src/Server.js

@@ -0,0 +1,238 @@
+import Koa from 'koa'
+import bodyParser from 'koa-bodyparser'
+import KoaRouter from 'koa-router'
+import cors from '@koa/cors'
+import { Application } from './configs/Application.js'
+import { Logger } from './singletons/Logger.js'
+import { SqliteStore } from './singletons/SqliteStore.js'
+import { ErrorCatcher } from './middleware/ErrorCatcher.js'
+import { Reply } from './middleware/Reply.js'
+import { TraceId } from './middleware/TraceId.js'
+import { Router as HealthRouter } from './api/health/Router.js'
+import { Router as DashboardRouter } from './api/dashboard/Router.js'
+import { Router as ServicesRouter } from './api/services/Router.js'
+import { Router as ExportRouter } from './api/export/Router.js'
+import { Router as PubsubRouter } from './api/pubsub/Router.js'
+import { Router as LoggingRouter } from './api/logging/Router.js'
+import { Router as MqttRouter } from './api/mqtt/Router.js'
+import { Router as DataRouter } from './api/data/Router.js'
+import { Router as FirestoreRouter } from './api/firestore/Router.js'
+import { Router as PostgresRouter } from './api/postgres/Router.js'
+import { PostgresClient } from './singletons/PostgresClient.js'
+import { RedisClient } from './singletons/RedisClient.js'
+import { Router as RedisRouter } from './api/redis/Router.js'
+import { Router as SchedulerRouter } from './api/scheduler/Router.js'
+import { Router as GatewayRouter } from './api/gateway/Router.js'
+import { Router as DockerRouter } from './api/docker/Router.js'
+import { Router as DocsRouter } from './api/docs/Router.js'
+import { Router as WebhooksRouter } from './api/webhooks/Router.js'
+import { Router as HttpTrafficRouter } from './api/httpTraffic/Router.js'
+import { Router as SnapshotsRouter } from './api/snapshots/Router.js'
+import { Router as FunctionsRouter } from './api/functions/Router.js'
+import { WebhookProxy } from './singletons/WebhookProxy.js'
+import { HttpProxy } from './singletons/HttpProxy.js'
+import { WebhookProxyMiddleware } from './middleware/WebhookProxy.js'
+import { HttpProxyMiddleware } from './middleware/HttpProxy.js'
+import { MqttServer } from './protocols/mqtt/Server.js'
+import { start as startLoggingServer } from './emulation/logging/Server.js'
+import { startPubSubServer } from './emulation/pubsub/Server.js'
+import { startAwsIotServer } from './emulation/awsiot/Server.js'
+import { startCleanupJob } from './jobs/MessageHistoryCleanup.js'
+import { startMaintenanceJob } from './jobs/DatabaseMaintenance.js'
+import { initializeDefaultTopics } from './emulation/pubsub/DefaultTopics.js'
+import { BlackboxLogsConsumer } from './consumers/BlackboxLogsConsumer.js'
+import { DockerLogsConsumer } from './consumers/DockerLogsConsumer.js'
+import { AppGatewayService } from './services/AppGatewayService.js'
+import { FunctionsService } from './singletons/FunctionsService.js'
+import { FunctionTriggerDispatcher } from './singletons/FunctionTriggerDispatcher.js'
+
+const { environment, runId, ports } = Application
+
+const start = async () => {
+  try {
+    SqliteStore.initialize()
+    WebhookProxy.initialize()
+    HttpProxy.initialize()
+    PostgresClient.initialize()
+    RedisClient.initialize()
+    const app = new Koa()
+    app.use(cors({
+      origin: '*', // Allow all origins in development
+      credentials: true,
+      allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'],
+      allowHeaders: ['Content-Type', 'Authorization', 'X-Trace-Id']
+    }))
+    app.use(ErrorCatcher())
+    app.use(TraceId())
+    app.use(WebhookProxyMiddleware())
+    app.use(HttpProxyMiddleware())
+    app.use(Reply())
+    app.use(bodyParser())
+    const router = new KoaRouter()
+    router.use(HealthRouter.routes())
+    router.use(DashboardRouter.routes())
+    router.use(ServicesRouter.routes())
+    router.use(ExportRouter.routes())
+    router.use(PubsubRouter.routes())
+    router.use(LoggingRouter.routes())
+    router.use(MqttRouter.routes())
+    router.use(DataRouter.routes())
+    router.use(FirestoreRouter.routes())
+    router.use(PostgresRouter.routes())
+    router.use(RedisRouter.routes())
+    router.use(SchedulerRouter.routes())
+    router.use(GatewayRouter.routes())
+    router.use(DockerRouter.routes())
+    router.use(DocsRouter.routes())
+    router.use(WebhooksRouter.routes())
+    router.use(HttpTrafficRouter.routes())
+    router.use(SnapshotsRouter.routes())
+    router.use(FunctionsRouter.routes())
+    app.use(router.routes())
+    app.use(router.allowedMethods())
+
+    // Frontend is served separately by goki-dev-tools-frontend container (nginx)
+    // Backend only handles API routes starting with /v1
+    Logger.log({
+      level: 'info',
+      message: 'Backend server ready - API endpoints only (frontend served separately)'
+    })
+    const server = app.listen(ports.webUi, () => {
+      Logger.log({
+        level: 'info',
+        message: `Server started on port ${ports.webUi}`,
+        data: { environment, runId, port: ports.webUi }
+      })
+    })
+
+    // Start MQTT broker
+    MqttServer.start()
+
+    // Start AWS IoT Core HTTPS API emulation server
+    await startAwsIotServer()
+
+    // Start Pub/Sub emulation server
+    await startPubSubServer()
+
+    // Initialize default Pub/Sub topics (systemOneMinuteTick, etc.)
+    await initializeDefaultTopics()
+
+    // Start Cloud Logging emulation server
+    await startLoggingServer()
+
+    // Initialize Bull queue consumer for logs
+    const redisConfigs = []
+    redisConfigs.push({
+      host: Application.redis.host,
+      port: Application.redis.port,
+      db: Application.redis.db
+    })
+    if (
+      Application.redisLogs.host !== Application.redis.host ||
+      Application.redisLogs.port !== Application.redis.port
+    ) {
+      redisConfigs.push({
+        host: Application.redisLogs.host,
+        port: Application.redisLogs.port,
+        db: Application.redisLogs.db
+      })
+      Logger.log({
+        level: 'info',
+        message: 'Using separate Redis instance for logs',
+        data: {
+          dataRedis: `${Application.redis.host}:${Application.redis.port}`,
+          logsRedis: `${Application.redisLogs.host}:${Application.redisLogs.port}`
+        }
+      })
+    } else {
+      Logger.log({
+        level: 'info',
+        message: 'Using shared Redis instance for data and logs',
+        data: { redis: `${Application.redis.host}:${Application.redis.port}` }
+      })
+    }
+    global.logsConsumer = new BlackboxLogsConsumer(redisConfigs)
+
+    // Start Docker console log capture — all containers on goki-network
+    global.dockerLogsConsumer = new DockerLogsConsumer({
+      enabled: Application.logging?.dockerCapture !== false,
+      networkName: Application.logging?.dockerNetwork || 'goki-network',
+      ...(Application.logging?.dockerExcludedContainers && {
+        excludedContainers: Application.logging.dockerExcludedContainers
+      })
+    })
+    await global.dockerLogsConsumer.start()
+
+    // Start message history cleanup job
+    if (Application.messageHistory.enabled) {
+      startCleanupJob()
+    }
+
+    // Start database maintenance job (table trimming, WAL checkpoint, ANALYZE)
+    startMaintenanceJob()
+
+    // Auto-start App Gateway (non-blocking)
+    AppGatewayService.autoStart().catch(error => {
+      Logger.log({
+        level: 'error',
+        message: 'Gateway auto-start error',
+        data: { error: error.message }
+      })
+    })
+
+    // Initialize Cloud Functions service
+    await FunctionsService.initialize()
+    await FunctionTriggerDispatcher.start()
+
+    return server
+  } catch (error) {
+    Logger.log({
+      level: 'error',
+      message: 'Failed to start server',
+      data: { error: error.message, stack: error.stack }
+    })
+    process.exit(1)
+  }
+}
+
+const shutdown = async () => {
+  Logger.log({
+    level: 'info',
+    message: 'Shutting down server'
+  })
+  await AppGatewayService.stop()
+  FunctionTriggerDispatcher.stop()
+  await FunctionsService.shutdown()
+  MqttServer.stop()
+  if (global.logsConsumer) {
+    await global.logsConsumer.close()
+  }
+  if (global.dockerLogsConsumer) {
+    await global.dockerLogsConsumer.stop()
+  }
+  await PostgresClient.shutdown()
+  await RedisClient.shutdown()
+  SqliteStore.shutdown()
+  process.exit(0)
+}
+
+start()
+
+process.on('SIGINT', shutdown)
+process.on('SIGTERM', shutdown)
+
+process.on('unhandledRejection', error => {
+  Logger.log({
+    level: 'error',
+    message: 'Unhandled rejection detected',
+    data: { error: error.message, stack: error.stack }
+  })
+})
+
+process.on('uncaughtException', error => {
+  Logger.log({
+    level: 'error',
+    message: 'Uncaught exception detected',
+    data: { error: error.message, stack: error.stack }
+  })
+})

package/src/api/dashboard/Controllers.js

@@ -0,0 +1,9 @@
+import { Logic } from './Logic.js'
+
+export const Controllers = {
+  async getStats (ctx) {
+    const { traceId } = ctx.state
+    const result = await Logic.getStats({ traceId })
+    ctx.reply(result)
+  }
+}

package/src/api/dashboard/Logic.js

@@ -0,0 +1,76 @@
+import { SqliteStore } from '../../singletons/SqliteStore.js'
+import {
+  PUBSUB_TOPICS,
+  PUBSUB_SUBSCRIPTIONS,
+  PUBSUB_MESSAGES,
+  LOGGING_ENTRIES,
+  MQTT_CLIENTS,
+  MQTT_MESSAGES
+} from '../../db/Tables.js'
+import { Logic as FirestoreLogic } from '../firestore/Logic.js'
+
+export const Logic = {
+  async getStats (params) {
+    const { traceId } = params
+    try {
+      const topics = SqliteStore.list(PUBSUB_TOPICS)
+      const subscriptions = SqliteStore.list(PUBSUB_SUBSCRIPTIONS)
+      const messages = SqliteStore.list(PUBSUB_MESSAGES)
+      const logEntries = SqliteStore.list(LOGGING_ENTRIES)
+      const loggingByLevel = {}
+      logEntries.data.forEach(entry => {
+        const level = entry.severity || 'UNKNOWN'
+        loggingByLevel[level] = (loggingByLevel[level] || 0) + 1
+      })
+      const mqttClients = SqliteStore.list(MQTT_CLIENTS)
+      const mqttMessages = SqliteStore.list(MQTT_MESSAGES)
+      let firestoreStats = { collections: 0, documents: 0 }
+      try {
+        const collectionsResult = await FirestoreLogic.listCollections({ traceId })
+        if (collectionsResult.status !== 'error' && collectionsResult.collections) {
+          const collectionDocCounts = await Promise.all(
+            collectionsResult.collections.map(async (collectionPath) => {
+              const docsResult = await FirestoreLogic.listDocuments({ collectionPath, page: { limit: 1000 }, traceId })
+              return docsResult.status !== 'error' ? docsResult.total : 0
+            })
+          )
+          const totalDocuments = collectionDocCounts.reduce((sum, count) => sum + count, 0)
+          firestoreStats = {
+            collections: collectionsResult.total,
+            documents: totalDocuments
+          }
+        }
+      } catch (firestoreError) {
+        firestoreStats = { collections: 0, documents: 0 }
+      }
+
+      const stats = {
+        pubsub: {
+          topics: topics.total,
+          subscriptions: subscriptions.total,
+          messages: messages.total
+        },
+        logging: {
+          entries: logEntries.total,
+          byLevel: loggingByLevel
+        },
+        mqtt: {
+          clients: mqttClients.total,
+          messages: mqttMessages.total
+        },
+        firestore: firestoreStats
+      }
+      return {
+        status: 'ok',
+        data: stats,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  }
+}

package/src/api/dashboard/Router.js

@@ -0,0 +1,11 @@
+import KoaRouter from 'koa-router'
+import { Controllers } from './Controllers.js'
+
+const Router = new KoaRouter()
+const v1 = new KoaRouter({ prefix: '/v1/dashboard' })
+
+v1.post('/stats', Controllers.getStats)
+
+Router.use(v1.routes())
+
+export { Router }

package/src/api/dashboard/Schemas.js

@@ -0,0 +1,47 @@
+import { Joi } from '@gokiteam/koa'
+
+export const Schemas = {
+  stats: {
+    request: {
+      body: {}
+    },
+    responses: {
+      success: {
+        stats: Joi.object({
+          topics: Joi.number().integer().min(0).required(),
+          subscriptions: Joi.number().integer().min(0).required(),
+          messages: Joi.number().integer().min(0).required(),
+          logEntries: Joi.number().integer().min(0).required(),
+          mqttClients: Joi.number().integer().min(0).required(),
+          mqttMessages: Joi.number().integer().min(0).required()
+        }).required()
+      }
+    }
+  },
+  recentActivity: {
+    request: {
+      body: {
+        limit: Joi.number().integer().min(1).max(100).default(20)
+      }
+    },
+    responses: {
+      success: {
+        activities: Joi.array().items(
+          Joi.object({
+            type: Joi.string().valid(
+              'pubsub.message',
+              'pubsub.topic.created',
+              'pubsub.subscription.created',
+              'mqtt.publish',
+              'mqtt.connect',
+              'mqtt.disconnect',
+              'logging.entry'
+            ).required(),
+            data: Joi.object().required(),
+            timestamp: Joi.string().isoDate().required()
+          })
+        ).required()
+      }
+    }
+  }
+}

package/src/api/data/Controllers.js

@@ -0,0 +1,26 @@
+import { Logic } from './Logic.js'
+
+export const Controllers = {
+  async export (ctx) {
+    const { traceId } = ctx.state
+    const result = await Logic.export({ traceId })
+    ctx.reply(result)
+  },
+  async import (ctx) {
+    const { traceId } = ctx.state
+    const { data } = ctx.request.body
+    const result = await Logic.import({ data, traceId })
+    ctx.reply(result)
+  },
+  async clear (ctx) {
+    const { traceId } = ctx.state
+    const result = await Logic.clear({ traceId })
+    ctx.reply(result)
+  },
+  async clearServices (ctx) {
+    const { traceId } = ctx.state
+    const { services, keepSystemData } = ctx.request.body
+    const result = await Logic.clearServices({ services, keepSystemData, traceId })
+    ctx.reply(result)
+  }
+}