@gokiteam/goki-dev 0.2.0

package/src/api/data/Logic.js

@@ -0,0 +1,188 @@
+import { SqliteStore } from '../../singletons/SqliteStore.js'
+import { Application } from '../../configs/Application.js'
+import { RedisClient } from '../../singletons/RedisClient.js'
+import {
+  PUBSUB_TOPICS,
+  PUBSUB_SUBSCRIPTIONS,
+  PUBSUB_MESSAGES,
+  PUBSUB_MESSAGE_HISTORY,
+  LOGGING_ENTRIES,
+  MQTT_CLIENTS,
+  MQTT_MESSAGES,
+  FIRESTORE_METADATA
+} from '../../db/Tables.js'
+
+const FIRESTORE_API = `http://localhost:${Application.ports.firestore}`
+const PROJECT_ID = Application.firestore.projectId
+const DATABASE = '(default)'
+
+async function clearFirestoreEmulator () {
+  try {
+    // Use Firebase emulator's clear endpoint to delete all data
+    const clearUrl = `${FIRESTORE_API}/emulator/v1/projects/${PROJECT_ID}/databases/${DATABASE}/documents`
+    await fetch(clearUrl, { method: 'DELETE' })
+  } catch {
+    // Ignore errors when clearing Firestore (emulator might not be running)
+  }
+}
+
+export const Logic = {
+  async export (params) {
+    const { traceId } = params
+    const topics = SqliteStore.list(PUBSUB_TOPICS)
+    const subscriptions = SqliteStore.list(PUBSUB_SUBSCRIPTIONS)
+    const messages = SqliteStore.list(PUBSUB_MESSAGES)
+    const logEntries = SqliteStore.list(LOGGING_ENTRIES)
+    const mqttClients = SqliteStore.list(MQTT_CLIENTS)
+    const mqttMessages = SqliteStore.list(MQTT_MESSAGES)
+    return {
+      data: {
+        pubsub: {
+          topics: topics.data,
+          subscriptions: subscriptions.data,
+          messages: messages.data
+        },
+        logging: {
+          entries: logEntries.data
+        },
+        mqtt: {
+          clients: mqttClients.data,
+          messages: mqttMessages.data
+        },
+        exportedAt: new Date().toISOString()
+      },
+      traceId
+    }
+  },
+  async import (params) {
+    const { data, traceId } = params
+    let imported = {
+      topics: 0,
+      subscriptions: 0,
+      messages: 0,
+      logEntries: 0,
+      mqttClients: 0,
+      mqttMessages: 0
+    }
+    if (data.pubsub) {
+      if (data.pubsub.topics) {
+        for (const topic of data.pubsub.topics) {
+          SqliteStore.create(PUBSUB_TOPICS, topic)
+          imported.topics++
+        }
+      }
+      if (data.pubsub.subscriptions) {
+        for (const subscription of data.pubsub.subscriptions) {
+          SqliteStore.create(PUBSUB_SUBSCRIPTIONS, subscription)
+          imported.subscriptions++
+        }
+      }
+      if (data.pubsub.messages) {
+        for (const message of data.pubsub.messages) {
+          SqliteStore.create(PUBSUB_MESSAGES, message)
+          imported.messages++
+        }
+      }
+    }
+    if (data.logging && data.logging.entries) {
+      for (const entry of data.logging.entries) {
+        SqliteStore.create(LOGGING_ENTRIES, entry)
+        imported.logEntries++
+      }
+    }
+    if (data.mqtt) {
+      if (data.mqtt.clients) {
+        for (const client of data.mqtt.clients) {
+          SqliteStore.create(MQTT_CLIENTS, client)
+          imported.mqttClients++
+        }
+      }
+      if (data.mqtt.messages) {
+        for (const message of data.mqtt.messages) {
+          SqliteStore.create(MQTT_MESSAGES, message)
+          imported.mqttMessages++
+        }
+      }
+    }
+    SqliteStore.flushAll()
+    return {
+      message: 'Data imported successfully',
+      imported,
+      traceId
+    }
+  },
+  async clear (params) {
+    const { traceId } = params
+    SqliteStore.clear(PUBSUB_TOPICS)
+    SqliteStore.clear(PUBSUB_SUBSCRIPTIONS)
+    SqliteStore.clear(PUBSUB_MESSAGES)
+    SqliteStore.clear(LOGGING_ENTRIES)
+    SqliteStore.clear(MQTT_CLIENTS)
+    SqliteStore.clear(MQTT_MESSAGES)
+    // Clear Firestore emulator data
+    await clearFirestoreEmulator()
+    SqliteStore.clear(FIRESTORE_METADATA)
+    return {
+      message: 'All data cleared successfully',
+      traceId
+    }
+  },
+
+  async clearServices (params) {
+    const { services, keepSystemData = true, traceId } = params
+    const cleared = {}
+    try {
+      for (const service of services) {
+        if (service === 'pubsub') {
+          if (keepSystemData) {
+            const systemTopics = ['systemOneMinuteTick']
+            const allMessages = SqliteStore.list(PUBSUB_MESSAGE_HISTORY).data
+            const toDelete = allMessages.filter(msg => !systemTopics.includes(msg.topic))
+            for (const msg of toDelete) {
+              SqliteStore.delete(PUBSUB_MESSAGE_HISTORY, msg.id)
+            }
+            cleared.pubsub = toDelete.length
+          } else {
+            SqliteStore.clear(PUBSUB_MESSAGE_HISTORY)
+            cleared.pubsub = 'all'
+          }
+        } else if (service === 'logging') {
+          if (keepSystemData) {
+            const systemServices = ['dev-tools']
+            const allLogs = SqliteStore.list(LOGGING_ENTRIES).data
+            const toDelete = allLogs.filter(log => !systemServices.includes(log.service_name))
+            for (const log of toDelete) {
+              SqliteStore.delete(LOGGING_ENTRIES, log.id)
+            }
+            cleared.logging = toDelete.length
+          } else {
+            SqliteStore.clear(LOGGING_ENTRIES)
+            SqliteStore.clear('logging-logs')
+            cleared.logging = 'all'
+          }
+        } else if (service === 'redis') {
+          await RedisClient.connect()
+          await RedisClient.deleteAll()
+          cleared.redis = 'all'
+        } else if (service === 'postgres') {
+          cleared.postgres = 'not implemented'
+        } else if (service === 'firestore') {
+          cleared.firestore = 'not implemented'
+        } else if (service === 'mqtt') {
+          cleared.mqtt = 'not applicable'
+        }
+      }
+      return {
+        cleared,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        cleared,
+        traceId
+      }
+    }
+  }
+}

package/src/api/data/Router.js

@@ -0,0 +1,16 @@
+import KoaRouter from 'koa-router'
+import { Controllers } from './Controllers.js'
+
+const Router = new KoaRouter()
+const v1 = new KoaRouter({ prefix: '/v1/data' })
+
+v1.post('/export', Controllers.export)
+v1.post('/import', Controllers.import)
+v1.post('/clear', Controllers.clear)
+
+// Testing Helpers
+v1.post('/clear-services', Controllers.clearServices)
+
+Router.use(v1.routes())
+
+export { Router }

package/src/api/docker/Controllers.js

@@ -0,0 +1,33 @@
+import { Logic } from './Logic.js'
+
+export const Controllers = {
+  async list (ctx) {
+    const { traceId } = ctx.state
+    const result = await Logic.listContainers({ traceId })
+    ctx.reply(result)
+  },
+  async start (ctx) {
+    const { traceId } = ctx.state
+    const { containerName } = ctx.request.body
+    const result = await Logic.startContainer({ containerName, traceId })
+    ctx.reply(result)
+  },
+  async stop (ctx) {
+    const { traceId } = ctx.state
+    const { containerName } = ctx.request.body
+    const result = await Logic.stopContainer({ containerName, traceId })
+    ctx.reply(result)
+  },
+  async restart (ctx) {
+    const { traceId } = ctx.state
+    const { containerName } = ctx.request.body
+    const result = await Logic.restartContainer({ containerName, traceId })
+    ctx.reply(result)
+  },
+  async logs (ctx) {
+    const { traceId } = ctx.state
+    const { containerName, lines } = ctx.request.body
+    const result = await Logic.getContainerLogs({ containerName, lines, traceId })
+    ctx.reply(result)
+  }
+}

package/src/api/docker/Logic.js

@@ -0,0 +1,268 @@
+import http from 'http'
+import { Logger } from '../../singletons/Logger.js'
+import { ContainerNames, ContainerDisplayNames, ContainerDescriptions, ContainerPorts } from '../../enums/ContainerNames.js'
+
+const DOCKER_SOCKET = '/var/run/docker.sock'
+const DOCKER_API_VERSION = 'v1.44'
+
+// Map docker container names to our service identifiers
+const CONTAINER_NAME_MAP = {
+  'goki-postgres': ContainerNames.POSTGRES,
+  'goki-redis': ContainerNames.REDIS,
+  'goki-redis-logs': ContainerNames.REDIS_LOGS,
+  'goki-pubsub-emulator': ContainerNames.PUBSUB,
+  'goki-firestore-emulator': ContainerNames.FIRESTORE,
+  'goki-elasticsearch': ContainerNames.ELASTICSEARCH,
+  'goki-kibana': ContainerNames.KIBANA,
+  'goki-dev-tools-backend': ContainerNames.DEV_TOOLS_BACKEND,
+  'goki-dev-tools-frontend': ContainerNames.DEV_TOOLS_FRONTEND,
+  'device-native-app': ContainerNames.DEVICE_NATIVE,
+  'device-simulator-app': ContainerNames.DEVICE_SIMULATOR,
+  'integration-apaleo-app': ContainerNames.INTEGRATION_APALEO,
+  'core-pms-app': ContainerNames.CORE_PMS,
+  'core-key-app': ContainerNames.CORE_KEY,
+  'integration-mews-app': ContainerNames.INTEGRATION_MEWS,
+  'integration-opera-app': ContainerNames.INTEGRATION_OPERA
+}
+
+// Reverse map for looking up docker names
+const SERVICE_TO_CONTAINER = Object.fromEntries(
+  Object.entries(CONTAINER_NAME_MAP).map(([k, v]) => [v, k])
+)
+
+const dockerRequest = (method, path, body) => {
+  return new Promise((resolve, reject) => {
+    const options = {
+      socketPath: DOCKER_SOCKET,
+      path: `/${DOCKER_API_VERSION}${path}`,
+      method,
+      headers: { 'Content-Type': 'application/json' }
+    }
+    const req = http.request(options, (res) => {
+      let data = ''
+      res.on('data', (chunk) => { data += chunk })
+      res.on('end', () => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          resolve(data ? JSON.parse(data) : null)
+        } else if (res.statusCode === 304) {
+          resolve(null)
+        } else {
+          const error = new Error(data ? JSON.parse(data).message : `Docker API returned ${res.statusCode}`)
+          error.statusCode = res.statusCode
+          reject(error)
+        }
+      })
+    })
+    req.on('error', reject)
+    if (body) req.write(JSON.stringify(body))
+    req.end()
+  })
+}
+
+const dockerRequestRaw = (method, path) => {
+  return new Promise((resolve, reject) => {
+    const options = {
+      socketPath: DOCKER_SOCKET,
+      path: `/${DOCKER_API_VERSION}${path}`,
+      method
+    }
+    const req = http.request(options, (res) => {
+      const chunks = []
+      res.on('data', (chunk) => chunks.push(chunk))
+      res.on('end', () => {
+        const data = Buffer.concat(chunks).toString('utf8')
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          resolve(data)
+        } else {
+          reject(new Error(data || `Docker API returned ${res.statusCode}`))
+        }
+      })
+    })
+    req.on('error', reject)
+    req.end()
+  })
+}
+
+const parseContainerStatus = (state) => {
+  if (!state) return 'unknown'
+  const lower = state.toLowerCase()
+  if (lower === 'running') return 'running'
+  if (lower === 'exited') return 'stopped'
+  if (lower === 'created') return 'created'
+  if (lower === 'restarting') return 'restarting'
+  if (lower === 'paused') return 'paused'
+  return 'unknown'
+}
+
+const parseUptime = (status) => {
+  if (!status) return null
+  const match = status.match(/Up\s+(.+?)(?:\s+\(|$)/i)
+  if (!match) return null
+  return match[1].trim()
+}
+
+const parsePublicPort = (ports) => {
+  if (!ports || !ports.length) return null
+  const publicPort = ports.find(p => p.PublicPort)
+  return publicPort ? publicPort.PublicPort : null
+}
+
+const generateDisplayName = (containerName) => {
+  return containerName
+    .replace(/-app$/, '')
+    .replace(/^goki-/, '')
+    .split('-')
+    .map(word => word.charAt(0).toUpperCase() + word.slice(1))
+    .join(' ')
+}
+
+export const Logic = {
+  async listContainers (params) {
+    const { traceId } = params
+    try {
+      const filters = JSON.stringify({ network: ['goki-network'] })
+      const rawContainers = await dockerRequest('GET', `/containers/json?all=true&filters=${encodeURIComponent(filters)}`)
+      const containers = rawContainers.map(c => {
+        const name = c.Names[0].replace(/^\//, '')
+        const serviceId = CONTAINER_NAME_MAP[name] || name
+        const isKnown = !!CONTAINER_NAME_MAP[name]
+        const isSelf = serviceId === ContainerNames.DEV_TOOLS_BACKEND
+        return {
+          name: serviceId,
+          containerName: name,
+          displayName: ContainerDisplayNames[serviceId] || generateDisplayName(name),
+          description: ContainerDescriptions[serviceId] || '',
+          status: parseContainerStatus(c.State),
+          statusText: c.Status,
+          uptime: parseUptime(c.Status),
+          ports: ContainerPorts[serviceId] || parsePublicPort(c.Ports),
+          image: c.Image,
+          category: isKnown ? 'infrastructure' : 'microservice',
+          capabilities: {
+            canStart: !isSelf,
+            canStop: !isSelf,
+            canRestart: !isSelf
+          }
+        }
+      })
+      containers.sort((a, b) => a.displayName.localeCompare(b.displayName))
+      return {
+        data: containers,
+        traceId
+      }
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to list Docker containers',
+        data: { error: error.message, traceId }
+      })
+      return {
+        data: [],
+        message: 'Failed to list containers: ' + error.message,
+        traceId
+      }
+    }
+  },
+  async startContainer (params) {
+    const { containerName, traceId } = params
+    const dockerName = SERVICE_TO_CONTAINER[containerName] || containerName
+    try {
+      await dockerRequest('POST', `/containers/${dockerName}/start`)
+      Logger.log({
+        level: 'info',
+        message: 'Container started',
+        data: { containerName: dockerName, traceId }
+      })
+      return {
+        message: `${ContainerDisplayNames[containerName] || containerName} started successfully`,
+        traceId
+      }
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to start container',
+        data: { containerName: dockerName, error: error.message, traceId }
+      })
+      return {
+        message: `Failed to start ${containerName}: ${error.message}`,
+        traceId
+      }
+    }
+  },
+  async stopContainer (params) {
+    const { containerName, traceId } = params
+    const dockerName = SERVICE_TO_CONTAINER[containerName] || containerName
+    try {
+      await dockerRequest('POST', `/containers/${dockerName}/stop`)
+      Logger.log({
+        level: 'info',
+        message: 'Container stopped',
+        data: { containerName: dockerName, traceId }
+      })
+      return {
+        message: `${ContainerDisplayNames[containerName] || containerName} stopped successfully`,
+        traceId
+      }
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to stop container',
+        data: { containerName: dockerName, error: error.message, traceId }
+      })
+      return {
+        message: `Failed to stop ${containerName}: ${error.message}`,
+        traceId
+      }
+    }
+  },
+  async restartContainer (params) {
+    const { containerName, traceId } = params
+    const dockerName = SERVICE_TO_CONTAINER[containerName] || containerName
+    try {
+      await dockerRequest('POST', `/containers/${dockerName}/restart`)
+      Logger.log({
+        level: 'info',
+        message: 'Container restarted',
+        data: { containerName: dockerName, traceId }
+      })
+      return {
+        message: `${ContainerDisplayNames[containerName] || containerName} restarted successfully`,
+        traceId
+      }
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to restart container',
+        data: { containerName: dockerName, error: error.message, traceId }
+      })
+      return {
+        message: `Failed to restart ${containerName}: ${error.message}`,
+        traceId
+      }
+    }
+  },
+  async getContainerLogs (params) {
+    const { containerName, lines = 100, traceId } = params
+    const dockerName = SERVICE_TO_CONTAINER[containerName] || containerName
+    try {
+      const raw = await dockerRequestRaw('GET', `/containers/${dockerName}/logs?stdout=true&stderr=true&tail=${lines}`)
+      // Docker log stream has 8-byte header per frame, strip them
+      const cleaned = raw.replace(/[\x00-\x08]/g, '').replace(/\r/g, '')
+      return {
+        data: cleaned,
+        traceId
+      }
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to get container logs',
+        data: { containerName: dockerName, error: error.message, traceId }
+      })
+      return {
+        data: '',
+        message: `Failed to get logs: ${error.message}`,
+        traceId
+      }
+    }
+  }
+}

package/src/api/docker/Router.js

@@ -0,0 +1,15 @@
+import KoaRouter from 'koa-router'
+import { Controllers } from './Controllers.js'
+
+const Router = new KoaRouter()
+const v1 = new KoaRouter({ prefix: '/v1/docker' })
+
+v1.post('/containers/list', Controllers.list)
+v1.post('/containers/start', Controllers.start)
+v1.post('/containers/stop', Controllers.stop)
+v1.post('/containers/restart', Controllers.restart)
+v1.post('/containers/logs', Controllers.logs)
+
+Router.use(v1.routes())
+
+export { Router }

package/src/api/docker/Schemas.js

@@ -0,0 +1,80 @@
+import { Joi } from '@gokiteam/koa'
+
+export const Schemas = {
+  list: {
+    request: {
+      body: {}
+    },
+    responses: {
+      success: {
+        data: Joi.array().items(
+          Joi.object({
+            name: Joi.string().required(),
+            containerName: Joi.string().required(),
+            displayName: Joi.string().required(),
+            description: Joi.string().allow('').required(),
+            status: Joi.string().valid('running', 'stopped', 'created', 'restarting', 'paused', 'unknown').required(),
+            statusText: Joi.string().required(),
+            uptime: Joi.string().allow(null).required(),
+            ports: Joi.number().integer().allow(null).required(),
+            image: Joi.string().required(),
+            capabilities: Joi.object({
+              canStart: Joi.boolean().required(),
+              canStop: Joi.boolean().required(),
+              canRestart: Joi.boolean().required()
+            }).required()
+          })
+        ).required()
+      }
+    }
+  },
+  start: {
+    request: {
+      body: {
+        containerName: Joi.string().required()
+      }
+    },
+    responses: {
+      success: {
+        message: Joi.string().required()
+      }
+    }
+  },
+  stop: {
+    request: {
+      body: {
+        containerName: Joi.string().required()
+      }
+    },
+    responses: {
+      success: {
+        message: Joi.string().required()
+      }
+    }
+  },
+  restart: {
+    request: {
+      body: {
+        containerName: Joi.string().required()
+      }
+    },
+    responses: {
+      success: {
+        message: Joi.string().required()
+      }
+    }
+  },
+  logs: {
+    request: {
+      body: {
+        containerName: Joi.string().required(),
+        lines: Joi.number().integer().min(1).max(1000).default(100)
+      }
+    },
+    responses: {
+      success: {
+        data: Joi.string().required()
+      }
+    }
+  }
+}

package/src/api/docs/Controllers.js

@@ -0,0 +1,15 @@
+import { Logic } from './Logic.js'
+
+export const Controllers = {
+  async resolve (ctx) {
+    const docPath = ctx.params[0] || ''
+    const result = Logic.resolve(docPath)
+    if (!result) {
+      ctx.status = 404
+      ctx.body = 'Not found'
+      return
+    }
+    ctx.type = result.contentType
+    ctx.body = result.content
+  }
+}

package/src/api/docs/Logic.js

@@ -0,0 +1,85 @@
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+const projectRoot = path.resolve(__dirname, '../../../')
+const docsRoot = path.join(projectRoot, 'docs')
+
+const ALLOWED_EXTENSIONS = ['.md', '.js', '.ts', '.json', '.txt']
+
+const CONTENT_TYPES = {
+  '.md': 'text/markdown; charset=utf-8',
+  '.js': 'text/javascript; charset=utf-8',
+  '.ts': 'text/typescript; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.txt': 'text/plain; charset=utf-8'
+}
+
+const isAllowedFile = (filename) => {
+  const ext = path.extname(filename).toLowerCase()
+  return ALLOWED_EXTENSIONS.includes(ext)
+}
+
+const getContentType = (filename) => {
+  const ext = path.extname(filename).toLowerCase()
+  return CONTENT_TYPES[ext] || 'text/plain; charset=utf-8'
+}
+
+const resolveSafe = (requestedPath) => {
+  const resolved = path.resolve(docsRoot, requestedPath)
+  if (!resolved.startsWith(docsRoot)) return null
+  if (resolved.includes('node_modules')) return null
+  return resolved
+}
+
+const toApiPath = (fsPath) => {
+  const relative = path.relative(docsRoot, fsPath)
+  return `/v1/docs/${relative}`.replace(/\/+/g, '/')
+}
+
+const generateDirectoryListing = (absDir, dirName) => {
+  const entries = fs.readdirSync(absDir, { withFileTypes: true })
+  const dirs = []
+  const files = []
+  for (const entry of entries) {
+    if (entry.name.startsWith('.')) continue
+    if (entry.name === 'node_modules') continue
+    if (entry.name === 'index.md') continue
+    const fullPath = path.join(absDir, entry.name)
+    if (entry.isDirectory()) {
+      dirs.push(`- [${entry.name}/](${toApiPath(fullPath)}/)`)
+    } else if (isAllowedFile(entry.name)) {
+      files.push(`- [${entry.name}](${toApiPath(fullPath)})`)
+    }
+  }
+  dirs.sort()
+  files.sort()
+  const title = dirName || 'docs'
+  const lines = [`# ${title}\n`]
+  if (dirs.length) lines.push('## Directories\n', ...dirs, '')
+  if (files.length) lines.push('## Files\n', ...files, '')
+  return lines.join('\n')
+}
+
+export const Logic = {
+  resolve (docPath) {
+    const cleaned = (docPath || '').replace(/^\/+/, '').replace(/\/+$/, '')
+    const absPath = cleaned ? resolveSafe(cleaned) : docsRoot
+    if (!absPath || !fs.existsSync(absPath)) return null
+    const stat = fs.statSync(absPath)
+    if (stat.isDirectory()) {
+      const indexPath = path.join(absPath, 'index.md')
+      const content = fs.existsSync(indexPath)
+        ? fs.readFileSync(indexPath, 'utf-8')
+        : generateDirectoryListing(absPath, cleaned.split('/').pop())
+      return { contentType: 'text/markdown; charset=utf-8', content }
+    }
+    if (stat.isFile()) {
+      if (!isAllowedFile(absPath)) return null
+      return { contentType: getContentType(absPath), content: fs.readFileSync(absPath, 'utf-8') }
+    }
+    return null
+  }
+}