@gokiteam/goki-dev 0.2.0

package/src/emulation/pubsub/ShadowSubscriptionManager.js

@@ -0,0 +1,199 @@
+import { Application } from '../../configs/Application.js'
+import { Logger } from '../../singletons/Logger.js'
+
+const { pubsub } = Application
+const SHADOW_SUFFIX = '-devtools-shadow'
+
+/**
+ * Manages shadow subscriptions on the official Pub/Sub emulator.
+ * Creates shadow subscriptions for each user subscription to intercept messages.
+ */
+export class ShadowSubscriptionManager {
+  constructor () {
+    this.baseUrl = `http://${pubsub.emulatorHost}:${pubsub.emulatorPort}`
+    this.projectId = pubsub.projectId
+    this.checkIntervalMs = pubsub.shadowSubscriptionCheckIntervalMs
+    this.knownSubscriptions = new Set()
+    this.shadowSubscriptions = new Set()
+    this.shadowToTopic = new Map()
+    this.intervalId = null
+    this.isRunning = false
+  }
+
+  async request (method, path, body = null) {
+    const url = `${this.baseUrl}${path}`
+    const options = {
+      method,
+      headers: { 'Content-Type': 'application/json' }
+    }
+    if (body) {
+      options.body = JSON.stringify(body)
+    }
+    const response = await fetch(url, options)
+    const text = await response.text()
+    if (!text) return { success: response.ok, status: response.status }
+    try {
+      return JSON.parse(text)
+    } catch {
+      return { success: response.ok, status: response.status, text }
+    }
+  }
+
+  async listSubscriptions () {
+    try {
+      const result = await this.request('GET', `/v1/projects/${this.projectId}/subscriptions`)
+      const subscriptions = result.subscriptions || []
+      return subscriptions
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to list subscriptions from emulator',
+        data: { error: error.message, baseUrl: this.baseUrl }
+      })
+      return []
+    }
+  }
+
+  async listTopics () {
+    try {
+      const result = await this.request('GET', `/v1/projects/${this.projectId}/topics`)
+      return result.topics || []
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to list topics from emulator',
+        data: { error: error.message }
+      })
+      return []
+    }
+  }
+
+  async createShadowSubscription (topicName, originalSubscriptionName) {
+    const shadowSubscriptionId = this.getShadowSubscriptionId(originalSubscriptionName)
+    const shadowSubscriptionName = `projects/${this.projectId}/subscriptions/${shadowSubscriptionId}`
+    if (this.shadowSubscriptions.has(shadowSubscriptionName)) {
+      return null
+    }
+    try {
+      const result = await this.request(
+        'PUT',
+        `/v1/projects/${this.projectId}/subscriptions/${shadowSubscriptionId}`,
+        {
+          topic: topicName,
+          ackDeadlineSeconds: 60
+        }
+      )
+      if (result.name || result.success) {
+        this.shadowSubscriptions.add(shadowSubscriptionName)
+        this.shadowToTopic.set(shadowSubscriptionName, topicName)
+        Logger.log({
+          level: 'info',
+          message: 'Shadow subscription created',
+          data: { shadowSubscriptionName, forSubscription: originalSubscriptionName, topic: topicName }
+        })
+        return shadowSubscriptionName
+      }
+      if (result.error?.code === 409 || result.error?.status === 'ALREADY_EXISTS') {
+        this.shadowSubscriptions.add(shadowSubscriptionName)
+        this.shadowToTopic.set(shadowSubscriptionName, topicName)
+        return shadowSubscriptionName
+      }
+      Logger.log({
+        level: 'warn',
+        message: 'Failed to create shadow subscription',
+        data: { result, originalSubscriptionName }
+      })
+      return null
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Error creating shadow subscription',
+        data: { error: error.message, originalSubscriptionName }
+      })
+      return null
+    }
+  }
+
+  getShadowSubscriptionId (originalSubscriptionName) {
+    const parts = originalSubscriptionName.split('/')
+    const subscriptionId = parts[parts.length - 1]
+    return `${subscriptionId}${SHADOW_SUFFIX}`
+  }
+
+  isShadowSubscription (subscriptionName) {
+    return subscriptionName.includes(SHADOW_SUFFIX)
+  }
+
+  async checkForNewSubscriptions () {
+    const subscriptions = await this.listSubscriptions()
+    if (subscriptions.length > 0) {
+      Logger.log({
+        level: 'info',
+        message: 'Checking subscriptions',
+        data: { count: subscriptions.length, shadowCount: this.shadowSubscriptions.size }
+      })
+    }
+    for (const sub of subscriptions) {
+      const subName = sub.name
+      if (this.isShadowSubscription(subName)) {
+        this.shadowSubscriptions.add(subName)
+        // Store topic mapping for discovered shadow subscriptions
+        if (sub.topic && !this.shadowToTopic.has(subName)) {
+          this.shadowToTopic.set(subName, sub.topic)
+        }
+        continue
+      }
+      if (!this.knownSubscriptions.has(subName)) {
+        this.knownSubscriptions.add(subName)
+        await this.createShadowSubscription(sub.topic, subName)
+      }
+    }
+  }
+
+  async start () {
+    if (this.isRunning) return
+    this.isRunning = true
+    Logger.log({
+      level: 'info',
+      message: 'Starting Shadow Subscription Manager',
+      data: {
+        emulatorHost: `${pubsub.emulatorHost}:${pubsub.emulatorPort}`,
+        checkIntervalMs: this.checkIntervalMs
+      }
+    })
+    await this.checkForNewSubscriptions()
+    this.intervalId = setInterval(async () => {
+      try {
+        await this.checkForNewSubscriptions()
+      } catch (error) {
+        Logger.log({
+          level: 'error',
+          message: 'Error in subscription check interval',
+          data: { error: error.message }
+        })
+      }
+    }, this.checkIntervalMs)
+  }
+
+  stop () {
+    if (this.intervalId) {
+      clearInterval(this.intervalId)
+      this.intervalId = null
+    }
+    this.isRunning = false
+    Logger.log({
+      level: 'info',
+      message: 'Shadow Subscription Manager stopped'
+    })
+  }
+
+  getShadowSubscriptions () {
+    return Array.from(this.shadowSubscriptions)
+  }
+
+  getTopicForShadow (shadowSubscriptionName) {
+    return this.shadowToTopic.get(shadowSubscriptionName) || null
+  }
+}
+
+export const shadowSubscriptionManager = new ShadowSubscriptionManager()

package/src/enums/ContainerNames.js

@@ -0,0 +1,106 @@
+/**
+ * Container Names Enum
+ * Normalized identifiers for all Docker containers in the platform
+ * Use these constants instead of hardcoding container names
+ */
+
+export const ContainerNames = Object.freeze({
+  POSTGRES: 'postgres',
+  REDIS: 'redis',
+  REDIS_LOGS: 'redis-logs',
+  PUBSUB: 'pubsub',
+  FIRESTORE: 'firestore',
+  ELASTICSEARCH: 'elasticsearch',
+  KIBANA: 'kibana',
+  DEV_TOOLS_BACKEND: 'dev-tools-backend',
+  DEV_TOOLS_FRONTEND: 'dev-tools-frontend',
+  DEVICE_NATIVE: 'device-native',
+  DEVICE_SIMULATOR: 'device-simulator',
+  INTEGRATION_APALEO: 'integration-apaleo',
+  CORE_PMS: 'core-pms',
+  CORE_KEY: 'core-key',
+  INTEGRATION_MEWS: 'integration-mews',
+  INTEGRATION_OPERA: 'integration-opera'
+})
+
+export const ContainerDisplayNames = Object.freeze({
+  [ContainerNames.POSTGRES]: 'PostgreSQL',
+  [ContainerNames.REDIS]: 'Redis',
+  [ContainerNames.REDIS_LOGS]: 'Redis (Logs)',
+  [ContainerNames.PUBSUB]: 'GCP Pub/Sub Emulator',
+  [ContainerNames.FIRESTORE]: 'Firestore Emulator',
+  [ContainerNames.ELASTICSEARCH]: 'Elasticsearch',
+  [ContainerNames.KIBANA]: 'Kibana',
+  [ContainerNames.DEV_TOOLS_BACKEND]: 'Dev Tools Backend',
+  [ContainerNames.DEV_TOOLS_FRONTEND]: 'Dev Tools Frontend',
+  [ContainerNames.DEVICE_NATIVE]: 'Device Native',
+  [ContainerNames.DEVICE_SIMULATOR]: 'Device Simulator',
+  [ContainerNames.INTEGRATION_APALEO]: 'Integration Apaleo',
+  [ContainerNames.CORE_PMS]: 'Core PMS',
+  [ContainerNames.CORE_KEY]: 'Core Key',
+  [ContainerNames.INTEGRATION_MEWS]: 'Integration Mews',
+  [ContainerNames.INTEGRATION_OPERA]: 'Integration Opera'
+})
+
+export const ContainerDescriptions = Object.freeze({
+  [ContainerNames.POSTGRES]: 'PostgreSQL database for application data',
+  [ContainerNames.REDIS]: 'Redis cache and data store',
+  [ContainerNames.REDIS_LOGS]: 'Redis instance dedicated to log storage (256MB LRU)',
+  [ContainerNames.PUBSUB]: 'Google Cloud Pub/Sub emulation for messaging',
+  [ContainerNames.FIRESTORE]: 'Google Cloud Firestore NoSQL database emulation',
+  [ContainerNames.ELASTICSEARCH]: 'Elasticsearch search and analytics engine',
+  [ContainerNames.KIBANA]: 'Kibana visualization dashboard for Elasticsearch',
+  [ContainerNames.DEV_TOOLS_BACKEND]: 'Dev Tools API server and emulators',
+  [ContainerNames.DEV_TOOLS_FRONTEND]: 'Dev Tools web UI (nginx)',
+  [ContainerNames.DEVICE_NATIVE]: 'Device native communication service',
+  [ContainerNames.DEVICE_SIMULATOR]: 'Device simulator for testing',
+  [ContainerNames.INTEGRATION_APALEO]: 'Apaleo PMS integration',
+  [ContainerNames.CORE_PMS]: 'Core PMS service',
+  [ContainerNames.CORE_KEY]: 'Core key management service',
+  [ContainerNames.INTEGRATION_MEWS]: 'Mews PMS integration',
+  [ContainerNames.INTEGRATION_OPERA]: 'Opera PMS integration'
+})
+
+export const ContainerPorts = Object.freeze({
+  [ContainerNames.POSTGRES]: 5432,
+  [ContainerNames.REDIS]: 6379,
+  [ContainerNames.REDIS_LOGS]: 6380,
+  [ContainerNames.PUBSUB]: 8085,
+  [ContainerNames.FIRESTORE]: 8080,
+  [ContainerNames.ELASTICSEARCH]: 9200,
+  [ContainerNames.KIBANA]: 5601,
+  [ContainerNames.DEV_TOOLS_BACKEND]: 9000,
+  [ContainerNames.DEV_TOOLS_FRONTEND]: 9001,
+  [ContainerNames.DEVICE_NATIVE]: 3000,
+  [ContainerNames.DEVICE_SIMULATOR]: 3001,
+  [ContainerNames.INTEGRATION_APALEO]: 3002,
+  [ContainerNames.CORE_PMS]: 3003,
+  [ContainerNames.CORE_KEY]: 3004,
+  [ContainerNames.INTEGRATION_MEWS]: 3005,
+  [ContainerNames.INTEGRATION_OPERA]: 3006
+})
+
+// Default service configuration (which services are on by default)
+export const ContainerDefaults = Object.freeze({
+  [ContainerNames.POSTGRES]: false,
+  [ContainerNames.REDIS]: true,
+  [ContainerNames.REDIS_LOGS]: false,
+  [ContainerNames.PUBSUB]: true,
+  [ContainerNames.FIRESTORE]: false,
+  [ContainerNames.ELASTICSEARCH]: false,
+  [ContainerNames.KIBANA]: false
+})
+
+/**
+ * Get all container names as array
+ */
+export const getAllContainerNames = () => {
+  return Object.values(ContainerNames)
+}
+
+/**
+ * Validate container name
+ */
+export const isValidContainerName = (name) => {
+  return getAllContainerNames().includes(name)
+}

package/src/enums/ErrorReason.js

@@ -0,0 +1,28 @@
+export const ErrorReason = Object.freeze({
+  // General errors
+  validationFailed: 'validationFailed',
+  resourceNotFound: 'resourceNotFound',
+  resourceAlreadyExists: 'resourceAlreadyExists',
+  collectionNotFound: 'collectionNotFound',
+  // Pub/Sub errors
+  topicNotFound: 'topicNotFound',
+  topicAlreadyExists: 'topicAlreadyExists',
+  subscriptionNotFound: 'subscriptionNotFound',
+  subscriptionAlreadyExists: 'subscriptionAlreadyExists',
+  messageNotFound: 'messageNotFound',
+  invalidMessageData: 'invalidMessageData',
+  // Logging errors
+  logEntryNotFound: 'logEntryNotFound',
+  invalidLogEntry: 'invalidLogEntry',
+  // Service errors
+  serviceNotFound: 'serviceNotFound',
+  serviceAlreadyRunning: 'serviceAlreadyRunning',
+  serviceNotRunning: 'serviceNotRunning',
+  serviceStartFailed: 'serviceStartFailed',
+  serviceStopFailed: 'serviceStopFailed',
+  // Storage errors
+  storageError: 'storageError',
+  flushFailed: 'flushFailed'
+})
+
+export const ErrorReasonValues = Object.values(ErrorReason)

package/src/enums/FunctionStatuses.js

@@ -0,0 +1,15 @@
+/**
+ * Cloud Function Runtime Statuses
+ * Tracks the lifecycle state of a function process
+ */
+
+export const FunctionStatuses = Object.freeze({
+  STOPPED: 'stopped',
+  STARTING: 'starting',
+  RUNNING: 'running',
+  ERROR: 'error'
+})
+
+export const FunctionStatusList = Object.values(FunctionStatuses)
+
+export const isValidFunctionStatus = (status) => FunctionStatusList.includes(status)

package/src/enums/FunctionTriggerTypes.js

@@ -0,0 +1,15 @@
+/**
+ * Cloud Function Trigger Types
+ * Defines how a function is invoked
+ */
+
+export const FunctionTriggerTypes = Object.freeze({
+  HTTP: 'http',
+  PUBSUB: 'pubsub',
+  FIRESTORE: 'firestore',
+  SCHEDULER: 'scheduler'
+})
+
+export const FunctionTriggerTypeList = Object.values(FunctionTriggerTypes)
+
+export const isValidTriggerType = (type) => FunctionTriggerTypeList.includes(type)

package/src/enums/GatewayState.js

@@ -0,0 +1,7 @@
+export const GatewayState = Object.freeze({
+  STOPPED: 'stopped',
+  WAITING_FOR_DEPENDENCIES: 'waitingForDependencies',
+  STARTING: 'starting',
+  RUNNING: 'running',
+  ERROR: 'error'
+})

package/src/enums/ServiceNames.js

@@ -0,0 +1,68 @@
+/**
+ * Service Names Enum
+ * Normalized identifiers for all services in the platform
+ * Use these constants instead of hardcoding service names
+ */
+
+export const ServiceNames = Object.freeze({
+  PUBSUB: 'pubsub',
+  LOGGING: 'logging',
+  MQTT: 'mqtt',
+  FIRESTORE: 'firestore',
+  WEBUI: 'webui',
+  APP_GATEWAY: 'appGateway',
+  HTTP_PROXY: 'http',
+  WEBHOOK_PROXY: 'webhook',
+  CLOUD_FUNCTIONS: 'cloudFunctions',
+  // Docker container services (managed externally)
+  ELASTICSEARCH: 'elasticsearch',
+  KIBANA: 'kibana',
+  POSTGRES: 'postgres',
+  REDIS: 'redis'
+})
+
+export const ServiceDisplayNames = Object.freeze({
+  [ServiceNames.PUBSUB]: 'GCP Pub/Sub Emulator',
+  [ServiceNames.LOGGING]: 'Cloud Logging Emulator',
+  [ServiceNames.MQTT]: 'MQTT Broker',
+  [ServiceNames.FIRESTORE]: 'Cloud Firestore Emulator',
+  [ServiceNames.WEBUI]: 'Web UI & API Server',
+  [ServiceNames.APP_GATEWAY]: 'App Gateway',
+  [ServiceNames.HTTP_PROXY]: 'HTTP',
+  [ServiceNames.WEBHOOK_PROXY]: 'Webhook',
+  [ServiceNames.CLOUD_FUNCTIONS]: 'Cloud Functions',
+  [ServiceNames.ELASTICSEARCH]: 'Elasticsearch',
+  [ServiceNames.KIBANA]: 'Kibana',
+  [ServiceNames.POSTGRES]: 'PostgreSQL',
+  [ServiceNames.REDIS]: 'Redis'
+})
+
+export const ServiceDescriptions = Object.freeze({
+  [ServiceNames.PUBSUB]: 'Google Cloud Pub/Sub emulation for topic and subscription management',
+  [ServiceNames.LOGGING]: 'Google Cloud Logging emulation for log entry management',
+  [ServiceNames.MQTT]: 'MQTT message broker for device communication',
+  [ServiceNames.FIRESTORE]: 'Google Cloud Firestore emulation for NoSQL database',
+  [ServiceNames.WEBUI]: 'Management dashboard and REST API server',
+  [ServiceNames.APP_GATEWAY]: 'Virtual gateway for device-native to device-simulator communication',
+  [ServiceNames.HTTP_PROXY]: 'HTTP proxy for inter-service traffic monitoring',
+  [ServiceNames.WEBHOOK_PROXY]: 'Webhook proxy for external service callbacks',
+  [ServiceNames.CLOUD_FUNCTIONS]: 'Google Cloud Functions emulation for serverless function management',
+  [ServiceNames.ELASTICSEARCH]: 'Elasticsearch search and analytics engine',
+  [ServiceNames.KIBANA]: 'Kibana visualization dashboard for Elasticsearch',
+  [ServiceNames.POSTGRES]: 'PostgreSQL database for application data',
+  [ServiceNames.REDIS]: 'Redis cache and data store'
+})
+
+/**
+ * Get all service names as array
+ */
+export const getAllServiceNames = () => {
+  return Object.values(ServiceNames)
+}
+
+/**
+ * Validate service name
+ */
+export const isValidServiceName = (name) => {
+  return getAllServiceNames().includes(name)
+}

package/src/jobs/DatabaseMaintenance.js

@@ -0,0 +1,184 @@
+// Periodic database maintenance job
+// Trims unbounded tables, checkpoints WAL, and updates query planner stats
+
+import { SqliteStore } from '../singletons/SqliteStore.js'
+import { Application } from '../configs/Application.js'
+import { Logger } from '../singletons/Logger.js'
+import {
+  LOGGING_ENTRIES,
+  HTTP_TRAFFIC,
+  MQTT_MESSAGES,
+  PUBSUB_MESSAGES,
+  CLOUD_FUNCTION_INVOCATIONS
+} from '../db/Tables.js'
+
+const TABLES_TO_TRIM = [
+  { table: LOGGING_ENTRIES, column: 'created_at', configKey: 'loggingEntries' },
+  { table: HTTP_TRAFFIC, column: 'created_at', configKey: 'httpTraffic' },
+  { table: MQTT_MESSAGES, column: 'created_at', configKey: 'mqttMessages' },
+  { table: PUBSUB_MESSAGES, column: 'created_at', configKey: 'pubsubMessages' },
+  { table: CLOUD_FUNCTION_INVOCATIONS, column: 'created_at', configKey: 'functionInvocations' }
+]
+
+/**
+ * Delete rows older than retention period from a table
+ */
+const trimTable = (table, column, retentionDays) => {
+  const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000).toISOString()
+  try {
+    const result = SqliteStore.db.prepare(
+      `DELETE FROM ${table} WHERE ${column} < ?`
+    ).run(cutoff)
+    return result.changes
+  } catch (error) {
+    Logger.log({
+      level: 'warn',
+      message: `Failed to trim ${table}`,
+      data: { error: error.message }
+    })
+    return 0
+  }
+}
+
+/**
+ * Enforce max row count on logging_entries by deleting oldest rows
+ */
+const enforceLoggingLimit = (maxEntries) => {
+  try {
+    const countResult = SqliteStore.db.prepare(
+      `SELECT COUNT(*) as count FROM ${LOGGING_ENTRIES}`
+    ).get()
+    const count = countResult.count
+    if (count <= maxEntries) return 0
+    const excess = count - maxEntries
+    const result = SqliteStore.db.prepare(`
+      DELETE FROM ${LOGGING_ENTRIES}
+      WHERE _id IN (
+        SELECT _id FROM ${LOGGING_ENTRIES}
+        ORDER BY created_at ASC
+        LIMIT ?
+      )
+    `).run(excess)
+    return result.changes
+  } catch (error) {
+    Logger.log({
+      level: 'warn',
+      message: 'Failed to enforce logging entry limit',
+      data: { error: error.message }
+    })
+    return 0
+  }
+}
+
+/**
+ * Run full database maintenance cycle
+ */
+export const runDatabaseMaintenance = () => {
+  const { retentionDays, maxLoggingEntries } = Application.dbMaintenance
+  const results = {}
+
+  // Trim tables by retention period
+  for (const { table, column, configKey } of TABLES_TO_TRIM) {
+    const days = retentionDays[configKey]
+    const deleted = trimTable(table, column, days)
+    if (deleted > 0) {
+      results[table] = { deleted, retentionDays: days }
+    }
+  }
+
+  // Enforce logging entry count limit
+  const loggingTrimmed = enforceLoggingLimit(maxLoggingEntries)
+  if (loggingTrimmed > 0) {
+    results[`${LOGGING_ENTRIES}_limit`] = { deleted: loggingTrimmed, maxEntries: maxLoggingEntries }
+  }
+
+  // WAL checkpoint
+  try {
+    SqliteStore.db.pragma('wal_checkpoint(PASSIVE)')
+  } catch (error) {
+    Logger.log({
+      level: 'warn',
+      message: 'WAL checkpoint failed',
+      data: { error: error.message }
+    })
+  }
+
+  // Update query planner stats
+  try {
+    SqliteStore.db.exec('ANALYZE')
+  } catch (error) {
+    Logger.log({
+      level: 'warn',
+      message: 'ANALYZE failed',
+      data: { error: error.message }
+    })
+  }
+
+  const totalDeleted = Object.values(results).reduce((sum, r) => sum + r.deleted, 0)
+  if (totalDeleted > 0) {
+    Logger.log({
+      level: 'info',
+      message: 'Database maintenance completed',
+      data: { totalDeleted, tables: results }
+    })
+  } else {
+    Logger.log({
+      level: 'debug',
+      message: 'Database maintenance completed - nothing to trim'
+    })
+  }
+
+  return results
+}
+
+/**
+ * Start the scheduled maintenance job
+ */
+export const startMaintenanceJob = () => {
+  const { intervalHours } = Application.dbMaintenance
+  const intervalMs = intervalHours * 60 * 60 * 1000
+
+  Logger.log({
+    level: 'info',
+    message: 'Starting database maintenance job',
+    data: {
+      intervalHours,
+      retentionDays: Application.dbMaintenance.retentionDays,
+      maxLoggingEntries: Application.dbMaintenance.maxLoggingEntries
+    }
+  })
+
+  // Delay first run by 30 seconds to let server fully start
+  setTimeout(() => {
+    try {
+      runDatabaseMaintenance()
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Initial database maintenance failed',
+        data: { error: error.message }
+      })
+    }
+  }, 30000)
+
+  return setInterval(() => {
+    try {
+      runDatabaseMaintenance()
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Scheduled database maintenance failed',
+        data: { error: error.message }
+      })
+    }
+  }, intervalMs)
+}
+
+/**
+ * Stop the maintenance job
+ */
+export const stopMaintenanceJob = (timer) => {
+  if (timer) {
+    clearInterval(timer)
+  }
+}