@gokiteam/goki-dev 0.2.0

package/src/api/snapshots/Router.js

@@ -0,0 +1,15 @@
+import KoaRouter from 'koa-router'
+import { Controllers } from './Controllers.js'
+
+const Router = new KoaRouter()
+const v1 = new KoaRouter({ prefix: '/v1/snapshots' })
+
+v1.post('/create', Controllers.createSnapshot)
+v1.post('/restore', Controllers.restoreSnapshot)
+v1.post('/list', Controllers.listSnapshots)
+v1.post('/details', Controllers.getSnapshotDetails)
+v1.post('/delete', Controllers.deleteSnapshot)
+
+Router.use(v1.routes())
+
+export { Router }

package/src/api/snapshots/Schemas.js

@@ -0,0 +1,23 @@
+import Joi from 'joi'
+
+export const Schemas = {
+  create: Joi.object({
+    services: Joi.array().items(Joi.string().valid('postgres', 'firestore', 'sqlite')).optional(),
+    metadata: Joi.object().optional()
+  }),
+
+  restore: Joi.object({
+    snapshotId: Joi.string().required(),
+    restartServices: Joi.boolean().default(true)
+  }),
+
+  delete: Joi.object({
+    snapshotId: Joi.string().optional()
+  }),
+
+  details: Joi.object({
+    snapshotId: Joi.string().required()
+  }),
+
+  list: Joi.object({})
+}

package/src/api/webhooks/Controllers.js

@@ -0,0 +1,49 @@
+import { Logic } from './Logic.js'
+
+export const Controllers = {
+  async register (ctx) {
+    const { traceId } = ctx.state
+    const { prefix, target, pathRewrite, description } = ctx.request.body
+    const result = await Logic.register({ prefix, target, pathRewrite, description, traceId })
+    ctx.reply(result)
+  },
+
+  async list (ctx) {
+    const { traceId } = ctx.state
+    const result = await Logic.list({ traceId })
+    ctx.reply(result)
+  },
+
+  async remove (ctx) {
+    const { traceId } = ctx.state
+    const { prefix } = ctx.request.body
+    const result = await Logic.remove({ prefix, traceId })
+    ctx.reply(result)
+  },
+
+  async test (ctx) {
+    const { traceId } = ctx.state
+    const { prefix } = ctx.request.body
+    const result = await Logic.test({ prefix, traceId })
+    ctx.reply(result)
+  },
+
+  async tunnelStatus (ctx) {
+    const { traceId } = ctx.state
+    const result = await Logic.tunnelStatus({ traceId })
+    ctx.reply(result)
+  },
+
+  async settingsGet (ctx) {
+    const { traceId } = ctx.state
+    const result = Logic.settingsGet({ traceId })
+    ctx.reply(result)
+  },
+
+  async settingsUpdate (ctx) {
+    const { traceId } = ctx.state
+    const { settings } = ctx.request.body
+    const result = Logic.settingsUpdate({ settings, traceId })
+    ctx.reply(result)
+  }
+}

package/src/api/webhooks/Logic.js

@@ -0,0 +1,137 @@
+import { WebhookProxy } from '../../singletons/WebhookProxy.js'
+
+export const Logic = {
+  async register (params) {
+    const { prefix, target, pathRewrite, description, traceId } = params
+    if (!prefix || !target) {
+      return {
+        success: false,
+        message: 'prefix and target are required',
+        traceId
+      }
+    }
+    WebhookProxy.register({ prefix, target, pathRewrite, description })
+    return {
+      route: { prefix, target, pathRewrite, description },
+      proxyUrl: `/v1/webhooks/proxy/${prefix}`,
+      message: `Webhook route registered: /v1/webhooks/proxy/${prefix} -> ${target}${pathRewrite || ''}`,
+      traceId
+    }
+  },
+
+  async list (params) {
+    const { traceId } = params
+    const { data, total } = WebhookProxy.listRoutes()
+    const ngrokDomain = WebhookProxy.getSetting('ngrok_domain')
+    const routes = data.map(route => ({
+      ...route,
+      proxyUrl: `/v1/webhooks/proxy/${route.prefix}`,
+      ...(ngrokDomain && {
+        webhookUrl: `https://${ngrokDomain}/v1/webhooks/proxy/${route.prefix}`
+      })
+    }))
+    return { routes, total, traceId }
+  },
+
+  async remove (params) {
+    const { prefix, traceId } = params
+    if (!prefix) {
+      return { success: false, message: 'prefix is required', traceId }
+    }
+    WebhookProxy.remove(prefix)
+    return { message: `Webhook route '${prefix}' removed`, traceId }
+  },
+
+  async test (params) {
+    const { prefix, traceId } = params
+    if (!prefix) {
+      return { success: false, message: 'prefix is required', traceId }
+    }
+    const route = WebhookProxy.getRoute(prefix)
+    if (!route) {
+      return { reachable: false, message: 'Route not found', traceId }
+    }
+    try {
+      const testUrl = route.pathRewrite
+        ? route.target + route.pathRewrite
+        : route.target
+      const response = await fetch(testUrl, {
+        method: 'GET',
+        signal: AbortSignal.timeout(5000)
+      })
+      return {
+        reachable: true,
+        statusCode: response.status,
+        message: `Target responded with ${response.status}`,
+        traceId
+      }
+    } catch (error) {
+      return {
+        reachable: false,
+        message: `Cannot reach target: ${error.message}`,
+        traceId
+      }
+    }
+  },
+
+  async tunnelStatus (params) {
+    const { traceId } = params
+    const ngrokDomain = WebhookProxy.getSetting('ngrok_domain')
+    const { data } = WebhookProxy.listRoutes()
+    // Check if ngrok tunnel is live by pinging the public domain directly
+    // This avoids Docker networking issues with ngrok's local API (port 4040)
+    let running = false
+    let tunnelUrl = null
+    if (ngrokDomain) {
+      tunnelUrl = `https://${ngrokDomain}`
+      try {
+        const response = await fetch(`${tunnelUrl}/v1/webhooks/list`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({}),
+          signal: AbortSignal.timeout(5000)
+        })
+        running = response.ok
+      } catch {
+        running = false
+      }
+      if (!running) tunnelUrl = null
+    }
+    const routes = data.map(route => ({
+      prefix: route.prefix,
+      proxyPath: `/v1/webhooks/proxy/${route.prefix}`,
+      ...(tunnelUrl && {
+        webhookUrl: `${tunnelUrl}/v1/webhooks/proxy/${route.prefix}`
+      })
+    }))
+    return {
+      running,
+      url: tunnelUrl,
+      domain: ngrokDomain,
+      webhookBaseUrl: tunnelUrl ? `${tunnelUrl}/v1/webhooks/proxy` : null,
+      routes,
+      traceId
+    }
+  },
+
+  settingsGet (params) {
+    const { traceId } = params
+    const settings = WebhookProxy.getSettings()
+    return { settings, traceId }
+  },
+
+  settingsUpdate (params) {
+    const { settings, traceId } = params
+    if (!settings || typeof settings !== 'object') {
+      return { success: false, message: 'settings object is required', traceId }
+    }
+    for (const [key, value] of Object.entries(settings)) {
+      WebhookProxy.setSetting(key, value)
+    }
+    return {
+      message: `Updated ${Object.keys(settings).length} setting(s)`,
+      settings: WebhookProxy.getSettings(),
+      traceId
+    }
+  }
+}

package/src/api/webhooks/Router.js

@@ -0,0 +1,12 @@
+import KoaRouter from 'koa-router'
+import { Controllers } from './Controllers.js'
+
+export const Router = new KoaRouter({ prefix: '/v1/webhooks' })
+
+Router.post('/register', Controllers.register)
+Router.post('/list', Controllers.list)
+Router.post('/remove', Controllers.remove)
+Router.post('/test', Controllers.test)
+Router.post('/tunnel/status', Controllers.tunnelStatus)
+Router.post('/settings/get', Controllers.settingsGet)
+Router.post('/settings/update', Controllers.settingsUpdate)

package/src/api/webhooks/Schemas.js

@@ -0,0 +1,31 @@
+import Joi from 'joi'
+
+export const Schemas = {
+  register: {
+    body: Joi.object({
+      prefix: Joi.string().pattern(/^[a-z0-9-]+$/).required()
+        .description('URL-safe prefix for the webhook route'),
+      target: Joi.string().uri({ scheme: ['http', 'https'] }).required()
+        .description('Target URL to forward requests to'),
+      pathRewrite: Joi.string().optional()
+        .description('Path prefix on the target (e.g., /v1/webhooks)'),
+      description: Joi.string().optional()
+        .description('Human-readable description')
+    })
+  },
+  remove: {
+    body: Joi.object({
+      prefix: Joi.string().required()
+    })
+  },
+  test: {
+    body: Joi.object({
+      prefix: Joi.string().required()
+    })
+  },
+  settingsUpdate: {
+    body: Joi.object({
+      settings: Joi.object().required()
+    })
+  }
+}

package/src/configs/Application.js

@@ -0,0 +1,147 @@
+import Moment from 'moment'
+
+const {
+  NODE_ENV = 'development',
+  LOG_LEVEL = 'info',
+  WEB_UI_PORT,
+  PUBSUB_EMULATOR_HOST,
+  PUBSUB_PORT,
+  PUBSUB_PROJECT_ID = 'tipi-development',
+  SHADOW_POLL_INTERVAL_MS = '200',
+  SHADOW_SUBSCRIPTION_CHECK_INTERVAL_MS = '5000',
+  LOGGING_PORT,
+  MQTT_PORT,
+  AWS_IOT_PORT,
+  FIRESTORE_PORT,
+  FIRESTORE_EMULATOR_HOST,
+  FIRESTORE_PROJECT_ID,
+  FIRESTORE_PROJECT_IDS,
+  DATA_DIR,
+  REDIS_HOST,
+  REDIS_PORT,
+  REDIS_LOGS_HOST,
+  REDIS_LOGS_PORT,
+  REDIS_LOGS_DB,
+  POSTGRES_HOST,
+  POSTGRES_PORT,
+  POSTGRES_USER,
+  POSTGRES_PASSWORD,
+  POSTGRES_DATABASE,
+  APP_GATEWAY_PROPERTY_ID,
+  APP_GATEWAY_DEVICE_NATIVE_URL,
+  APP_GATEWAY_DEVICE_SIMULATOR_URL,
+  APP_GATEWAY_SCAN_INTERVAL_SECONDS,
+  APP_GATEWAY_AUTO_START,
+  APP_GATEWAY_RETRY_INTERVAL_SECONDS,
+  DOCKER_LOG_CAPTURE_ENABLED,
+  DOCKER_LOG_NETWORK,
+  DOCKER_LOG_EXCLUDED_CONTAINERS,
+  FUNCTIONS_SOURCE_PATH,
+  FUNCTIONS_BASE_PORT,
+  FUNCTIONS_MAX_INSTANCES,
+  HOST_PROJECT_DIR,
+  DB_MAINTENANCE_INTERVAL_HOURS,
+  DB_RETENTION_LOGGING_DAYS,
+  DB_RETENTION_HTTP_TRAFFIC_DAYS,
+  DB_RETENTION_MQTT_DAYS,
+  DB_RETENTION_PUBSUB_DAYS,
+  DB_RETENTION_INVOCATIONS_DAYS,
+  DB_MAX_LOGGING_ENTRIES
+} = process.env
+
+const runId = Moment().valueOf()
+const envShortNameMap = {
+  development: 'dev',
+  production: 'prd',
+  test: 'test'
+}
+
+export const Application = {
+  runId,
+  environment: NODE_ENV,
+  envShortName: envShortNameMap[NODE_ENV],
+  logLevel: LOG_LEVEL,
+  ports: {
+    webUi: parseInt(WEB_UI_PORT),
+    pubsub: parseInt(PUBSUB_PORT) || 8085,
+    logging: parseInt(LOGGING_PORT),
+    mqtt: parseInt(MQTT_PORT),
+    awsIot: parseInt(AWS_IOT_PORT) || 8086,
+    firestore: parseInt(FIRESTORE_PORT)
+  },
+  pubsub: {
+    projectId: PUBSUB_PROJECT_ID,
+    emulatorHost: PUBSUB_EMULATOR_HOST?.split(':')[0] || 'localhost',
+    emulatorPort: parseInt(PUBSUB_EMULATOR_HOST?.split(':')[1]) || parseInt(PUBSUB_PORT) || 8085,
+    shadowPollIntervalMs: parseInt(SHADOW_POLL_INTERVAL_MS) || 200,
+    shadowSubscriptionCheckIntervalMs: parseInt(SHADOW_SUBSCRIPTION_CHECK_INTERVAL_MS) || 5000
+  },
+  storage: {
+    dataDir: DATA_DIR
+  },
+  redis: {
+    host: REDIS_HOST || 'localhost',
+    port: parseInt(REDIS_PORT) || 6379,
+    db: 0,
+    url: `redis://${REDIS_HOST || 'localhost'}:${REDIS_PORT || 6379}`
+  },
+  redisLogs: {
+    host: REDIS_LOGS_HOST || REDIS_HOST || 'localhost',
+    port: parseInt(REDIS_LOGS_PORT) || parseInt(REDIS_PORT) || 6379,
+    db: parseInt(REDIS_LOGS_DB) || 0,
+    url: `redis://${REDIS_LOGS_HOST || REDIS_HOST || 'localhost'}:${REDIS_LOGS_PORT || REDIS_PORT || 6379}`
+  },
+  postgres: {
+    host: POSTGRES_HOST || 'localhost',
+    port: parseInt(POSTGRES_PORT) || 5432,
+    user: POSTGRES_USER || 'postgres',
+    password: POSTGRES_PASSWORD || 'postgres',
+    database: POSTGRES_DATABASE || 'postgres'
+  },
+  firestore: {
+    projectId: FIRESTORE_PROJECT_ID || 'tipi-development',
+    projectIds: FIRESTORE_PROJECT_IDS
+      ? FIRESTORE_PROJECT_IDS.split(',').map(s => s.trim())
+      : [FIRESTORE_PROJECT_ID || 'tipi-development'],
+    emulatorHost: FIRESTORE_EMULATOR_HOST || `localhost:${FIRESTORE_PORT || 8080}`
+  },
+  messageHistory: {
+    enabled: true,
+    maxMessages: 10000, // Global limit (size-based, not time-based)
+    minMessages: 100, // Always keep at least this many
+    cleanupIntervalHours: 24, // How often to check
+    cleanupThreshold: 0.9 // Trigger cleanup at 90% capacity
+  },
+  appGateway: {
+    propertyId: APP_GATEWAY_PROPERTY_ID || '550e8400-e29b-41d4-a716-446655440000',
+    deviceNativeUrl: APP_GATEWAY_DEVICE_NATIVE_URL || 'http://localhost:3000',
+    deviceSimulatorUrl: APP_GATEWAY_DEVICE_SIMULATOR_URL || 'http://localhost:3001',
+    scanIntervalSeconds: parseInt(APP_GATEWAY_SCAN_INTERVAL_SECONDS) || 30,
+    autoStart: APP_GATEWAY_AUTO_START !== 'false',
+    retryIntervalSeconds: parseInt(APP_GATEWAY_RETRY_INTERVAL_SECONDS) || 10
+  },
+  logging: {
+    dockerCapture: DOCKER_LOG_CAPTURE_ENABLED !== 'false',
+    dockerNetwork: DOCKER_LOG_NETWORK || 'goki-network',
+    dockerExcludedContainers: DOCKER_LOG_EXCLUDED_CONTAINERS
+      ? DOCKER_LOG_EXCLUDED_CONTAINERS.split(',').map(s => s.trim())
+      : null
+  },
+  functions: {
+    sourcePath: FUNCTIONS_SOURCE_PATH || null,
+    basePort: parseInt(FUNCTIONS_BASE_PORT) || 9100,
+    maxInstances: parseInt(FUNCTIONS_MAX_INSTANCES) || 50
+  },
+  hostProjectDir: HOST_PROJECT_DIR || null,
+  dbMaintenance: {
+    intervalHours: parseInt(DB_MAINTENANCE_INTERVAL_HOURS) || 6,
+    maxLoggingEntries: parseInt(DB_MAX_LOGGING_ENTRIES) || 100000,
+    retentionDays: {
+      loggingEntries: parseInt(DB_RETENTION_LOGGING_DAYS) || 7,
+      httpTraffic: parseInt(DB_RETENTION_HTTP_TRAFFIC_DAYS) || 3,
+      mqttMessages: parseInt(DB_RETENTION_MQTT_DAYS) || 3,
+      pubsubMessages: parseInt(DB_RETENTION_PUBSUB_DAYS) || 3,
+      functionInvocations: parseInt(DB_RETENTION_INVOCATIONS_DAYS) || 7
+    }
+  }
+}

package/src/configs/Default.js

@@ -0,0 +1,13 @@
+const {
+  AUTO_FLUSH_INTERVAL_MS
+} = process.env
+
+export const Default = {
+  storage: {
+    autoFlushIntervalMs: parseInt(AUTO_FLUSH_INTERVAL_MS)
+  },
+  pagination: Object.freeze({
+    defaultLimit: 50,
+    maxLimit: 1000
+  })
+}

package/src/consumers/BlackboxLogsConsumer.js

@@ -0,0 +1,235 @@
+import Queue from 'bull'
+import { SqliteStore } from '../singletons/SqliteStore.js'
+import { LOGGING_ENTRIES, LOGGING_LOGS } from '../db/Tables.js'
+import { Logger } from '../singletons/Logger.js'
+import { LogBroadcaster } from '../singletons/LogBroadcaster.js'
+
+export class BlackboxLogsConsumer {
+  constructor (redisConfigs) {
+    this.queues = []
+    this.redisConfigs = redisConfigs
+    redisConfigs.forEach((config, index) => {
+      const queueName = 'blackboxLogs'
+      const queue = new Queue(queueName, {
+        redis: config,
+        settings: {
+          drainDelay: 1000,
+          stalledInterval: 30000
+        },
+        limiter: {
+          max: 20,
+          duration: 1000
+        },
+        defaultJobOptions: {
+          attempts: 3,
+          backoff: {
+            type: 'exponential',
+            delay: 2000
+          },
+          removeOnComplete: true,
+          removeOnFail: true,
+          timeout: 30000
+        }
+      })
+      queue.on('error', (error) => {
+        Logger.log({
+          level: 'error',
+          message: `Bull queue error on Redis ${config.host}:${config.port}`,
+          data: { error: error.message, stack: error.stack }
+        })
+      })
+      queue.on('ready', () => {
+        Logger.log({
+          level: 'info',
+          message: `Bull queue consumer connected to Redis ${config.host}:${config.port}`,
+          data: { queueName, redis: config }
+        })
+      })
+      queue.process(async (job) => {
+        const startTime = Date.now()
+        try {
+          Logger.log({
+            level: 'debug',
+            message: 'Bull queue job received',
+            data: {
+              jobId: job.id,
+              redisHost: config.host,
+              redisPort: config.port,
+              service: job.data?.service
+            }
+          })
+          await this.processLog(job.data, config, job)
+        } catch (error) {
+          Logger.log({
+            level: 'error',
+            message: 'Failed to process Bull queue log',
+            data: {
+              jobId: job.id,
+              attemptNumber: job.attemptsMade + 1,
+              maxAttempts: job.opts.attempts,
+              willRetry: job.attemptsMade < job.opts.attempts - 1,
+              error: error.message,
+              stack: error.stack,
+              redisHost: config.host,
+              redisPort: config.port,
+              processingTime: Date.now() - startTime
+            }
+          })
+          throw error
+        }
+      })
+      this.queues.push({ queue, config })
+      Logger.log({
+        level: 'info',
+        message: 'Bull queue consumer registered',
+        data: { queueName, redisHost: config.host, redisPort: config.port }
+      })
+    })
+  }
+
+  async processLog (data, redisConfig, job) {
+    const { service, log } = data
+    Logger.log({
+      level: 'debug',
+      message: 'Processing Bull queue job',
+      data: {
+        service,
+        message: log?.message,
+        redisHost: redisConfig.host,
+        redisPort: redisConfig.port
+      }
+    })
+    if (!log || !log.level) {
+      Logger.log({
+        level: 'warn',
+        message: 'Invalid log entry in Bull queue - job will be marked complete',
+        data: { service, log, jobId: job?.id }
+      })
+      return
+    }
+    const severityMap = {
+      error: 'ERROR',
+      warn: 'WARNING',
+      info: 'INFO',
+      http: 'INFO',
+      verbose: 'DEBUG',
+      debug: 'DEBUG'
+    }
+    const logName = `projects/dev/logs/${service || 'unknown'}`
+    const severity = severityMap[log.level] || 'INFO'
+    const timestamp = log.timestamp || new Date().toISOString()
+    const timeWindow = 2000
+    const timestampDate = new Date(timestamp)
+    const startTime = new Date(timestampDate.getTime() - timeWindow).toISOString()
+    const endTime = new Date(timestampDate.getTime() + timeWindow).toISOString()
+    const serviceName = service || 'unknown'
+    const isDuplicate = SqliteStore.exists(LOGGING_ENTRIES,
+      'text_payload = ? AND service_name = ? AND timestamp >= ? AND timestamp <= ?',
+      [log.message, serviceName, startTime, endTime]
+    )
+    if (isDuplicate) {
+      return
+    }
+    const logId = serviceName
+    try {
+      SqliteStore.create(LOGGING_LOGS, {
+        projectId: 'dev',
+        logId,
+        logName,
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString()
+      })
+    } catch (error) {
+      // Ignore duplicate errors - log stream already exists
+      if (!error.message.includes('UNIQUE')) {
+        throw error
+      }
+    }
+    // Merge log.metadata into labels (minus traceId/requestId which get own columns)
+    const metadata = log.metadata || {}
+    const labels = { ...metadata }
+    delete labels.traceId
+    delete labels.requestId
+    const logEntry = {
+      logName,
+      serviceName,
+      severity,
+      source: 'bull-queue',
+      level: log.level,
+      timestamp,
+      receiveTimestamp: new Date().toISOString(),
+      textPayload: log.message,
+      jsonPayload: JSON.stringify(log),
+      labels: JSON.stringify(labels)
+    }
+    if (metadata.traceId) logEntry.trace = metadata.traceId
+    if (metadata.requestId) logEntry.insertId = metadata.requestId
+    // Extract error info
+    if (metadata.error) {
+      logEntry.errorMessage = typeof metadata.error === 'string'
+        ? metadata.error
+        : metadata.error.message || JSON.stringify(metadata.error)
+    } else if (log.stack) {
+      logEntry.errorMessage = log.message
+    }
+    const entry = await SqliteStore.create(LOGGING_ENTRIES, logEntry)
+    LogBroadcaster.broadcast(logEntry)
+    Logger.log({
+      level: 'debug',
+      message: 'Processed Bull queue log successfully',
+      data: {
+        service,
+        severity,
+        logName,
+        redisHost: redisConfig.host,
+        redisPort: redisConfig.port,
+        entryId: entry._id
+      }
+    })
+  }
+
+  async close () {
+    Logger.log({
+      level: 'info',
+      message: 'Closing Bull queue consumers'
+    })
+    for (const { queue, config } of this.queues) {
+      await queue.close()
+      Logger.log({
+        level: 'info',
+        message: 'Closed Bull queue consumer',
+        data: { redis: `${config.host}:${config.port}` }
+      })
+    }
+  }
+
+  getStats () {
+    return Promise.all(
+      this.queues.map(async ({ queue, config }) => {
+        const counts = await queue.getJobCounts()
+        return {
+          redis: `${config.host}:${config.port}`,
+          counts
+        }
+      })
+    )
+  }
+
+  async getHealth () {
+    const health = []
+    for (const { queue, config } of this.queues) {
+      const counts = await queue.getJobCounts()
+      const isHealthy = counts.failed < 100 && counts.active < 50
+      health.push({
+        redis: `${config.host}:${config.port}`,
+        healthy: isHealthy,
+        counts,
+        warnings: [
+          counts.failed >= 100 ? 'High failed job count' : null,
+          counts.active >= 50 ? 'High active job count' : null
+        ].filter(Boolean)
+      })
+    }
+    return health
+  }
+}