@gokiteam/goki-dev 0.2.0

package/src/api/logging/Logic.js

@@ -0,0 +1,461 @@
+import { SqliteStore } from '../../singletons/SqliteStore.js'
+import { LOGGING_ENTRIES, LOGGING_LOGS } from '../../db/Tables.js'
+
+const COMPACT_COLUMNS = '_id, timestamp, severity, service_name, text_payload, error_message, trace, event'
+
+/**
+ * Convert MongoDB-style filter to SQL WHERE clause
+ * Supports: exact match, $in, $regex, $gte, $lte, $gt, $lt, $ne, $or, $nor, $not, $search
+ */
+const filterToSQL = (filter) => {
+  const conditions = []
+  const params = []
+
+  for (const [key, value] of Object.entries(filter)) {
+    // Special operators
+    if (key === '$or') {
+      // OR operator: { $or: [{...}, {...}] }
+      const orConditions = value.map(subFilter => filterToSQL(subFilter))
+      const orSQL = orConditions.map(({ sql }) => `(${sql})`).join(' OR ')
+      conditions.push(`(${orSQL})`)
+      orConditions.forEach(({ params: subParams }) => params.push(...subParams))
+      continue
+    }
+
+    if (key === '$nor') {
+      // NOR operator: { $nor: [{...}, {...}] }
+      const norConditions = value.map(subFilter => filterToSQL(subFilter))
+      const norSQL = norConditions.map(({ sql }) => `(${sql})`).join(' OR ')
+      conditions.push(`NOT (${norSQL})`)
+      norConditions.forEach(({ params: subParams }) => params.push(...subParams))
+      continue
+    }
+
+    if (key === '$search') {
+      // SEARCH function: { $search: { term: "...", fields: [...] } }
+      const { term, fields } = value
+      const searchConditions = fields.map(field => {
+        const column = mapFieldToColumn(field)
+        params.push(`%${term}%`)
+        return `${column} LIKE ?`
+      })
+      conditions.push(`(${searchConditions.join(' OR ')})`)
+      continue
+    }
+
+    // Regular field conditions
+    const column = mapFieldToColumn(key)
+
+    if (value === null || value === undefined) {
+      conditions.push(`${column} IS NULL`)
+      continue
+    }
+
+    if (typeof value === 'object' && !Array.isArray(value)) {
+      // MongoDB operators
+      if (value.$in) {
+        const placeholders = value.$in.map(() => '?').join(', ')
+        conditions.push(`${column} IN (${placeholders})`)
+        params.push(...value.$in)
+      } else if (value.$nin) {
+        const placeholders = value.$nin.map(() => '?').join(', ')
+        conditions.push(`${column} NOT IN (${placeholders})`)
+        params.push(...value.$nin)
+      } else if (value.$regex) {
+        const pattern = `%${value.$regex}%`
+        conditions.push(`${column} LIKE ?`)
+        params.push(pattern)
+      } else if (value.$not) {
+        if (value.$not.$regex) {
+          const pattern = `%${value.$not.$regex}%`
+          conditions.push(`${column} NOT LIKE ?`)
+          params.push(pattern)
+        } else {
+          // Generic NOT
+          const notFilter = { [key]: value.$not }
+          const { sql: notSQL, params: notParams } = filterToSQL(notFilter)
+          conditions.push(`NOT (${notSQL})`)
+          params.push(...notParams)
+        }
+      } else if (value.$gte !== undefined) {
+        conditions.push(`${column} >= ?`)
+        params.push(value.$gte)
+      } else if (value.$lte !== undefined) {
+        conditions.push(`${column} <= ?`)
+        params.push(value.$lte)
+      } else if (value.$gt !== undefined) {
+        conditions.push(`${column} > ?`)
+        params.push(value.$gt)
+      } else if (value.$lt !== undefined) {
+        conditions.push(`${column} < ?`)
+        params.push(value.$lt)
+      } else if (value.$ne !== undefined) {
+        conditions.push(`${column} != ?`)
+        params.push(value.$ne)
+      } else if (value.$eq !== undefined) {
+        conditions.push(`${column} = ?`)
+        params.push(value.$eq)
+      }
+    } else {
+      // Exact match
+      conditions.push(`${column} = ?`)
+      params.push(value)
+    }
+  }
+
+  const sql = conditions.length > 0 ? conditions.join(' AND ') : '1=1'
+  return { sql, params }
+}
+
+/**
+ * Map camelCase field names to snake_case column names
+ */
+const mapFieldToColumn = (field) => {
+  const fieldMap = {
+    severity: 'severity',
+    logName: 'log_name',
+    serviceName: 'service_name',
+    timestamp: 'timestamp',
+    textPayload: 'text_payload',
+    jsonPayload: 'json_payload',
+    protoPayload: 'proto_payload',
+    trace: 'trace',
+    spanId: 'span_id',
+    traceSampled: 'trace_sampled',
+    insertId: 'insert_id',
+    receiveTimestamp: 'receive_timestamp',
+    httpRequest: 'http_request',
+    operation: 'operation',
+    sourceLocation: 'source_location',
+    source: 'source',
+    level: 'level',
+    errorMessage: 'error_message',
+    stackTrace: 'stack_trace',
+    event: 'event',
+    duration: 'duration',
+    errorCode: 'error_code',
+    errorStatus: 'error_status',
+    errorReason: 'error_reason',
+    errorResource: 'error_resource',
+    propertyId: 'property_id',
+    deviceId: 'device_id',
+    doorId: 'door_id',
+    subscriptionName: 'subscription_name',
+    messageId: 'message_id',
+    labels: 'labels'
+  }
+
+  // Handle nested fields (e.g., labels.environment)
+  if (field.includes('.')) {
+    const parts = field.split('.')
+    const base = fieldMap[parts[0]] || parts[0]
+    // For JSON fields, use SQLite JSON operators
+    if (base === 'labels' || base === 'json_payload') {
+      return `json_extract(${base}, '$.${parts.slice(1).join('.')}')`
+    }
+    return field
+  }
+
+  return fieldMap[field] || field
+}
+
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+export const Logic = {
+  async listServices (params) {
+    const { traceId } = params
+    try {
+      // Use raw SQL for efficiency - get unique service names directly from database
+      const query = `
+        SELECT DISTINCT service_name
+        FROM ${LOGGING_ENTRIES}
+        WHERE service_name IS NOT NULL
+        ORDER BY service_name ASC
+      `
+      const rows = SqliteStore.db.prepare(query).all()
+      const services = rows.map(row => row.service_name)
+      return {
+        services,
+        total: services.length,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+  async listEntries (params) {
+    const { filter = {}, page = {}, compact = false, traceId } = params
+    const startTime = Date.now()
+    try {
+      // Convert MongoDB-style filter to SQL WHERE clause
+      const { sql: whereSQL, params: whereParams } = filterToSQL(filter)
+      // Select only needed columns in compact mode
+      const columns = compact ? COMPACT_COLUMNS : '*'
+      let sql = `SELECT ${columns} FROM ${LOGGING_ENTRIES}`
+      const queryParams = []
+      if (whereSQL && whereSQL !== '1=1') {
+        sql += ` WHERE ${whereSQL}`
+        queryParams.push(...whereParams)
+      }
+      // Count total matching entries
+      const countSQL = `SELECT COUNT(*) as count FROM ${LOGGING_ENTRIES}` +
+        (whereSQL && whereSQL !== '1=1' ? ` WHERE ${whereSQL}` : '')
+      const totalResult = SqliteStore.db.prepare(countSQL).get(...whereParams)
+      const total = totalResult.count
+      // Add ordering (newest first)
+      sql += ' ORDER BY timestamp DESC'
+      // Add pagination
+      const limit = page.limit || 100
+      const offset = page.offset || 0
+      sql += ' LIMIT ? OFFSET ?'
+      queryParams.push(limit, offset)
+      // Log query for debugging (only in development)
+      if (process.env.NODE_ENV !== 'production') {
+        console.log('[Query Debug]', {
+          sql,
+          params: queryParams,
+          filter
+        })
+      }
+      // Execute query
+      const stmt = SqliteStore.db.prepare(sql)
+      let entries = stmt.all(...queryParams)
+      // Convert snake_case to camelCase and parse JSON fields
+      entries = entries.map(row => SqliteStore.toCamelCase(row))
+      if (!compact) {
+        entries = entries.map(row => SqliteStore.parseJsonFields(LOGGING_ENTRIES, row))
+      }
+      const executionTime = Date.now() - startTime
+      // Log performance metrics
+      if (process.env.NODE_ENV !== 'production' || executionTime > 1000) {
+        console.log('[Query Performance]', {
+          executionTime: `${executionTime}ms`,
+          total,
+          returned: entries.length,
+          hasFilter: Object.keys(filter).length > 0
+        })
+      }
+      return {
+        entries,
+        total,
+        executionTime,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        executionTime: Date.now() - startTime,
+        traceId
+      }
+    }
+  },
+  async getEntriesById (params) {
+    const { ids, traceId } = params
+    try {
+      const placeholders = ids.map(() => '?').join(', ')
+      const sql = `SELECT * FROM ${LOGGING_ENTRIES} WHERE _id IN (${placeholders}) ORDER BY timestamp DESC`
+      let entries = SqliteStore.db.prepare(sql).all(...ids)
+      entries = entries.map(row => SqliteStore.toCamelCase(row))
+        .map(row => SqliteStore.parseJsonFields(LOGGING_ENTRIES, row))
+      return {
+        entries,
+        total: entries.length,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+  async clearEntries (params) {
+    const { traceId } = params
+    try {
+      SqliteStore.clear(LOGGING_ENTRIES)
+      SqliteStore.clear(LOGGING_LOGS)
+      return {
+        message: 'All log entries cleared',
+        traceId
+      }
+    } catch (error) {
+      throw error
+    }
+  },
+  async exportEntries (params) {
+    const { traceId } = params
+    try {
+      const result = SqliteStore.list(LOGGING_ENTRIES)
+      return {
+        entries: result.data,
+        total: result.total,
+        exportedAt: new Date().toISOString(),
+        traceId
+      }
+    } catch (error) {
+      throw error
+    }
+  },
+  async getTrace (params) {
+    const { traceId } = params
+    try {
+      // Get all entries with the specified traceId using SQL WHERE
+      const result = SqliteStore.list(LOGGING_ENTRIES, {
+        where: { trace: traceId },
+        orderBy: 'timestamp ASC'
+      })
+
+      const entries = result.data
+
+      // Extract unique services
+      const services = [...new Set(entries.map(entry => {
+        const logName = entry.logName || ''
+        const parts = logName.split('/')
+        return parts[parts.length - 1] || logName
+      }))]
+
+      // Calculate total duration
+      let duration = 0
+      if (entries.length > 1) {
+        const firstTimestamp = new Date(entries[0].timestamp)
+        const lastTimestamp = new Date(entries[entries.length - 1].timestamp)
+        duration = lastTimestamp - firstTimestamp
+      }
+
+      // Count errors and warnings
+      const errorCount = entries.filter(e =>
+        e.severity === 'ERROR' || e.severity === 'CRITICAL'
+      ).length
+      const warningCount = entries.filter(e =>
+        e.severity === 'WARNING'
+      ).length
+
+      return {
+        trace: {
+          traceId,
+          entries,
+          services,
+          duration,
+          errorCount,
+          warningCount
+        },
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Wait for a log entry matching filter criteria
+   */
+  async waitForLog (params) {
+    const { filter, timeout = 5000, traceId } = params
+    const startTime = Date.now()
+    const pollInterval = 100
+    try {
+      while (Date.now() - startTime < timeout) {
+        const logFilter = {}
+        if (filter.traceId) logFilter.trace = filter.traceId
+        if (filter.service) logFilter.service_name = filter.service
+        if (filter.severity) logFilter.severity = filter.severity
+        const logsResult = await Logic.listEntries({
+          filter: logFilter,
+          page: { limit: 50 },
+          traceId
+        })
+        if (logsResult.entries && logsResult.entries.length > 0) {
+          for (const entry of logsResult.entries) {
+            if (!filter.textContains) {
+              return {
+                entry,
+                foundAt: Date.now() - startTime,
+                traceId
+              }
+            }
+            const textPayload = entry.textPayload || entry.jsonPayload?.message || ''
+            if (textPayload.includes(filter.textContains)) {
+              return {
+                entry,
+                foundAt: Date.now() - startTime,
+                traceId
+              }
+            }
+          }
+        }
+        await sleep(pollInterval)
+      }
+      return {
+        status: 'error',
+        message: 'Timeout waiting for log entry',
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Assert that no errors occurred for a given trace
+   */
+  async assertNoErrors (params) {
+    const { targetTraceId, sinceTimestamp, traceId } = params
+    try {
+      const logFilter = {
+        trace: targetTraceId
+      }
+      const logsResult = await Logic.listEntries({
+        filter: logFilter,
+        page: { limit: 1000 },
+        traceId
+      })
+      let errors = []
+      let criticalErrors = []
+      if (logsResult.entries && logsResult.entries.length > 0) {
+        const filteredEntries = sinceTimestamp
+          ? logsResult.entries.filter(e => new Date(e.timestamp) >= new Date(sinceTimestamp))
+          : logsResult.entries
+        errors = filteredEntries.filter(e => e.severity === 'ERROR')
+        criticalErrors = filteredEntries.filter(e => e.severity === 'CRITICAL')
+      }
+      return {
+        hasErrors: errors.length > 0 || criticalErrors.length > 0,
+        errorCount: errors.length,
+        criticalCount: criticalErrors.length,
+        errors: [...errors, ...criticalErrors],
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  async getQueueStats (params) {
+    const { traceId } = params
+    try {
+      const stats = await global.logsConsumer.getStats()
+      const health = await global.logsConsumer.getHealth()
+      return { stats, health, traceId }
+    } catch (error) {
+      return { status: 'error', message: error.message, traceId }
+    }
+  }
+}

package/src/api/logging/Router.js

@@ -0,0 +1,24 @@
+import KoaRouter from 'koa-router'
+import { Controllers } from './Controllers.js'
+
+const Router = new KoaRouter()
+const v1 = new KoaRouter({ prefix: '/v1/logging' })
+
+v1.post('/services/list', Controllers.listServices)
+v1.post('/entries/list', Controllers.listEntries)
+v1.post('/entries/get', Controllers.getEntriesById)
+v1.post('/entries/clear', Controllers.clearEntries)
+v1.post('/entries/export', Controllers.exportEntries)
+v1.get('/entries/stream', Controllers.streamLogs)
+v1.post('/trace/:traceId', Controllers.getTrace)
+
+// Testing Helpers
+v1.post('/entries/wait-for', Controllers.waitForLog)
+v1.post('/entries/assert-no-errors', Controllers.assertNoErrors)
+
+// Queue Stats
+v1.post('/queue-stats', Controllers.getQueueStats)
+
+Router.use(v1.routes())
+
+export { Router }

package/src/api/logging/Schemas.js

@@ -0,0 +1,43 @@
+import { Joi } from '@gokiteam/koa'
+
+export const Schemas = {
+  waitForLog: {
+    request: {
+      body: {
+        filter: Joi.object({
+          traceId: Joi.string().optional(),
+          service: Joi.string().optional(),
+          severity: Joi.string().valid('DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL', 'ALERT', 'EMERGENCY').optional(),
+          textContains: Joi.string().optional()
+        }).required(),
+        timeout: Joi.number().integer().min(100).max(30000).default(5000),
+        traceId: Joi.string().optional()
+      }
+    },
+    responses: {
+      success: {
+        entry: Joi.object().required(),
+        foundAt: Joi.number().integer().required(),
+        traceId: Joi.string().required()
+      }
+    }
+  },
+
+  assertNoErrors: {
+    request: {
+      body: {
+        traceId: Joi.string().required(),
+        sinceTimestamp: Joi.string().isoDate().optional()
+      }
+    },
+    responses: {
+      success: {
+        hasErrors: Joi.boolean().required(),
+        errorCount: Joi.number().integer().min(0).required(),
+        criticalCount: Joi.number().integer().min(0).required(),
+        errors: Joi.array().items(Joi.object()).optional(),
+        traceId: Joi.string().required()
+      }
+    }
+  }
+}

package/src/api/mqtt/Controllers.js

@@ -0,0 +1,17 @@
+import { Logic } from './Logic.js'
+
+export const Controllers = {
+  async listClients (ctx) {
+    const { traceId } = ctx.state
+    const { filter, page } = ctx.request.body
+    const result = await Logic.listClients({ filter, page, traceId })
+    ctx.reply(result)
+  },
+
+  async listMessages (ctx) {
+    const { traceId } = ctx.state
+    const { filter, page } = ctx.request.body
+    const result = await Logic.listMessages({ filter, page, traceId })
+    ctx.reply(result)
+  }
+}

package/src/api/mqtt/Logic.js

@@ -0,0 +1,66 @@
+import { SqliteStore } from '../../singletons/SqliteStore.js'
+import { MQTT_CLIENTS, MQTT_MESSAGES, MQTT_SUBSCRIPTIONS } from '../../db/Tables.js'
+
+export const Logic = {
+  async listClients (params) {
+    const { filter = {}, page = {}, traceId } = params
+    try {
+      const options = {
+        limit: page.limit || 50,
+        offset: page.offset || 0,
+        // Only return currently connected clients (disconnectedAt is null)
+        where: { disconnectedAt: null }
+      }
+
+      // Convert object filter to predicate function if needed
+      if (Object.keys(filter).length > 0) {
+        options.filter = (item) => {
+          return Object.entries(filter).every(([key, value]) => item[key] === value)
+        }
+      }
+
+      const result = SqliteStore.list(MQTT_CLIENTS, options)
+      return {
+        clients: result.data,
+        total: result.total,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  async listMessages (params) {
+    const { filter = {}, page = {}, traceId } = params
+    try {
+      const options = {
+        limit: page.limit || 50,
+        offset: page.offset || 0
+      }
+
+      // Convert object filter to predicate function if needed
+      if (Object.keys(filter).length > 0) {
+        options.filter = (item) => {
+          return Object.entries(filter).every(([key, value]) => item[key] === value)
+        }
+      }
+
+      const result = SqliteStore.list(MQTT_MESSAGES, options)
+      return {
+        messages: result.data,
+        total: result.total,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  }
+}

package/src/api/mqtt/Router.js

@@ -0,0 +1,12 @@
+import KoaRouter from 'koa-router'
+import { Controllers } from './Controllers.js'
+
+const Router = new KoaRouter()
+const v1 = new KoaRouter({ prefix: '/v1/mqtt' })
+
+v1.post('/clients/list', Controllers.listClients)
+v1.post('/messages/list', Controllers.listMessages)
+
+Router.use(v1.routes())
+
+export { Router }