@gokiteam/goki-dev 0.2.0

package/src/singletons/RedisClient.js

@@ -0,0 +1,184 @@
+import Redis from 'ioredis'
+import { Application } from '../configs/Application.js'
+
+class RedisClientClass {
+  constructor () {
+    this.client = null
+    this.isInitialized = false
+  }
+
+  initialize () {
+    if (this.isInitialized) return
+    const config = Application.redis
+    this.client = new Redis({
+      host: config.host,
+      port: config.port,
+      lazyConnect: true,
+      retryStrategy: (times) => {
+        if (times > 3) return null
+        return Math.min(times * 100, 3000)
+      }
+    })
+    this.client.on('error', (err) => {
+      console.error('Redis client error:', err.message)
+    })
+    this.isInitialized = true
+  }
+
+  async connect () {
+    if (!this.isInitialized) {
+      this.initialize()
+    }
+    try {
+      await this.client.connect()
+    } catch (error) {
+      // Already connected or connection failed
+      if (!error.message.includes('already connected')) {
+        throw error
+      }
+    }
+  }
+
+  async testConnection () {
+    try {
+      if (!this.isInitialized) {
+        this.initialize()
+      }
+      const pong = await this.client.ping()
+      return { connected: pong === 'PONG' }
+    } catch (error) {
+      return { connected: false, error: error.message }
+    }
+  }
+
+  async getInfo () {
+    const info = await this.client.info()
+    const parsed = {}
+    info.split('\n').forEach(line => {
+      if (line && !line.startsWith('#')) {
+        const [key, value] = line.split(':')
+        if (key && value) {
+          parsed[key.trim()] = value.trim()
+        }
+      }
+    })
+    return parsed
+  }
+
+  async getDbSize () {
+    return await this.client.dbsize()
+  }
+
+  async scanKeys (pattern = '*', cursor = '0', count = 100) {
+    const [nextCursor, keys] = await this.client.scan(cursor, 'MATCH', pattern, 'COUNT', count)
+    return {
+      cursor: nextCursor,
+      keys,
+      done: nextCursor === '0'
+    }
+  }
+
+  async getKeyType (key) {
+    return await this.client.type(key)
+  }
+
+  async getKeyTtl (key) {
+    return await this.client.ttl(key)
+  }
+
+  async getKeyValue (key) {
+    const type = await this.getKeyType(key)
+    let value
+    switch (type) {
+      case 'string':
+        value = await this.client.get(key)
+        break
+      case 'list':
+        value = await this.client.lrange(key, 0, -1)
+        break
+      case 'set':
+        value = await this.client.smembers(key)
+        break
+      case 'zset':
+        value = await this.client.zrange(key, 0, -1, 'WITHSCORES')
+        // Convert flat array to array of [member, score] pairs
+        const pairs = []
+        for (let i = 0; i < value.length; i += 2) {
+          pairs.push({ member: value[i], score: parseFloat(value[i + 1]) })
+        }
+        value = pairs
+        break
+      case 'hash':
+        value = await this.client.hgetall(key)
+        break
+      case 'stream':
+        value = await this.client.xrange(key, '-', '+', 'COUNT', 100)
+        break
+      default:
+        value = null
+    }
+    const ttl = await this.getKeyTtl(key)
+    const memoryUsage = await this.getKeyMemory(key)
+    return {
+      key,
+      type,
+      value,
+      ttl,
+      memoryUsage
+    }
+  }
+
+  async getKeyMemory (key) {
+    try {
+      return await this.client.memory('USAGE', key)
+    } catch (error) {
+      return null
+    }
+  }
+
+  async getAllKeys (pattern = '*', maxKeys = 1000) {
+    const keys = []
+    let cursor = '0'
+    do {
+      const result = await this.scanKeys(pattern, cursor, 100)
+      keys.push(...result.keys)
+      cursor = result.cursor
+      if (keys.length >= maxKeys) break
+    } while (cursor !== '0')
+    return keys.slice(0, maxKeys)
+  }
+
+  async deleteKey (key) {
+    if (!this.isInitialized) {
+      this.initialize()
+    }
+    return await this.client.del(key)
+  }
+
+  async setKey (key, value, ttl = null) {
+    if (!this.isInitialized) {
+      this.initialize()
+    }
+    if (ttl && ttl > 0) {
+      return await this.client.set(key, value, 'EX', ttl)
+    }
+    return await this.client.set(key, value)
+  }
+
+  async deleteAll () {
+    if (!this.isInitialized) {
+      this.initialize()
+    }
+    return await this.client.flushall()
+  }
+
+  async shutdown () {
+    if (this.client) {
+      await this.client.quit()
+      this.client = null
+      this.isInitialized = false
+    }
+  }
+}
+
+export const RedisClient = new RedisClientClass()

package/src/singletons/SqliteStore.js

@@ -0,0 +1,480 @@
+import fs from 'fs'
+import path from 'path'
+import Database from 'better-sqlite3'
+import { Application } from '../configs/Application.js'
+import { PUBSUB_SCHEMAS } from '../db/schemas/pubsub.js'
+import { LOGGING_SCHEMAS } from '../db/schemas/logging.js'
+import { MQTT_SCHEMAS } from '../db/schemas/mqtt.js'
+import { FIRESTORE_SCHEMAS } from '../db/schemas/firestore.js'
+import { WEBHOOK_SCHEMAS } from '../db/schemas/webhooks.js'
+import { HTTP_TRAFFIC_SCHEMAS } from '../db/schemas/httpTraffic.js'
+import { PUBSUB_TOPIC_REGISTRY_SCHEMAS } from '../db/schemas/pubsubRegistry.js'
+import { FUNCTIONS_SCHEMAS } from '../db/schemas/functions.js'
+import { MIGRATIONS_SCHEMA, CURRENT_SCHEMA_VERSION } from '../db/schemas/migrations.js'
+
+class SqliteStoreClass {
+  constructor () {
+    this.db = null
+    this.isInitialized = false
+  }
+
+  initialize (dbPath = null) {
+    if (this.isInitialized) return
+
+    const dataDir = Application.storage.dataDir || './data'
+    if (!fs.existsSync(dataDir)) {
+      fs.mkdirSync(dataDir, { recursive: true })
+    }
+
+    const finalDbPath = dbPath || path.join(dataDir, 'dev-tools.db')
+    this.db = new Database(finalDbPath)
+    this.db.pragma('journal_mode = WAL')
+    this.db.pragma('synchronous = NORMAL')
+    this.db.pragma('busy_timeout = 5000')
+    this.db.pragma('wal_autocheckpoint = 1000')
+    this.db.pragma('cache_size = -64000')
+    this.db.pragma('temp_store = MEMORY')
+    this.db.pragma('foreign_keys = ON')
+
+    this.createTables()
+    this.isInitialized = true
+  }
+
+  createTables () {
+    this.db.exec(MIGRATIONS_SCHEMA)
+
+    const schemas = [
+      ...PUBSUB_SCHEMAS,
+      ...LOGGING_SCHEMAS,
+      ...MQTT_SCHEMAS,
+      ...FIRESTORE_SCHEMAS,
+      ...WEBHOOK_SCHEMAS,
+      ...HTTP_TRAFFIC_SCHEMAS,
+      ...PUBSUB_TOPIC_REGISTRY_SCHEMAS,
+      ...FUNCTIONS_SCHEMAS
+    ]
+
+    const currentVersion = this.getSchemaVersion()
+    if (currentVersion < CURRENT_SCHEMA_VERSION) {
+      this.db.transaction(() => {
+        // Drop and recreate logging tables for clean schema
+        this.db.exec('DROP TABLE IF EXISTS logging_entries')
+        this.db.exec('DROP TABLE IF EXISTS logging_logs')
+        for (const schema of schemas) {
+          this.db.exec(schema)
+        }
+        this.db.prepare('INSERT OR REPLACE INTO schema_migrations (version, name) VALUES (?, ?)').run(
+          CURRENT_SCHEMA_VERSION,
+          'db_maintenance_indexes'
+        )
+      })()
+    }
+  }
+
+  getSchemaVersion () {
+    try {
+      const result = this.db.prepare('SELECT MAX(version) as version FROM schema_migrations').get()
+      return result?.version || 0
+    } catch (error) {
+      return 0
+    }
+  }
+
+  registerCollection (collectionName, filePath) {
+    // No-op for compatibility - tables created at initialization
+  }
+
+  get (tableName, id, idField = 'id') {
+    try {
+      const snakeIdField = idField.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`)
+      const stmt = this.db.prepare(`SELECT * FROM ${tableName} WHERE ${snakeIdField} = ?`)
+      const result = stmt.get(id)
+      if (!result) return undefined
+      const camelResult = this.toCamelCase(result)
+      return this.parseJsonFields(tableName, camelResult)
+    } catch (error) {
+      throw new Error(`Failed to get ${id} from ${tableName}: ${error.message}`)
+    }
+  }
+
+  /**
+   * Check if a row exists matching a SQL WHERE clause
+   * @param {string} tableName - Table to query
+   * @param {string} whereClause - SQL WHERE conditions (without WHERE keyword)
+   * @param {Array} params - Bind parameters
+   * @returns {boolean}
+   */
+  exists (tableName, whereClause, params = []) {
+    try {
+      const stmt = this.db.prepare(`SELECT 1 FROM ${tableName} WHERE ${whereClause} LIMIT 1`)
+      return !!stmt.get(...params)
+    } catch (error) {
+      throw new Error(`Failed to check existence in ${tableName}: ${error.message}`)
+    }
+  }
+
+  find (tableName, predicate) {
+    try {
+      if (typeof predicate === 'function') {
+        const all = this.db.prepare(`SELECT * FROM ${tableName}`).all()
+        return all.map(row => this.toCamelCase(row)).map(row => this.parseJsonFields(tableName, row)).filter(predicate)
+      } else if (typeof predicate === 'object') {
+        const { sql, params } = this.buildWhereClause(predicate)
+        const stmt = this.db.prepare(`SELECT * FROM ${tableName} ${sql}`)
+        const results = stmt.all(...params)
+        return results.map(row => this.toCamelCase(row)).map(row => this.parseJsonFields(tableName, row))
+      }
+      return []
+    } catch (error) {
+      throw new Error(`Failed to find in ${tableName}: ${error.message}`)
+    }
+  }
+
+  list (tableName, options = {}) {
+    try {
+      let sql = `SELECT * FROM ${tableName}`
+      const params = []
+
+      if (options.filter && typeof options.filter === 'object') {
+        const where = this.buildWhereClause(options.filter)
+        sql += ` ${where.sql}`
+        params.push(...where.params)
+      }
+
+      if (options.where) {
+        const where = this.buildWhereClause(options.where)
+        sql += ` ${where.sql}`
+        params.push(...where.params)
+      }
+
+      const countSql = `SELECT COUNT(*) as count FROM ${tableName}` +
+        (params.length > 0 ? ` ${this.buildWhereClause(options.where || options.filter).sql}` : '')
+      const totalResult = this.db.prepare(countSql).get(...params)
+      const total = totalResult.count
+
+      if (options.orderBy) {
+        sql += ` ORDER BY ${options.orderBy}`
+      } else if (options.sort) {
+        sql += ` ORDER BY ${options.sort}`
+      }
+
+      if (options.limit !== undefined) {
+        sql += ' LIMIT ?'
+        params.push(options.limit)
+      }
+
+      if (options.offset !== undefined) {
+        sql += ' OFFSET ?'
+        params.push(options.offset)
+      }
+
+      const stmt = this.db.prepare(sql)
+      let data = stmt.all(...params)
+
+      data = data.map(row => this.toCamelCase(row)).map(row => this.parseJsonFields(tableName, row))
+
+      if (options.filter && typeof options.filter === 'function') {
+        data = data.filter(options.filter)
+      }
+
+      if (options.sort && typeof options.sort === 'function') {
+        data = data.sort(options.sort)
+      }
+
+      return { data, total }
+    } catch (error) {
+      throw new Error(`Failed to list ${tableName}: ${error.message}`)
+    }
+  }
+
+  create (tableName, item) {
+    try {
+      const preparedItem = this.stringifyJsonFields(tableName, item)
+      const snakeCaseItem = this.toSnakeCase(preparedItem)
+      const fields = Object.keys(snakeCaseItem).filter(f => snakeCaseItem[f] !== undefined)
+      const placeholders = fields.map(() => '?').join(', ')
+      const sql = `INSERT INTO ${tableName} (${fields.join(', ')}) VALUES (${placeholders})`
+      const stmt = this.db.prepare(sql)
+      const values = fields.map(f => {
+        const value = snakeCaseItem[f]
+        // Convert boolean to integer for SQLite
+        if (typeof value === 'boolean') return value ? 1 : 0
+        return value
+      })
+
+      stmt.run(...values)
+      return item
+    } catch (error) {
+      throw new Error(`Failed to create in ${tableName}: ${error.message}`)
+    }
+  }
+
+  update (tableName, id, updates, idField = 'id') {
+    try {
+      const preparedUpdates = this.stringifyJsonFields(tableName, updates)
+      const snakeCaseUpdates = this.toSnakeCase(preparedUpdates)
+      const snakeIdField = idField.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`)
+      const fields = Object.keys(snakeCaseUpdates)
+      const setClause = fields.map(f => `${f} = ?`).join(', ')
+      const sql = `UPDATE ${tableName} SET ${setClause} WHERE ${snakeIdField} = ?`
+      const stmt = this.db.prepare(sql)
+      const values = fields.map(f => {
+        const value = snakeCaseUpdates[f]
+        // Convert boolean to integer for SQLite
+        if (typeof value === 'boolean') return value ? 1 : 0
+        return value
+      })
+      const params = [...values, id]
+      const result = stmt.run(...params)
+
+      if (result.changes === 0) {
+        throw new Error(`Item with ${idField}=${id} not found in ${tableName}`)
+      }
+
+      const updated = this.get(tableName, id, idField)
+      return updated
+    } catch (error) {
+      throw new Error(`Failed to update ${tableName}: ${error.message}`)
+    }
+  }
+
+  delete (tableName, id, idField = 'id') {
+    try {
+      const item = this.get(tableName, id, idField)
+      if (!item) {
+        throw new Error(`Item with ${idField}=${id} not found in ${tableName}`)
+      }
+
+      const snakeIdField = idField.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`)
+      const stmt = this.db.prepare(`DELETE FROM ${tableName} WHERE ${snakeIdField} = ?`)
+      stmt.run(id)
+      return item
+    } catch (error) {
+      throw new Error(`Failed to delete from ${tableName}: ${error.message}`)
+    }
+  }
+
+  clear (tableName) {
+    try {
+      this.db.prepare(`DELETE FROM ${tableName}`).run()
+    } catch (error) {
+      throw new Error(`Failed to clear ${tableName}: ${error.message}`)
+    }
+  }
+
+  clearAll () {
+    const tables = [
+      'pubsub_messages',
+      'pubsub_subscriptions',
+      'pubsub_topics',
+      'logging_entries',
+      'logging_logs',
+      'mqtt_subscriptions',
+      'mqtt_messages',
+      'mqtt_clients',
+      'firestore_metadata',
+      'webhook_routes',
+      'webhook_settings',
+      'http_traffic',
+      'pubsub_topic_registry',
+      'cloud_function_invocations',
+      'cloud_functions'
+    ]
+
+    this.db.transaction(() => {
+      for (const table of tables) {
+        try {
+          this.clear(table)
+        } catch (error) {
+          // Ignore if table doesn't exist
+        }
+      }
+    })()
+  }
+
+  transaction (fn) {
+    return this.db.transaction(fn)()
+  }
+
+  buildWhereClause (filter) {
+    const keys = Object.keys(filter)
+    if (keys.length === 0) {
+      return { sql: '', params: [] }
+    }
+
+    const snakeCaseFilter = this.toSnakeCase(filter)
+    const snakeKeys = Object.keys(snakeCaseFilter)
+
+    const conditions = snakeKeys.map(key => {
+      if (snakeCaseFilter[key] === null) {
+        return `${key} IS NULL`
+      }
+      return `${key} = ?`
+    })
+
+    const params = snakeKeys.map(key => snakeCaseFilter[key]).filter(v => v !== null)
+    return {
+      sql: `WHERE ${conditions.join(' AND ')}`,
+      params
+    }
+  }
+
+  parseJsonFields (tableName, row) {
+    const jsonFields = this.getJsonFields(tableName)
+    const parsed = { ...row }
+
+    for (const field of jsonFields) {
+      if (parsed[field] && typeof parsed[field] === 'string') {
+        try {
+          parsed[field] = JSON.parse(parsed[field])
+        } catch (error) {
+          // Keep as string if not valid JSON
+        }
+      }
+    }
+
+    return parsed
+  }
+
+  stringifyJsonFields (tableName, item) {
+    const stringified = { ...item }
+
+    // Stringify ALL object fields (except null)
+    for (const field in stringified) {
+      const value = stringified[field]
+      if (value !== null && typeof value === 'object' && !Buffer.isBuffer(value)) {
+        stringified[field] = JSON.stringify(value)
+      }
+    }
+
+    return stringified
+  }
+
+  getJsonFields (tableName) {
+    const jsonFieldsMap = {
+      pubsub_topics: ['labels', 'messageStoragePolicy', 'schemaSettings'],
+      pubsub_subscriptions: ['pushConfig', 'labels', 'expirationPolicy', 'deadLetterPolicy', 'retryPolicy'],
+      pubsub_messages: ['attributes'],
+      pubsub_message_history: ['attributes'],
+      logging_entries: ['resource', 'labels', 'jsonPayload', 'protoPayload', 'httpRequest', 'operation', 'sourceLocation'],
+      mqtt_clients: ['will'],
+      mqtt_messages: [],
+      mqtt_subscriptions: [],
+      logging_logs: [],
+      firestore_metadata: [],
+      webhook_routes: [],
+      webhook_settings: [],
+      http_traffic: ['queryParams', 'requestHeaders', 'responseHeaders', 'requestCookies', 'mockResponse'],
+      pubsub_topic_registry: [],
+      cloud_functions: ['triggerConfig', 'environmentVariables'],
+      cloud_function_invocations: ['cloudEvent']
+    }
+
+    return jsonFieldsMap[tableName] || []
+  }
+
+  toSnakeCase (obj) {
+    const result = {}
+    for (const key in obj) {
+      const snakeKey = key.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`)
+      result[snakeKey] = obj[key]
+    }
+    return result
+  }
+
+  toCamelCase (obj) {
+    const result = {}
+    for (const key in obj) {
+      const camelKey = key.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase())
+      result[camelKey] = obj[key]
+    }
+    return result
+  }
+
+  exportToJson (outputPath = null) {
+    const tables = [
+      'pubsub_topics',
+      'pubsub_subscriptions',
+      'pubsub_messages',
+      'logging_entries',
+      'logging_logs',
+      'mqtt_clients',
+      'mqtt_messages',
+      'mqtt_subscriptions',
+      'firestore_metadata',
+      'pubsub_topic_registry'
+    ]
+
+    const exportData = {}
+
+    for (const table of tables) {
+      try {
+        const stmt = this.db.prepare(`SELECT * FROM ${table}`)
+        const rows = stmt.all()
+        exportData[table] = rows.map(row => this.parseJsonFields(table, row))
+      } catch (error) {
+        exportData[table] = []
+      }
+    }
+
+    const finalPath = outputPath || path.join(Application.storage.dataDir, 'dev-tools-export.json')
+    fs.writeFileSync(finalPath, JSON.stringify(exportData, null, 2))
+
+    return {
+      success: true,
+      path: finalPath,
+      tables: Object.keys(exportData).length,
+      records: Object.values(exportData).reduce((sum, arr) => sum + arr.length, 0)
+    }
+  }
+
+  getStats () {
+    try {
+      const tables = this.db.prepare(`
+        SELECT name FROM sqlite_master
+        WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name != 'schema_migrations'
+        ORDER BY name
+      `).all()
+
+      const stats = {}
+
+      for (const { name } of tables) {
+        const countResult = this.db.prepare(`SELECT COUNT(*) as count FROM ${name}`).get()
+        stats[name] = countResult.count
+      }
+
+      const dbPath = this.db.name
+      let fileSize = 0
+      let fileSizeMb = '0.00'
+
+      if (dbPath !== ':memory:' && fs.existsSync(dbPath)) {
+        fileSize = fs.statSync(dbPath).size
+        fileSizeMb = (fileSize / 1024 / 1024).toFixed(2)
+      }
+
+      return {
+        file_size: fileSize,
+        file_size_mb: fileSizeMb,
+        tables: stats,
+        total_records: Object.values(stats).reduce((sum, count) => sum + count, 0)
+      }
+    } catch (error) {
+      throw new Error(`Failed to get stats: ${error.message}`)
+    }
+  }
+
+  shutdown () {
+    if (this.db) {
+      try {
+        this.db.pragma('wal_checkpoint(TRUNCATE)')
+      } catch (error) {
+        // Best-effort checkpoint before close
+      }
+      this.db.close()
+      this.db = null
+      this.isInitialized = false
+    }
+  }
+}
+
+export const SqliteStore = new SqliteStoreClass()