@gokiteam/goki-dev 0.2.0

package/src/api/pubsub/Logic.js

@@ -0,0 +1,732 @@
+import { SqliteStore } from '../../singletons/SqliteStore.js'
+import { Application } from '../../configs/Application.js'
+import { PUBSUB_MESSAGE_HISTORY, PUBSUB_TOPIC_REGISTRY } from '../../db/Tables.js'
+import { shadowSubscriptionManager } from '../../emulation/pubsub/ShadowSubscriptionManager.js'
+import { JSONPath } from 'jsonpath-plus'
+
+const { pubsub } = Application
+const PUBSUB_API = `http://${pubsub.emulatorHost}:${pubsub.emulatorPort}`
+const PROJECT_ID = pubsub.projectId
+
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+/**
+ * Decodes base64-encoded PubSub message data to a parsed object or string.
+ * The PubSub emulator stores data as base64-encoded JSON strings.
+ */
+const decodePubSubData = (data) => {
+  if (!data) return null
+  let decoded = data
+  try {
+    decoded = Buffer.from(data, 'base64').toString('utf-8')
+  } catch {
+    // not base64, use as-is
+  }
+  try {
+    return JSON.parse(decoded)
+  } catch {
+    return decoded
+  }
+}
+
+/**
+ * Decodes the data field for all messages in an array.
+ */
+const decodeMessages = (messages) =>
+  messages.map(msg => ({ ...msg, data: decodePubSubData(msg.data) }))
+
+const deepEqual = (a, b) => {
+  if (a === b) return true
+  if (a == null || b == null) return false
+  if (typeof a !== 'object' || typeof b !== 'object') return false
+  const keysA = Object.keys(a)
+  const keysB = Object.keys(b)
+  if (keysA.length !== keysB.length) return false
+  for (const key of keysA) {
+    if (!keysB.includes(key)) return false
+    if (!deepEqual(a[key], b[key])) return false
+  }
+  return true
+}
+
+export const Logic = {
+  async listMessages (params) {
+    return Logic.getHistory(params)
+  },
+
+  async listTopics (params) {
+    const { traceId } = params
+    try {
+      const response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics`)
+      const result = await response.json()
+      const topics = (result.topics || []).filter(t => !t.name.includes('-devtools-shadow'))
+      return {
+        topics,
+        total: topics.length,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  async listSubscriptions (params) {
+    const { traceId } = params
+    try {
+      const response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/subscriptions`)
+      const result = await response.json()
+      const subscriptions = (result.subscriptions || []).filter(s => !s.name.includes('-devtools-shadow'))
+      return {
+        subscriptions,
+        total: subscriptions.length,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  async createTopic (params) {
+    const { topicName, traceId } = params
+    try {
+      const response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics/${topicName}`, {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({})
+      })
+      const topic = await response.json()
+      return {
+        topic,
+        traceId
+      }
+    } catch (error) {
+      throw error
+    }
+  },
+
+  async deleteTopic (params) {
+    const { topicName, traceId } = params
+    try {
+      await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics/${topicName}`, {
+        method: 'DELETE'
+      })
+      return {
+        message: 'Topic deleted',
+        traceId
+      }
+    } catch (error) {
+      throw error
+    }
+  },
+
+  async createSubscription (params) {
+    const { topicName, subscriptionName, traceId } = params
+    try {
+      const response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/subscriptions/${subscriptionName}`, {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ topic: `projects/${PROJECT_ID}/topics/${topicName}` })
+      })
+      const subscription = await response.json()
+      return {
+        subscription,
+        traceId
+      }
+    } catch (error) {
+      throw error
+    }
+  },
+
+  async deleteSubscription (params) {
+    const { subscriptionName, traceId } = params
+    try {
+      await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/subscriptions/${subscriptionName}`, {
+        method: 'DELETE'
+      })
+      return {
+        message: 'Subscription deleted',
+        traceId
+      }
+    } catch (error) {
+      throw error
+    }
+  },
+
+  async publishMessage (params) {
+    const { topicName, message, attributes, traceId } = params
+    try {
+      let messageString
+      if (typeof message === 'string') {
+        messageString = message
+      } else if (Buffer.isBuffer(message)) {
+        messageString = message.toString()
+      } else if (typeof message === 'object') {
+        messageString = JSON.stringify(message)
+      } else {
+        messageString = String(message)
+      }
+      const messageData = Buffer.from(messageString).toString('base64')
+      // Try to publish
+      let response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics/${topicName}:publish`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          messages: [{ data: messageData, attributes: attributes || {} }]
+        })
+      })
+      let result = await response.json()
+      // Auto-create topic if not found
+      if (response.status === 404 || result.error?.code === 404) {
+        const fullTopicName = `projects/${PROJECT_ID}/topics/${topicName}`
+        const shadowSubName = `${topicName}-devtools-shadow`
+        const fullShadowSubName = `projects/${PROJECT_ID}/subscriptions/${shadowSubName}`
+        // Create topic
+        await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics/${topicName}`, {
+          method: 'PUT',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({})
+        })
+        // Create shadow subscription for message capture
+        await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/subscriptions/${shadowSubName}`, {
+          method: 'PUT',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ topic: fullTopicName })
+        })
+        // Register with shadow subscription manager so poller captures messages
+        shadowSubscriptionManager.shadowSubscriptions.add(fullShadowSubName)
+        shadowSubscriptionManager.shadowToTopic.set(fullShadowSubName, fullTopicName)
+        // Retry publish
+        response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/topics/${topicName}:publish`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            messages: [{ data: messageData, attributes: attributes || {} }]
+          })
+        })
+        result = await response.json()
+      }
+      if (!response.ok || result.error) {
+        throw new Error(result.error?.message || `Failed to publish: ${response.status}`)
+      }
+      return {
+        messageIds: result.messageIds,
+        traceId
+      }
+    } catch (error) {
+      throw error
+    }
+  },
+
+  async pullMessages (params) {
+    const { subscriptionName, maxMessages = 10, traceId } = params
+    try {
+      const response = await fetch(`${PUBSUB_API}/v1/projects/${PROJECT_ID}/subscriptions/${subscriptionName}:pull`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ maxMessages, returnImmediately: true })
+      })
+      const result = await response.json()
+      return {
+        messages: result.receivedMessages || [],
+        traceId
+      }
+    } catch (error) {
+      throw error
+    }
+  },
+
+  /**
+   * Get message history with filters and view tracking
+   */
+  async getHistory (params) {
+    const { filter = {}, page = {}, traceId } = params
+    try {
+      const limit = page.limit || 50
+      const offset = page.offset || 0
+
+      // Build SQL query with all filters in WHERE clause for correct pagination
+      let sql = `SELECT * FROM ${PUBSUB_MESSAGE_HISTORY}`
+      let countSql = `SELECT COUNT(*) as count FROM ${PUBSUB_MESSAGE_HISTORY}`
+      const conditions = []
+      const queryParams = []
+
+      if (filter.topic) {
+        conditions.push('topic = ?')
+        queryParams.push(filter.topic)
+      }
+      if (filter.sender) {
+        conditions.push('sender = ?')
+        queryParams.push(filter.sender)
+      }
+      if (filter.before) {
+        conditions.push('publish_time < ?')
+        queryParams.push(filter.before)
+      }
+      if (filter.timeRange) {
+        if (filter.timeRange.start) {
+          conditions.push('publish_time >= ?')
+          queryParams.push(filter.timeRange.start)
+        }
+        if (filter.timeRange.end) {
+          conditions.push('publish_time <= ?')
+          queryParams.push(filter.timeRange.end)
+        }
+      }
+
+      if (conditions.length > 0) {
+        const whereClause = ` WHERE ${conditions.join(' AND ')}`
+        sql += whereClause
+        countSql += whereClause
+      }
+
+      sql += ' ORDER BY publish_time DESC'
+      sql += ` LIMIT ${limit} OFFSET ${offset}`
+
+      const totalResult = SqliteStore.db.prepare(countSql).get(...queryParams)
+      const total = totalResult.count
+
+      let data = SqliteStore.db.prepare(sql).all(...queryParams)
+      // Convert snake_case → camelCase and parse JSON fields (same as SqliteStore.list)
+      data = data.map(row => {
+        const camel = {}
+        for (const key in row) {
+          const camelKey = key.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase())
+          camel[camelKey] = row[key]
+        }
+        // Parse JSON fields
+        if (camel.attributes && typeof camel.attributes === 'string') {
+          try { camel.attributes = JSON.parse(camel.attributes) } catch {}
+        }
+        return camel
+      })
+
+      const result = { data, total }
+
+      // Increment viewCount for all returned messages
+      for (const msg of result.data) {
+        SqliteStore.update(PUBSUB_MESSAGE_HISTORY, msg.id, {
+          viewCount: (msg.viewCount || 0) + 1,
+          lastViewedAt: Date.now()
+        })
+      }
+
+      return {
+        messages: decodeMessages(result.data),
+        total: result.total,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Auto-suggest topics based on query
+   */
+  async suggestTopics (params) {
+    const { query = '', limit = 10, traceId } = params
+    try {
+      // Get all distinct topics from history with message counts
+      const sql = `
+        SELECT
+          topic as name,
+          COUNT(*) as messageCount,
+          MAX(publish_time) as lastPublishTime
+        FROM ${PUBSUB_MESSAGE_HISTORY}
+        WHERE topic LIKE ?
+        GROUP BY topic
+        ORDER BY messageCount DESC
+        LIMIT ?
+      `
+
+      const topics = SqliteStore.db.prepare(sql).all(`%${query}%`, limit)
+
+      return {
+        topics,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Search messages by content
+   */
+  async searchMessages (params) {
+    const { query, searchIn = ['data'], filter = {}, page = {}, traceId } = params
+    try {
+      const options = {
+        limit: page.limit || 50,
+        offset: page.offset || 0
+      }
+
+      // Build search predicate
+      options.filter = (msg) => {
+        // Apply additional filters first
+        if (filter.topic && msg.topic !== filter.topic) {
+          return false
+        }
+        if (filter.sender && msg.sender !== filter.sender) {
+          return false
+        }
+
+        // Search in specified fields
+        const queryLower = query.toLowerCase()
+
+        if (searchIn.includes('data')) {
+          const dataStr = typeof msg.data === 'string' ? msg.data : JSON.stringify(msg.data || {})
+          if (dataStr.toLowerCase().includes(queryLower)) {
+            return true
+          }
+        }
+
+        if (searchIn.includes('attributes')) {
+          const attrStr = JSON.stringify(msg.attributes || {})
+          if (attrStr.toLowerCase().includes(queryLower)) {
+            return true
+          }
+        }
+
+        return false
+      }
+
+      const result = SqliteStore.list(PUBSUB_MESSAGE_HISTORY, options)
+
+      return {
+        messages: decodeMessages(result.data),
+        total: result.total,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Clear message history (LRU cleanup)
+   */
+  async clearHistory (params) {
+    const { keepCount, topic, traceId } = params
+    try {
+      let deletedCount = 0
+
+      if (keepCount) {
+        // Delete least viewed messages beyond keepCount
+        const minKeep = 100 // Always keep minimum 100 messages
+
+        // Get current count
+        let countSql = `SELECT COUNT(*) as count FROM ${PUBSUB_MESSAGE_HISTORY}`
+        if (topic) {
+          countSql += ' WHERE topic = ?'
+        }
+        const stmt = topic ? SqliteStore.db.prepare(countSql).get(topic) : SqliteStore.db.prepare(countSql).get()
+        const currentCount = stmt.count
+
+        if (currentCount > keepCount && currentCount - keepCount >= minKeep) {
+          const deleteCount = currentCount - keepCount
+
+          // Delete least viewed messages (LRU)
+          let deleteSql = `
+            DELETE FROM ${PUBSUB_MESSAGE_HISTORY}
+            WHERE id IN (
+              SELECT id FROM ${PUBSUB_MESSAGE_HISTORY}
+          `
+          if (topic) {
+            deleteSql += ' WHERE topic = ?'
+          }
+          deleteSql += `
+              ORDER BY view_count ASC, created_at ASC
+              LIMIT ?
+            )
+          `
+
+          const result = topic
+            ? SqliteStore.db.prepare(deleteSql).run(topic, deleteCount)
+            : SqliteStore.db.prepare(deleteSql).run(deleteCount)
+          deletedCount = result.changes
+        }
+      } else if (topic) {
+        // Delete all messages for specific topic
+        const messages = SqliteStore.find(PUBSUB_MESSAGE_HISTORY, { topic })
+        for (const msg of messages) {
+          SqliteStore.delete(PUBSUB_MESSAGE_HISTORY, msg.id)
+          deletedCount++
+        }
+      } else {
+        // Clear ALL messages when no filters provided
+        const deleteSql = `DELETE FROM ${PUBSUB_MESSAGE_HISTORY}`
+        const result = SqliteStore.db.prepare(deleteSql).run()
+        deletedCount = result.changes
+      }
+
+      return {
+        deletedCount,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Get message history statistics
+   */
+  async getHistoryStats (params) {
+    const { traceId } = params
+    try {
+      // Total messages
+      const totalSql = `SELECT COUNT(*) as count FROM ${PUBSUB_MESSAGE_HISTORY}`
+      const totalResult = SqliteStore.db.prepare(totalSql).get()
+      const totalMessages = totalResult.count
+
+      // Total unique topics
+      const topicsSql = `SELECT COUNT(DISTINCT topic) as count FROM ${PUBSUB_MESSAGE_HISTORY}`
+      const topicsResult = SqliteStore.db.prepare(topicsSql).get()
+      const totalTopics = topicsResult.count
+
+      // Oldest and newest message times
+      const timesSql = `
+        SELECT
+          MIN(publish_time) as oldest,
+          MAX(publish_time) as newest
+        FROM ${PUBSUB_MESSAGE_HISTORY}
+      `
+      const timesResult = SqliteStore.db.prepare(timesSql).get()
+
+      // Top 10 topics by message count
+      const topTopicsSql = `
+        SELECT
+          topic,
+          COUNT(*) as count
+        FROM ${PUBSUB_MESSAGE_HISTORY}
+        GROUP BY topic
+        ORDER BY count DESC
+        LIMIT 10
+      `
+      const topTopics = SqliteStore.db.prepare(topTopicsSql).all()
+
+      return {
+        totalMessages,
+        totalTopics,
+        oldestMessage: timesResult.oldest,
+        newestMessage: timesResult.newest,
+        topTopics,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Wait for a message matching filter criteria
+   */
+  async waitForMessage (params) {
+    const { filter, timeout = 5000, traceId } = params
+    const startTime = Date.now()
+    const pollInterval = 100
+    try {
+      while (Date.now() - startTime < timeout) {
+        const historyResult = await Logic.getHistory({
+          filter: {
+            topic: filter.topic,
+            sender: filter.sender
+          },
+          page: { limit: 50 },
+          traceId
+        })
+        if (historyResult.messages && historyResult.messages.length > 0) {
+          for (const message of historyResult.messages) {
+            if (!filter.predicate) {
+              return {
+                message,
+                foundAt: Date.now() - startTime,
+                traceId
+              }
+            }
+            const { jsonPath, operator, value } = filter.predicate
+            const messageData = typeof message.data === 'object' ? message.data : decodePubSubData(message.data)
+            const results = JSONPath({ path: jsonPath, json: messageData })
+            if (results && results.length > 0) {
+              const actualValue = results[0]
+              let matches = false
+              switch (operator) {
+                case 'equals':
+                  matches = deepEqual(actualValue, value)
+                  break
+                case 'notEquals':
+                  matches = !deepEqual(actualValue, value)
+                  break
+                case 'contains':
+                  matches = String(actualValue).includes(String(value))
+                  break
+                case 'greaterThan':
+                  matches = actualValue > value
+                  break
+                case 'lessThan':
+                  matches = actualValue < value
+                  break
+              }
+              if (matches) {
+                return {
+                  message,
+                  foundAt: Date.now() - startTime,
+                  traceId
+                }
+              }
+            }
+          }
+        }
+        await sleep(pollInterval)
+      }
+      return {
+        status: 'error',
+        message: 'Timeout waiting for message',
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Assert that a message was published with specific criteria
+   */
+  async assertMessagePublished (params) {
+    const { filter, traceId } = params
+    try {
+      const historyFilter = {
+        topic: filter.topic
+      }
+      if (filter.since) {
+        historyFilter.timeRange = {
+          start: filter.since,
+          end: new Date().toISOString()
+        }
+      }
+      const historyResult = await Logic.getHistory({
+        filter: historyFilter,
+        page: { limit: 1000 },
+        traceId
+      })
+      let matches = []
+      if (historyResult.messages && historyResult.messages.length > 0) {
+        matches = historyResult.messages.filter(msg => {
+          if (filter.dataContains) {
+            const dataStr = typeof msg.data === 'string' ? msg.data : JSON.stringify(msg.data)
+            if (!dataStr.includes(filter.dataContains)) {
+              return false
+            }
+          }
+          if (filter.attributesMatch) {
+            for (const [key, value] of Object.entries(filter.attributesMatch)) {
+              if (msg.attributes?.[key] !== value) {
+                return false
+              }
+            }
+          }
+          return true
+        })
+      }
+      return {
+        found: matches.length > 0,
+        count: matches.length,
+        firstMatch: matches[0] || null,
+        traceId
+      }
+    } catch (error) {
+      return {
+        status: 'error',
+        message: error.message,
+        traceId
+      }
+    }
+  },
+
+  /**
+   * Register topics and subscriptions in the persistent registry
+   */
+  async registerTopics (params) {
+    const { projectName, topics, traceId } = params
+    const normalized = topics.map(t => typeof t === 'string' ? { name: t } : t)
+    const registered = []
+    for (const entry of normalized) {
+      SqliteStore.db.prepare(`
+        INSERT OR REPLACE INTO ${PUBSUB_TOPIC_REGISTRY} (topic_name, subscription_name, project_name, registered_at)
+        VALUES (?, NULL, ?, unixepoch())
+      `).run(entry.name, projectName)
+      registered.push({ topicName: entry.name, subscriptionName: null })
+      if (entry.subscription) {
+        SqliteStore.db.prepare(`
+          INSERT OR REPLACE INTO ${PUBSUB_TOPIC_REGISTRY} (topic_name, subscription_name, project_name, registered_at)
+          VALUES (?, ?, ?, unixepoch())
+        `).run(entry.name, entry.subscription, projectName)
+        registered.push({ topicName: entry.name, subscriptionName: entry.subscription })
+      }
+    }
+    return { registered, count: registered.length, traceId }
+  },
+
+  /**
+   * List registered topics from the persistent registry
+   */
+  async listRegistry (params) {
+    const { projectName, traceId } = params
+    let entries
+    if (projectName) {
+      entries = SqliteStore.find(PUBSUB_TOPIC_REGISTRY, { projectName })
+    } else {
+      entries = SqliteStore.list(PUBSUB_TOPIC_REGISTRY, { orderBy: 'topic_name ASC, project_name ASC' }).data
+    }
+    return { entries, total: entries.length, traceId }
+  },
+
+  /**
+   * Unregister topics for a project from the persistent registry
+   */
+  async unregisterTopics (params) {
+    const { projectName, topicNames, traceId } = params
+    let removed = 0
+    for (const topicName of topicNames) {
+      const result = SqliteStore.db.prepare(`
+        DELETE FROM ${PUBSUB_TOPIC_REGISTRY}
+        WHERE topic_name = ? AND project_name = ?
+      `).run(topicName, projectName)
+      removed += result.changes
+    }
+    return { removed, traceId }
+  }
+}