@gokiteam/goki-dev 0.2.0

package/src/jobs/MessageHistoryCleanup.js

@@ -0,0 +1,152 @@
+// LRU cleanup job for message history
+// Runs periodically to maintain size limits while keeping frequently viewed messages
+
+import { SqliteStore } from '../singletons/SqliteStore.js'
+import { Application } from '../configs/Application.js'
+import { Logger } from '../singletons/Logger.js'
+import { PUBSUB_MESSAGE_HISTORY } from '../db/Tables.js'
+
+/**
+ * Run the LRU cleanup process
+ * Deletes least viewed messages when approaching size limit
+ * @returns {Object} - { cleaned: boolean, deletedCount?: number, reason?: string }
+ */
+export const runMessageHistoryCleanup = () => {
+  const { maxMessages, minMessages, cleanupThreshold } = Application.messageHistory
+
+  // Get current count
+  const countResult = SqliteStore.db.prepare(`SELECT COUNT(*) as count FROM ${PUBSUB_MESSAGE_HISTORY}`).get()
+  const count = countResult.count
+
+  // Only cleanup if above threshold
+  if (count < maxMessages * cleanupThreshold) {
+    Logger.log({
+      level: 'debug',
+      message: 'Message history cleanup skipped - below threshold',
+      data: { currentCount: count, threshold: maxMessages * cleanupThreshold }
+    })
+    return { cleaned: false, reason: 'below threshold' }
+  }
+
+  // Calculate how many to delete (target 80% capacity)
+  const targetCount = Math.floor(maxMessages * 0.8)
+  const deleteCount = Math.max(0, count - targetCount)
+
+  // Safety check: ensure we don't delete below minimum
+  if (deleteCount === 0 || count - deleteCount < minMessages) {
+    Logger.log({
+      level: 'debug',
+      message: 'Message history cleanup skipped - minimum threshold',
+      data: { currentCount: count, minMessages, targetCount }
+    })
+    return { cleaned: false, reason: 'minimum threshold' }
+  }
+
+  // Delete least viewed messages (LRU strategy)
+  // Order by: view_count ASC (least viewed first), created_at ASC (oldest first)
+  const deleteSql = `
+    DELETE FROM ${PUBSUB_MESSAGE_HISTORY}
+    WHERE id IN (
+      SELECT id FROM ${PUBSUB_MESSAGE_HISTORY}
+      ORDER BY view_count ASC, created_at ASC
+      LIMIT ?
+    )
+  `
+
+  try {
+    const deleted = SqliteStore.db.prepare(deleteSql).run(deleteCount)
+
+    Logger.log({
+      level: 'info',
+      message: 'Message history LRU cleanup completed',
+      data: {
+        totalCount: count,
+        deletedCount: deleted.changes,
+        remainingCount: count - deleted.changes,
+        targetCount,
+        maxMessages,
+        minMessages
+      }
+    })
+
+    return { cleaned: true, deletedCount: deleted.changes }
+  } catch (error) {
+    Logger.log({
+      level: 'error',
+      message: 'Message history cleanup failed',
+      data: { error: error.message, count, deleteCount }
+    })
+    throw error
+  }
+}
+
+/**
+ * Check if cleanup is needed and run if necessary
+ * Used for on-demand cleanup (e.g., after publish)
+ * @returns {Object} - Cleanup result
+ */
+export const checkAndCleanup = () => {
+  const { maxMessages, cleanupThreshold } = Application.messageHistory
+
+  const countResult = SqliteStore.db.prepare(`SELECT COUNT(*) as count FROM ${PUBSUB_MESSAGE_HISTORY}`).get()
+  const count = countResult.count
+
+  if (count >= maxMessages * cleanupThreshold) {
+    return runMessageHistoryCleanup()
+  }
+
+  return { cleaned: false, reason: 'below threshold' }
+}
+
+/**
+ * Start the scheduled cleanup job
+ * @returns {NodeJS.Timer} - Interval timer
+ */
+export const startCleanupJob = () => {
+  const { cleanupIntervalHours } = Application.messageHistory
+  const intervalMs = cleanupIntervalHours * 60 * 60 * 1000
+
+  Logger.log({
+    level: 'info',
+    message: 'Starting message history cleanup job',
+    data: { intervalHours: cleanupIntervalHours }
+  })
+
+  // Run immediately on start
+  try {
+    runMessageHistoryCleanup()
+  } catch (error) {
+    Logger.log({
+      level: 'error',
+      message: 'Initial cleanup failed',
+      data: { error: error.message }
+    })
+  }
+
+  // Schedule periodic cleanup
+  return setInterval(() => {
+    try {
+      runMessageHistoryCleanup()
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Scheduled cleanup failed',
+        data: { error: error.message }
+      })
+    }
+  }, intervalMs)
+}
+
+/**
+ * Stop the cleanup job
+ * @param {NodeJS.Timer} timer - The interval timer to clear
+ */
+export const stopCleanupJob = (timer) => {
+  if (timer) {
+    clearInterval(timer)
+    Logger.log({
+      level: 'info',
+      message: 'Message history cleanup job stopped'
+    })
+  }
+}

package/src/mcp/ApiClient.js

@@ -0,0 +1,25 @@
+export class ApiClient {
+  constructor (baseUrl) {
+    this.baseUrl = baseUrl
+  }
+
+  async post (path, body = {}) {
+    const url = `${this.baseUrl}${path}`
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body)
+    })
+    const result = await response.json()
+    if (!result.success) {
+      throw new Error(result.data?.message || result.message || 'API call failed')
+    }
+    return result.data
+  }
+
+  async get (path) {
+    const url = `${this.baseUrl}${path}`
+    const response = await fetch(url)
+    return response.json()
+  }
+}

package/src/mcp/Server.js

@@ -0,0 +1,52 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
+import { ApiClient } from './ApiClient.js'
+import { registerPubsubTools } from './tools/pubsub.js'
+import { registerLoggingTools } from './tools/logging.js'
+import { registerFirestoreTools } from './tools/firestore.js'
+import { registerPostgresTools } from './tools/postgres.js'
+import { registerRedisTools } from './tools/redis.js'
+import { registerDockerTools } from './tools/docker.js'
+import { registerMqttTools } from './tools/mqtt.js'
+import { registerServicesTools } from './tools/services.js'
+import { registerHttpTrafficTools } from './tools/httpTraffic.js'
+import { registerDataTools } from './tools/data.js'
+import { registerWebhooksTools } from './tools/webhooks.js'
+import { registerSnapshotTools } from './tools/snapshots.js'
+import { registerFunctionsTools } from './tools/functions.js'
+import { registerPlatformResources } from './resources/platform.js'
+import { registerDebuggingPrompts } from './prompts/debugging.js'
+
+export async function startMcpServer ({ devToolsUrl }) {
+  const server = new McpServer({
+    name: 'goki-dev-tools',
+    version: '1.0.0'
+  })
+
+  const apiClient = new ApiClient(devToolsUrl)
+
+  // Register tool groups
+  registerPubsubTools(server, apiClient)
+  registerLoggingTools(server, apiClient)
+  registerFirestoreTools(server, apiClient)
+  registerPostgresTools(server, apiClient)
+  registerRedisTools(server, apiClient)
+  registerDockerTools(server, apiClient)
+  registerMqttTools(server, apiClient)
+  registerServicesTools(server, apiClient)
+  registerHttpTrafficTools(server, apiClient)
+  registerDataTools(server, apiClient)
+  registerWebhooksTools(server, apiClient)
+  registerSnapshotTools(server, apiClient)
+  registerFunctionsTools(server, apiClient)
+
+  // Register resources and prompts
+  registerPlatformResources(server, apiClient)
+  registerDebuggingPrompts(server)
+
+  // Connect via stdio transport
+  const transport = new StdioServerTransport()
+  await server.connect(transport)
+  console.error('Goki Dev Tools MCP server running on stdio')
+  console.error(`Backend URL: ${devToolsUrl}`)
+}

package/src/mcp/prompts/debugging.js

@@ -0,0 +1,104 @@
+export function registerDebuggingPrompts (server) {
+  server.prompt(
+    'debug-pubsub',
+    'Debug Pub/Sub message flow — checks topics, subscriptions, and recent messages',
+    { topicName: { type: 'string', description: 'Optional topic name to focus on' } },
+    async ({ topicName }) => ({
+      messages: [{
+        role: 'user',
+        content: {
+          type: 'text',
+          text: [
+            'Debug Pub/Sub message flow in the Goki dev-tools platform.',
+            topicName ? `Focus on topic: ${topicName}` : '',
+            '',
+            'Steps:',
+            '1. List all Pub/Sub topics and subscriptions',
+            '2. Check message history stats for overall volume',
+            topicName
+              ? `3. Search recent messages on topic "${topicName}"`
+              : '3. Show recent messages across all topics',
+            '4. Identify any issues (missing subscriptions, undelivered messages)',
+            '5. Suggest fixes if problems are found'
+          ].filter(Boolean).join('\n')
+        }
+      }]
+    })
+  )
+
+  server.prompt(
+    'monitor-services',
+    'Check health of all platform services and Docker containers',
+    {},
+    async () => ({
+      messages: [{
+        role: 'user',
+        content: {
+          type: 'text',
+          text: [
+            'Check the health of all Goki dev-tools platform services:',
+            '',
+            '1. Get platform dashboard stats',
+            '2. List all Docker containers and their status',
+            '3. Check the gateway status',
+            '4. Test Redis connection',
+            '5. Test PostgreSQL connection',
+            '6. Check scheduler tick status',
+            '',
+            'Report any services that are down or unhealthy, and suggest remediation steps.'
+          ].join('\n')
+        }
+      }]
+    })
+  )
+
+  server.prompt(
+    'trace-request',
+    'Trace a request through multiple services using a traceId',
+    { traceId: { type: 'string', description: 'The trace ID to follow across services' } },
+    async ({ traceId }) => ({
+      messages: [{
+        role: 'user',
+        content: {
+          type: 'text',
+          text: [
+            `Trace request with traceId "${traceId}" through the Goki platform:`,
+            '',
+            '1. Search log entries for this traceId',
+            '2. Search HTTP traffic records for this traceId',
+            '3. Search Pub/Sub message history for this traceId in attributes',
+            '4. Build a timeline of events across services',
+            '5. Identify any errors, delays, or gaps in the flow'
+          ].join('\n')
+        }
+      }]
+    })
+  )
+
+  server.prompt(
+    'setup-test-environment',
+    'Prepare the dev-tools platform for a fresh test run',
+    { services: { type: 'string', description: 'Comma-separated service names to clear, or "all"' } },
+    async ({ services }) => ({
+      messages: [{
+        role: 'user',
+        content: {
+          type: 'text',
+          text: [
+            'Prepare the Goki dev-tools platform for a fresh test run:',
+            '',
+            services && services !== 'all'
+              ? `1. Clear data for these services: ${services}`
+              : '1. Clear all platform data',
+            '2. Verify all Docker containers are running',
+            '3. Confirm Pub/Sub emulator is ready (list topics)',
+            '4. Confirm logging is ready (list services)',
+            '5. Trigger a scheduler tick to initialize periodic tasks',
+            '',
+            'Report readiness status when done.'
+          ].join('\n')
+        }
+      }]
+    })
+  )
+}

package/src/mcp/resources/platform.js

@@ -0,0 +1,118 @@
+import { readFile } from 'fs/promises'
+import { join, dirname } from 'path'
+import { fileURLToPath } from 'url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const projectRoot = join(__dirname, '..', '..', '..')
+
+async function readProjectFile (relativePath) {
+  try {
+    return await readFile(join(projectRoot, relativePath), 'utf-8')
+  } catch {
+    return `File not found: ${relativePath}`
+  }
+}
+
+export function registerPlatformResources (server, apiClient) {
+  server.resource(
+    'platform-stats',
+    'devtools://stats',
+    async (uri) => {
+      try {
+        const data = await apiClient.post('/v1/dashboard/stats')
+        return {
+          contents: [{
+            uri: uri.href,
+            mimeType: 'application/json',
+            text: JSON.stringify(data, null, 2)
+          }]
+        }
+      } catch (error) {
+        return {
+          contents: [{
+            uri: uri.href,
+            mimeType: 'text/plain',
+            text: `Error fetching stats: ${error.message}`
+          }]
+        }
+      }
+    }
+  )
+
+  server.resource(
+    'architecture',
+    'devtools://docs/architecture',
+    async (uri) => {
+      const text = await readProjectFile('CLAUDE.md')
+      return {
+        contents: [{
+          uri: uri.href,
+          mimeType: 'text/markdown',
+          text
+        }]
+      }
+    }
+  )
+
+  server.resource(
+    'port-registry',
+    'devtools://config/ports',
+    async (uri) => {
+      return {
+        contents: [{
+          uri: uri.href,
+          mimeType: 'application/json',
+          text: JSON.stringify({
+            management: { port: 9000, description: 'Dev Tools API + Web UI' },
+            pubsub: { port: 8085, description: 'GCP Pub/Sub emulator' },
+            awsIot: { port: 8086, description: 'AWS IoT Core HTTPS API' },
+            logging: { port: 8087, description: 'Google Cloud Logging emulator' },
+            mqtt: { port: 8883, description: 'MQTT broker (Aedes)' },
+            firestore: { port: 8080, description: 'GCP Firestore emulator' },
+            postgres: { port: 5432, description: 'PostgreSQL database' },
+            redis: { port: 6379, description: 'Redis cache' },
+            redisLogs: { port: 6380, description: 'Redis for logs (LRU)' },
+            elasticsearch: { port: 9200, description: 'Elasticsearch' },
+            kibana: { port: 5601, description: 'Kibana dashboard' }
+          }, null, 2)
+        }]
+      }
+    }
+  )
+
+  server.resource(
+    'topic-conventions',
+    'devtools://docs/topic-conventions',
+    async (uri) => {
+      return {
+        contents: [{
+          uri: uri.href,
+          mimeType: 'text/markdown',
+          text: [
+            '# Goki MQTT & Pub/Sub Topic Conventions',
+            '',
+            '## MQTT Topic Patterns',
+            '- **Request:** `rq/{action}/{clientId}`',
+            '- **Response:** `rs/{action}/{clientId}`',
+            '- **Notification:** `nf/{action}/{clientId}`',
+            '- **Will:** `w/{action}/{clientId}`',
+            '- **Lifecycle:** `simulator/connected`, `simulator/disconnected`',
+            '',
+            '## Pub/Sub Bridge',
+            '- MQTT messages are bridged to Pub/Sub topic `mqttMessageReceived`',
+            '- Format: `{ clientId, topicName, packet (base64), receivedAtMilliseconds }`',
+            '',
+            '## Common Pub/Sub Topics',
+            '- `mqttMessageReceived` — MQTT messages bridged to Pub/Sub',
+            '- `systemOneMinuteTick` — Scheduler tick every 60 seconds',
+            '',
+            '## Inter-Service Communication',
+            '- Services publish/subscribe via the Pub/Sub emulator on port 8085',
+            '- Backend-to-device messages go through AWS IoT HTTPS API on port 8086',
+            '- Device-to-backend messages go through MQTT broker on port 8883'
+          ].join('\n')
+        }]
+      }
+    }
+  )
+}

package/src/mcp/tools/data.js

@@ -0,0 +1,84 @@
+import { z } from 'zod'
+
+export function registerDataTools (server, apiClient) {
+  server.tool(
+    'platform_stats',
+    'Get platform dashboard statistics (topic count, message count, log entries, MQTT clients, etc.)',
+    {},
+    async () => {
+      try {
+        const data = await apiClient.post('/v1/dashboard/stats')
+        return { content: [{ type: 'text', text: JSON.stringify(data, null, 2) }] }
+      } catch (error) {
+        return { content: [{ type: 'text', text: `Failed to get platform stats: ${error.message}` }], isError: true }
+      }
+    }
+  )
+
+  server.tool(
+    'platform_export',
+    'Export all platform data (topics, subscriptions, messages, logs, etc.)',
+    {},
+    async () => {
+      try {
+        const data = await apiClient.post('/v1/data/export')
+        return { content: [{ type: 'text', text: JSON.stringify(data, null, 2) }] }
+      } catch (error) {
+        return { content: [{ type: 'text', text: `Failed to export platform data: ${error.message}` }], isError: true }
+      }
+    }
+  )
+
+  server.tool(
+    'platform_import',
+    'Import platform data (optionally clearing existing data first)',
+    {
+      data: z.record(z.any()).describe('Platform data object to import'),
+      clearExisting: z.boolean().optional().describe('Whether to clear existing data before importing')
+    },
+    async ({ data, clearExisting }) => {
+      try {
+        const body = { data }
+        if (clearExisting !== undefined) body.clearExisting = clearExisting
+        const result = await apiClient.post('/v1/data/import', body)
+        return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] }
+      } catch (error) {
+        return { content: [{ type: 'text', text: `Failed to import platform data: ${error.message}` }], isError: true }
+      }
+    }
+  )
+
+  server.tool(
+    'platform_clear',
+    'Clear all platform data (topics, messages, logs, traffic records)',
+    {},
+    async () => {
+      try {
+        const data = await apiClient.post('/v1/data/clear')
+        return { content: [{ type: 'text', text: JSON.stringify(data, null, 2) }] }
+      } catch (error) {
+        return { content: [{ type: 'text', text: `Failed to clear platform data: ${error.message}` }], isError: true }
+      }
+    }
+  )
+
+  server.tool(
+    'platform_clear_services',
+    'Clear data for specific services. Useful for resetting test state without clearing everything.',
+    {
+      services: z.array(z.string()).optional().describe('Optional list of service names to clear'),
+      keepSystemData: z.boolean().optional().describe('Whether to keep system/infrastructure data')
+    },
+    async ({ services, keepSystemData }) => {
+      try {
+        const body = {}
+        if (services !== undefined) body.services = services
+        if (keepSystemData !== undefined) body.keepSystemData = keepSystemData
+        const data = await apiClient.post('/v1/data/clear-services', body)
+        return { content: [{ type: 'text', text: JSON.stringify(data, null, 2) }] }
+      } catch (error) {
+        return { content: [{ type: 'text', text: `Failed to clear service data: ${error.message}` }], isError: true }
+      }
+    }
+  )
+}