@gokiteam/goki-dev 0.2.0

package/src/singletons/FunctionsService.js

@@ -0,0 +1,418 @@
+import { v4 as uuid } from 'uuid'
+import path from 'path'
+import chokidar from 'chokidar'
+import { FunctionRunner } from '../runtime/FunctionRunner.js'
+import { SqliteStore } from './SqliteStore.js'
+import { LogBroadcaster } from './LogBroadcaster.js'
+import { Logger } from './Logger.js'
+import { Application } from '../configs/Application.js'
+import { CLOUD_FUNCTIONS, LOGGING_ENTRIES } from '../db/Tables.js'
+import { FunctionStatuses } from '../enums/FunctionStatuses.js'
+
+const { functions } = Application
+
+/**
+ * Process manager singleton for Cloud Functions.
+ * Manages all FunctionRunner instances, port allocation, and SSE broadcasting.
+ */
+class FunctionsServiceClass {
+  constructor () {
+    this.runners = new Map()
+    this.sseClients = new Map()
+    this.invocationSseClients = new Map()
+    this.fileChangeClients = new Map()
+    this.fileWatchers = new Map()
+    this.portPool = new Set()
+    this.heartbeatId = null
+    this.heartbeatIntervalMs = 30000
+    const basePort = functions.basePort || 9100
+    const maxInstances = functions.maxInstances || 50
+    for (let i = 0; i < maxInstances; i++) {
+      this.portPool.add(basePort + i)
+    }
+  }
+
+  async initialize () {
+    try {
+      const enabledFunctions = SqliteStore.find(CLOUD_FUNCTIONS, { enabled: 1 })
+      for (const fn of enabledFunctions) {
+        if (fn.status === FunctionStatuses.RUNNING || fn.status === FunctionStatuses.STARTING) {
+          SqliteStore.update(CLOUD_FUNCTIONS, fn.id, { status: FunctionStatuses.STOPPED, pid: null })
+        }
+      }
+      for (const fn of enabledFunctions) {
+        try {
+          await this.deployFunction(fn)
+        } catch (error) {
+          Logger.log({
+            level: 'error',
+            message: `Failed to start function on initialize: ${fn.name}`,
+            data: { error: error.message }
+          })
+        }
+      }
+      Logger.log({
+        level: 'info',
+        message: 'FunctionsService initialized',
+        data: { functionCount: enabledFunctions.length, runningCount: this.runners.size }
+      })
+    } catch (error) {
+      Logger.log({
+        level: 'error',
+        message: 'Failed to initialize FunctionsService',
+        data: { error: error.message }
+      })
+    }
+  }
+
+  async shutdown () {
+    const stopPromises = []
+    for (const [, runner] of this.runners) {
+      stopPromises.push(runner.stop().catch(err => {
+        Logger.log({
+          level: 'warn',
+          message: `Error stopping function during shutdown: ${runner.name}`,
+          data: { error: err.message }
+        })
+      }))
+    }
+    await Promise.all(stopPromises)
+    this.runners.clear()
+    this.stopHeartbeat()
+    this.sseClients.clear()
+    this.invocationSseClients.clear()
+    Logger.log({ level: 'info', message: 'FunctionsService shut down' })
+  }
+
+  async deployFunction (config) {
+    const port = config.port || this.allocatePort()
+    if (!port) {
+      throw new Error('No available ports for function deployment')
+    }
+    const runner = new FunctionRunner({
+      name: config.name,
+      source: config.source,
+      sourcePath: config.sourcePath,
+      entryPoint: config.entryPoint,
+      port,
+      signatureType: config.signatureType || 'cloudevent',
+      env: typeof config.environmentVariables === 'string'
+        ? JSON.parse(config.environmentVariables)
+        : config.environmentVariables || {},
+      timeoutSeconds: config.timeoutSeconds || 60
+    })
+    runner.onLog = (name, stream, message) => {
+      this.broadcastLog(name, stream, message)
+    }
+    runner.onStatusChange = (name, status) => {
+      this.updateFunctionStatus(name, status)
+    }
+    try {
+      await runner.start()
+      this.runners.set(config.name, runner)
+      this.updateFunctionStatus(config.name, FunctionStatuses.RUNNING, null, port, runner.process?.pid)
+      return runner.getStatus()
+    } catch (error) {
+      this.releasePort(port)
+      this.updateFunctionStatus(config.name, FunctionStatuses.ERROR, error.message)
+      throw error
+    }
+  }
+
+  async removeFunction (name) {
+    const runner = this.runners.get(name)
+    if (runner) {
+      const port = runner.port
+      await runner.stop()
+      this.releasePort(port)
+      this.runners.delete(name)
+    }
+  }
+
+  async startFunction (name) {
+    const fn = SqliteStore.find(CLOUD_FUNCTIONS, { name }).pop()
+    if (!fn) throw new Error(`Function not found: ${name}`)
+    if (this.runners.has(name)) {
+      const runner = this.runners.get(name)
+      if (runner.status === FunctionStatuses.RUNNING) return runner.getStatus()
+    }
+    return this.deployFunction(fn)
+  }
+
+  async stopFunction (name) {
+    const runner = this.runners.get(name)
+    if (!runner) throw new Error(`Function not running: ${name}`)
+    const port = runner.port
+    await runner.stop()
+    this.releasePort(port)
+    this.runners.delete(name)
+    this.updateFunctionStatus(name, FunctionStatuses.STOPPED)
+  }
+
+  async restartFunction (name) {
+    await this.stopFunction(name)
+    return this.startFunction(name)
+  }
+
+  getEndpointUrl (name) {
+    const runner = this.runners.get(name)
+    if (!runner || runner.status !== FunctionStatuses.RUNNING) return null
+    return `http://localhost:${runner.port}`
+  }
+
+  getAllStatuses () {
+    const statuses = []
+    for (const [, runner] of this.runners) {
+      statuses.push(runner.getStatus())
+    }
+    return statuses
+  }
+
+  allocatePort () {
+    const iterator = this.portPool.values()
+    const first = iterator.next()
+    if (first.done) return null
+    this.portPool.delete(first.value)
+    return first.value
+  }
+
+  releasePort (port) {
+    if (port) this.portPool.add(port)
+  }
+
+  updateFunctionStatus (name, status, errorMessage, port, pid) {
+    try {
+      const fn = SqliteStore.find(CLOUD_FUNCTIONS, { name }).pop()
+      if (!fn) return
+      const updates = { status, updatedAt: new Date().toISOString() }
+      if (errorMessage !== undefined) updates.errorMessage = errorMessage
+      if (port !== undefined) updates.port = port
+      if (pid !== undefined) updates.pid = pid
+      SqliteStore.update(CLOUD_FUNCTIONS, fn.id, updates)
+    } catch (error) {
+      Logger.log({
+        level: 'warn',
+        message: 'Failed to update function status in DB',
+        data: { name, status, error: error.message }
+      })
+    }
+  }
+
+  // --- Log SSE ---
+
+  addLogClient (id, ctx, filters = {}) {
+    this.sseClients.set(id, { ctx, filters })
+    this.startHeartbeat()
+    Logger.log({
+      level: 'info',
+      message: 'Functions log SSE client connected',
+      data: { connectionId: id, totalClients: this.sseClients.size }
+    })
+  }
+
+  removeLogClient (id) {
+    const removed = this.sseClients.delete(id)
+    if (removed && this.sseClients.size === 0 && this.invocationSseClients.size === 0) {
+      this.stopHeartbeat()
+    }
+  }
+
+  broadcastLog (functionName, level, message) {
+    const trimmedMessage = message.trimEnd()
+    const now = new Date().toISOString()
+    // Broadcast to Functions-specific SSE clients
+    if (this.sseClients.size > 0) {
+      const payload = { functionName, level, message: trimmedMessage, timestamp: now }
+      for (const [id, { ctx, filters }] of this.sseClients) {
+        if (filters.functionName && filters.functionName !== functionName) continue
+        try {
+          this.sendSSE(ctx, 'log', payload)
+        } catch {
+          this.removeLogClient(id)
+        }
+      }
+    }
+    // Also write to the unified logging system
+    try {
+      const severity = level === 'stderr' ? 'ERROR' : 'INFO'
+      let jsonPayload = null
+      try {
+        const parsed = JSON.parse(trimmedMessage)
+        if (parsed && typeof parsed === 'object') {
+          jsonPayload = JSON.stringify(parsed)
+        }
+      } catch {}
+      const entry = {
+        _id: uuid(),
+        logName: 'projects/dev-tools/logs/cloud-functions',
+        serviceName: functionName,
+        severity,
+        textPayload: trimmedMessage,
+        jsonPayload,
+        timestamp: now,
+        receiveTimestamp: now,
+        insertId: uuid(),
+        source: 'cloud-functions',
+        labels: JSON.stringify({ functionName, source: 'cloud-functions' })
+      }
+      SqliteStore.create(LOGGING_ENTRIES, entry)
+      LogBroadcaster.broadcast(entry)
+    } catch (error) {
+      Logger.log({ level: 'error', message: 'Failed to write function log to logging system', data: { functionName, error: error.message } })
+    }
+  }
+
+  // --- Invocation SSE ---
+
+  addInvocationClient (id, ctx, filters = {}) {
+    this.invocationSseClients.set(id, { ctx, filters })
+    this.startHeartbeat()
+    Logger.log({
+      level: 'info',
+      message: 'Functions invocation SSE client connected',
+      data: { connectionId: id, totalClients: this.invocationSseClients.size }
+    })
+  }
+
+  removeInvocationClient (id) {
+    const removed = this.invocationSseClients.delete(id)
+    if (removed && this.sseClients.size === 0 && this.invocationSseClients.size === 0) {
+      this.stopHeartbeat()
+    }
+  }
+
+  broadcastInvocation (invocation) {
+    if (this.invocationSseClients.size === 0) return
+    for (const [id, { ctx, filters }] of this.invocationSseClients) {
+      if (filters.functionName && filters.functionName !== invocation.functionName) continue
+      try {
+        this.sendSSE(ctx, 'invocation', invocation)
+      } catch {
+        this.removeInvocationClient(id)
+      }
+    }
+  }
+
+  // --- File Change SSE ---
+
+  addFileChangeClient (id, ctx, functionName) {
+    this.fileChangeClients.set(id, { ctx, functionName })
+    this.startHeartbeat()
+    this.startFileWatcher(functionName)
+    Logger.log({ level: 'info', message: 'File change SSE client connected', data: { connectionId: id, functionName } })
+  }
+
+  removeFileChangeClient (id) {
+    const client = this.fileChangeClients.get(id)
+    this.fileChangeClients.delete(id)
+    if (client) {
+      const stillWatching = Array.from(this.fileChangeClients.values())
+        .some(c => c.functionName === client.functionName)
+      if (!stillWatching) {
+        this.stopFileWatcher(client.functionName)
+      }
+    }
+    if (this.sseClients.size === 0 && this.invocationSseClients.size === 0 && this.fileChangeClients.size === 0) {
+      this.stopHeartbeat()
+    }
+  }
+
+  startFileWatcher (functionName) {
+    if (this.fileWatchers.has(functionName)) return
+    const { data } = SqliteStore.list(CLOUD_FUNCTIONS, { filter: { name: functionName }, limit: 1 })
+    const fn = data[0]
+    if (!fn || !fn.sourcePath) return
+    const watchDir = fn.sourcePath
+    Logger.log({ level: 'info', message: 'Starting file watcher', data: { functionName, watchDir } })
+    const watcher = chokidar.watch(watchDir, {
+      ignoreInitial: true,
+      ignored: [
+        '**/node_modules/**',
+        '**/.git/**',
+        '**/package-lock.json'
+      ],
+      usePolling: true,
+      interval: 1000,
+      awaitWriteFinish: {
+        stabilityThreshold: 500,
+        pollInterval: 200
+      }
+    })
+    const handleChange = (eventType, filePath) => {
+      const relativePath = path.relative(watchDir, filePath)
+      if (relativePath.startsWith('node_modules')) return
+      Logger.log({
+        level: 'debug',
+        message: 'File change detected',
+        data: { functionName, eventType, relativePath }
+      })
+      for (const [id, client] of this.fileChangeClients) {
+        if (client.functionName !== functionName) continue
+        try {
+          this.sendSSE(client.ctx, 'fileChange', { functionName, eventType, filePath: relativePath })
+        } catch {
+          this.removeFileChangeClient(id)
+        }
+      }
+    }
+    watcher.on('ready', () => {
+      Logger.log({ level: 'info', message: 'File watcher ready', data: { functionName, watchDir } })
+    })
+    watcher.on('error', (err) => {
+      Logger.log({ level: 'error', message: 'File watcher error', data: { functionName, error: err.message } })
+    })
+    watcher.on('change', (fp) => handleChange('change', fp))
+    watcher.on('add', (fp) => handleChange('add', fp))
+    watcher.on('unlink', (fp) => handleChange('unlink', fp))
+    this.fileWatchers.set(functionName, watcher)
+  }
+
+  stopFileWatcher (functionName) {
+    const watcher = this.fileWatchers.get(functionName)
+    if (watcher) {
+      watcher.close()
+      this.fileWatchers.delete(functionName)
+    }
+  }
+
+  // --- SSE Helpers ---
+
+  sendSSE (ctx, event, data) {
+    const message = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`
+    ctx.res.write(message)
+  }
+
+  startHeartbeat () {
+    if (this.heartbeatId) return
+    this.heartbeatId = setInterval(() => this.sendHeartbeat(), this.heartbeatIntervalMs)
+  }
+
+  stopHeartbeat () {
+    if (this.heartbeatId) {
+      clearInterval(this.heartbeatId)
+      this.heartbeatId = null
+    }
+  }
+
+  sendHeartbeat () {
+    const allClients = [
+      ...Array.from(this.sseClients.entries()),
+      ...Array.from(this.invocationSseClients.entries()),
+      ...Array.from(this.fileChangeClients.entries())
+    ]
+    for (const [id, { ctx }] of allClients) {
+      try {
+        ctx.res.write(': heartbeat\n\n')
+      } catch {
+        this.sseClients.delete(id)
+        this.invocationSseClients.delete(id)
+        this.fileChangeClients.delete(id)
+      }
+    }
+    if (this.sseClients.size === 0 && this.invocationSseClients.size === 0 && this.fileChangeClients.size === 0) {
+      this.stopHeartbeat()
+    }
+  }
+}
+
+export const FunctionsService = new FunctionsServiceClass()

package/src/singletons/HttpProxy.js

@@ -0,0 +1,224 @@
+import httpProxy from 'http-proxy'
+import { v4 as uuidv4 } from 'uuid'
+import { SqliteStore } from './SqliteStore.js'
+import { Logger } from './Logger.js'
+import { LogBroadcaster } from './LogBroadcaster.js'
+import { HTTP_TRAFFIC } from '../db/Tables.js'
+import { ServiceNames } from '../enums/ServiceNames.js'
+
+const BINARY_CONTENT_TYPES = [
+  'image/', 'audio/', 'video/', 'application/octet-stream',
+  'application/zip', 'application/gzip', 'application/pdf'
+]
+
+function isBinaryContentType (contentType) {
+  if (!contentType) return false
+  return BINARY_CONTENT_TYPES.some(type => contentType.startsWith(type))
+}
+
+function tryParseJson (str) {
+  if (!str || typeof str !== 'string') return null
+  const trimmed = str.trim()
+  if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
+    try { return JSON.parse(trimmed) } catch { return null }
+  }
+  return null
+}
+
+function parseCookies (cookieHeader) {
+  if (!cookieHeader) return null
+  const cookies = {}
+  cookieHeader.split(';').forEach(part => {
+    const [key, ...rest] = part.trim().split('=')
+    if (key) cookies[key.trim()] = rest.join('=').trim()
+  })
+  return Object.keys(cookies).length > 0 ? cookies : null
+}
+
+function extractServiceName (hostname) {
+  if (!hostname) return null
+  return hostname.split(':')[0]
+}
+
+class HttpProxyClass {
+  constructor () {
+    this.proxy = null
+  }
+
+  initialize () {
+    this.proxy = httpProxy.createProxyServer({
+      changeOrigin: true,
+      secure: false,
+      xfwd: true,
+      proxyTimeout: 30000
+    })
+    this.proxy.on('error', (err, req, res) => {
+      Logger.log({
+        level: 'error',
+        message: 'HTTP proxy error',
+        data: { error: err.message, url: req.url, method: req.method }
+      })
+    })
+    Logger.log({
+      level: 'info',
+      message: 'HttpProxy initialized'
+    })
+  }
+
+  parseTargetUrl (rawPath) {
+    const match = rawPath.match(/^\/proxy\/(.+)$/)
+    if (!match) return null
+    let rawTarget = match[1]
+    if (!rawTarget.match(/^https?:\/\//)) {
+      rawTarget = 'http://' + rawTarget
+    }
+    try {
+      const url = new URL(rawTarget)
+      return {
+        origin: url.origin,
+        host: url.host,
+        hostname: url.hostname,
+        path: url.pathname + url.search,
+        fullUrl: rawTarget,
+        serviceName: extractServiceName(url.hostname)
+      }
+    } catch {
+      return null
+    }
+  }
+
+  logTraffic (entry) {
+    const {
+      method, targetUrl, targetHost, targetPath,
+      queryParams, requestHeaders, requestBody, requestCookies,
+      contentType, statusCode, responseHeaders, responseBody,
+      responseContentType, responseTimeMs, startedAt, completedAt,
+      sourceService, error, traceId
+    } = entry
+    const id = uuidv4()
+    const trafficBody = isBinaryContentType(contentType)
+      ? `[binary, ${requestBody?.length || 0} bytes]`
+      : requestBody || null
+    const trafficResponseBody = isBinaryContentType(responseContentType)
+      ? `[binary, ${responseBody?.length || 0} bytes]`
+      : responseBody || null
+    try {
+      SqliteStore.create(HTTP_TRAFFIC, {
+        _id: id,
+        method,
+        targetUrl,
+        targetHost,
+        targetPath,
+        queryParams: queryParams || null,
+        requestHeaders: requestHeaders || null,
+        requestBody: trafficBody,
+        requestCookies: requestCookies || null,
+        contentType: contentType || null,
+        statusCode: statusCode || null,
+        responseHeaders: responseHeaders || null,
+        responseBody: trafficResponseBody,
+        responseContentType: responseContentType || null,
+        responseTimeMs: responseTimeMs || null,
+        startedAt,
+        completedAt: completedAt || null,
+        sourceService: sourceService || null,
+        error: error || null,
+        traceId: traceId || null
+      })
+    } catch (err) {
+      Logger.log({ level: 'error', message: 'Failed to store http traffic', data: { error: err.message } })
+    }
+    const now = new Date().toISOString()
+    const severity = error || (statusCode && statusCode >= 400) ? 'ERROR' : 'INFO'
+    const statusText = error ? `ERR ${error}` : `${statusCode}`
+    const textPayload = `${method} ${targetUrl} → ${statusText} (${responseTimeMs}ms)`
+    const logRequestBody = tryParseJson(requestBody) || requestBody || null
+    const logResponseBody = tryParseJson(responseBody) || responseBody || null
+    const jsonPayload = {
+      method,
+      targetUrl,
+      targetHost,
+      statusCode,
+      responseTimeMs,
+      error: error || null,
+      ...(queryParams && { queryParams }),
+      ...(logRequestBody && { requestBody: logRequestBody }),
+      ...(logResponseBody && { responseBody: logResponseBody })
+    }
+    const logEntry = {
+      _id: uuidv4(),
+      logName: 'projects/dev-tools/logs/http',
+      source: ServiceNames.HTTP_PROXY,
+      serviceName: ServiceNames.HTTP_PROXY,
+      severity,
+      textPayload,
+      jsonPayload,
+      timestamp: now,
+      receiveTimestamp: now,
+      insertId: uuidv4(),
+      createdAt: now
+    }
+    try {
+      SqliteStore.create('logging_entries', logEntry)
+    } catch (err) {
+      Logger.log({ level: 'error', message: 'Failed to store http proxy log', data: { error: err.message } })
+    }
+    LogBroadcaster.broadcast(logEntry)
+  }
+
+  listTraffic (options = {}) {
+    const { filter, limit = 50, offset = 0 } = options
+    const listOptions = {
+      orderBy: 'created_at DESC',
+      limit,
+      offset
+    }
+    if (filter && Object.keys(filter).length > 0) {
+      listOptions.filter = filter
+    }
+    return SqliteStore.list(HTTP_TRAFFIC, listOptions)
+  }
+
+  getTrafficEntry (id) {
+    return SqliteStore.get(HTTP_TRAFFIC, id, '_id')
+  }
+
+  clearTraffic () {
+    SqliteStore.clear(HTTP_TRAFFIC)
+  }
+
+  getStats () {
+    try {
+      const total = SqliteStore.db.prepare(`SELECT COUNT(*) as count FROM ${HTTP_TRAFFIC}`).get().count
+      const byMethod = SqliteStore.db.prepare(`
+        SELECT method, COUNT(*) as count FROM ${HTTP_TRAFFIC} GROUP BY method ORDER BY count DESC
+      `).all()
+      const byHost = SqliteStore.db.prepare(`
+        SELECT source_service, COUNT(*) as count FROM ${HTTP_TRAFFIC} GROUP BY source_service ORDER BY count DESC
+      `).all()
+      const byStatus = SqliteStore.db.prepare(`
+        SELECT status_code, COUNT(*) as count FROM ${HTTP_TRAFFIC} GROUP BY status_code ORDER BY count DESC
+      `).all()
+      const avgResponseTime = SqliteStore.db.prepare(`
+        SELECT AVG(response_time_ms) as avg_ms FROM ${HTTP_TRAFFIC} WHERE response_time_ms IS NOT NULL
+      `).get()
+      const errorCount = SqliteStore.db.prepare(`
+        SELECT COUNT(*) as count FROM ${HTTP_TRAFFIC} WHERE status_code >= 400 OR error IS NOT NULL
+      `).get().count
+      return {
+        total,
+        errorCount,
+        errorRate: total > 0 ? (errorCount / total * 100).toFixed(1) : '0.0',
+        avgResponseTimeMs: Math.round(avgResponseTime?.avg_ms || 0),
+        byMethod,
+        byHost,
+        byStatus
+      }
+    } catch (err) {
+      Logger.log({ level: 'error', message: 'Failed to get http traffic stats', data: { error: err.message } })
+      return { total: 0, errorCount: 0, errorRate: '0.0', avgResponseTimeMs: 0, byMethod: [], byHost: [], byStatus: [] }
+    }
+  }
+}
+
+export const HttpProxy = new HttpProxyClass()