@platformatic/watt-admin 0.6.0-alpha.1 → 0.6.0-alpha.10

package/web/backend/routes/root.ts

@@ -0,0 +1,204 @@
+import type { FastifyInstance } from 'fastify'
+import type { JsonSchemaToTsProvider } from '@fastify/type-provider-json-schema-to-ts'
+import { RuntimeApiClient } from '@platformatic/control'
+import { getPidToLoad, getSelectableRuntimes } from '../utils/runtimes.ts'
+import { writeFile, readFile } from 'fs/promises'
+import { checkRecordState } from '../utils/states.ts'
+import { join } from 'path'
+import { pidParamSchema, selectableRuntimeSchema, modeSchema, profileSchema } from '../schemas/index.ts'
+
+const __dirname = import.meta.dirname
+
+export default async function (fastify: FastifyInstance) {
+  const typedFastify = fastify.withTypeProvider<JsonSchemaToTsProvider>()
+
+  // FIXME: types have not been properly implemented in `@platformatic/control` and they should be updated as form the cast in the following line
+  const api = new RuntimeApiClient() as RuntimeApiClient & { startApplicationProfiling: (...args: unknown[]) => Promise<unknown>, stopApplicationProfiling: (...args: unknown[]) => Promise<string> }
+
+  typedFastify.get('/runtimes', {
+    schema: {
+      querystring: {
+        type: 'object',
+        properties: {
+          includeAdmin: {
+            type: 'boolean',
+            default: false,
+          },
+        },
+      },
+      response: { 200: { type: 'array', items: selectableRuntimeSchema } }
+    }
+  }, async (request) => getSelectableRuntimes(await api.getRuntimes(), request.query.includeAdmin))
+
+  typedFastify.get('/runtimes/:pid/health', {
+    schema: {
+      params: pidParamSchema,
+      response: {
+        200: {
+          type: 'object',
+          additionalProperties: false,
+          properties: {
+            status: {
+              type: 'string',
+              enum: ['OK', 'KO'],
+              description: "Status can only be 'OK' or 'KO'"
+            }
+          },
+          required: ['status']
+        }
+      }
+    }
+  }, async ({ params: { pid } }) => {
+    const ok = { status: 'OK' as const }
+    const ko = { status: 'KO' as const }
+
+    try {
+      const result = await api.getMatchingRuntime({ pid: pid.toString() })
+      return (result.pid === pid) ? ok : ko
+    } catch {
+      return ko
+    }
+  })
+
+  typedFastify.get('/runtimes/:pid/services', {
+    schema: {
+      params: pidParamSchema,
+      response: {
+        200: {
+          type: 'object',
+          additionalProperties: false,
+          required: ['entrypoint', 'production', 'applications'],
+          properties: {
+            entrypoint: {
+              type: 'string'
+            },
+            production: {
+              type: 'boolean'
+            },
+            applications: {
+              type: 'array',
+              items: {
+                anyOf: [
+                  {
+                    additionalProperties: false,
+                    type: 'object',
+                    required: ['id', 'type', 'status', 'version', 'localUrl', 'entrypoint', 'dependencies'],
+                    properties: {
+                      id: {
+                        type: 'string'
+                      },
+                      type: {
+                        type: 'string'
+                      },
+                      status: {
+                        type: 'string'
+                      },
+                      version: {
+                        type: 'string'
+                      },
+                      localUrl: {
+                        type: 'string'
+                      },
+                      entrypoint: {
+                        type: 'boolean'
+                      },
+                      workers: {
+                        type: 'number'
+                      },
+                      url: {
+                        type: 'string'
+                      },
+                      dependencies: {
+                        type: 'array',
+                        items: { type: 'string' }
+                      }
+                    }
+                  },
+                  {
+                    additionalProperties: false,
+                    type: 'object',
+                    required: ['id', 'status'],
+                    properties: {
+                      id: {
+                        type: 'string'
+                      },
+                      status: {
+                        type: 'string'
+                      }
+                    }
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    }
+  }, async (request) => api.getRuntimeApplications(request.params.pid))
+
+  typedFastify.get('/runtimes/:pid/openapi/:serviceId', {
+    schema: {
+      params: { type: 'object', properties: { pid: { type: 'number' }, serviceId: { type: 'string' } }, required: ['pid', 'serviceId'] }
+    }
+  }, async ({ params: { pid, serviceId } }) => api.getRuntimeOpenapi(pid, serviceId))
+
+  typedFastify.post('/runtimes/:pid/restart', {
+    schema: { params: pidParamSchema, body: { type: 'object' } }
+  }, async (request) => {
+    try {
+      await api.restartRuntime(request.params.pid)
+    } catch (err) {
+      fastify.log.warn({ err }, 'Issue restarting the runtime')
+    }
+  })
+
+  typedFastify.post('/record/:pid', {
+    schema: {
+      params: pidParamSchema,
+      body: {
+        type: 'object',
+        additionalProperties: false,
+        properties: { mode: modeSchema, profile: profileSchema },
+        required: ['mode', 'profile']
+      }
+    }
+  }, async ({ body: { mode, profile: type }, params: { pid } }) => {
+    const from = fastify.loaded.mode
+    const to = mode
+    if (!checkRecordState({ from, to })) {
+      return fastify.log.error({ from, to }, 'Invalid record state machine transition')
+    }
+
+    const { applications } = await api.getRuntimeApplications(pid)
+    fastify.loaded.mode = mode
+    if (mode === 'start') {
+      for (const { id } of applications) {
+        await api.startApplicationProfiling(pid, id, { type })
+      }
+      fastify.loaded.type = type
+      fastify.loaded.metrics = {}
+    }
+
+    if (mode === 'stop') {
+      try {
+        const runtimes = getSelectableRuntimes(await api.getRuntimes(), false)
+        const services = await api.getRuntimeApplications(getPidToLoad(runtimes))
+
+        const profile: Record<string, Uint8Array> = {}
+        for (const { id } of applications) {
+          const profileData = Buffer.from(await api.stopApplicationProfiling(pid, id, { type }))
+          await writeFile(join(__dirname, '..', '..', 'frontend', 'dist', `${fastify.loaded.type}-profile-${id}.pb`), profileData)
+          profile[id] = new Uint8Array(profileData)
+        }
+
+        const loadedJson = JSON.stringify({ runtimes, services, metrics: fastify.loaded.metrics[getPidToLoad(runtimes)], profile, type })
+
+        const scriptToAppend = `  <script>window.LOADED_JSON=${loadedJson}</script>\n</body>`
+        const bundlePath = join(__dirname, '..', '..', 'frontend', 'dist', 'index.html')
+        await writeFile(bundlePath, (await readFile(bundlePath, 'utf8')).replace('</body>', scriptToAppend), 'utf8')
+      } catch (err) {
+        fastify.log.error({ err }, 'Unable to save the loaded JSON')
+      }
+    }
+  })
+}

package/web/backend/routes/ws.ts

@@ -0,0 +1,45 @@
+import type { WebSocket } from 'ws'
+import split2 from 'split2'
+import type { FastifyInstance } from 'fastify'
+import type { JsonSchemaToTsProvider } from '@fastify/type-provider-json-schema-to-ts'
+import { RuntimeApiClient } from '@platformatic/control'
+import { pidParamSchema } from '../schemas/index.ts'
+import type { PidParam } from '../schemas/index.ts'
+import { pipeline } from 'node:stream/promises'
+
+export default async function (fastify: FastifyInstance) {
+  const typedFastify = fastify.withTypeProvider<JsonSchemaToTsProvider>()
+  const api = new RuntimeApiClient()
+
+  const wsSendAsync = (socket: WebSocket, data: string): Promise<void> => new Promise((resolve, reject) => setTimeout(() => socket.send(data, (err) => (err)
+    ? reject(err)
+    : resolve()
+  ), 100)
+  )
+
+  typedFastify.get<{ Params: PidParam }>('/runtimes/:pid/logs/ws', {
+    schema: { params: pidParamSchema },
+    websocket: true
+  }, async (socket, { params: { pid } }) => {
+    try {
+      const clientStream = api.getRuntimeLiveLogsStream(pid)
+
+      socket.on('close', () => {
+        clientStream.destroy()
+      })
+
+      await pipeline(
+        clientStream,
+        split2(),
+        async function * (source: AsyncIterable<string>) {
+          for await (const line of source) {
+            await wsSendAsync(socket, line)
+          }
+        }
+      )
+    } catch (error) {
+      fastify.log.error({ error }, 'fatal error on runtime logs ws')
+      socket.close()
+    }
+  })
+}

package/web/backend/schemas/index.ts

@@ -0,0 +1,226 @@
+import { type FromSchema } from 'json-schema-to-ts'
+
+const dateSchema = { type: 'string', format: 'date-time' } as const
+
+export const modeSchema = { type: 'string', enum: ['start', 'stop'] } as const
+export type Mode = FromSchema<typeof modeSchema>
+
+const memoryDataPointSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    date: dateSchema,
+    rss: { type: 'number' },
+    totalHeap: { type: 'number' },
+    usedHeap: { type: 'number' },
+    newSpace: { type: 'number' },
+    oldSpace: { type: 'number' }
+  },
+  required: ['date', 'rss', 'totalHeap', 'usedHeap', 'newSpace', 'oldSpace']
+} as const
+export type MemoryDataPoint = FromSchema<typeof memoryDataPointSchema>
+
+const cpuDataPointSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    date: dateSchema,
+    cpu: { type: 'number' },
+    eventLoop: { type: 'number' }
+  },
+  required: ['date', 'cpu', 'eventLoop']
+} as const
+export type CpuDataPoint = FromSchema<typeof cpuDataPointSchema>
+
+const latencyDataPointSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    date: { type: 'string', format: 'date-time' },
+    p90: { type: 'number' },
+    p95: { type: 'number' },
+    p99: { type: 'number' }
+  },
+  required: ['date', 'p90', 'p95', 'p99']
+} as const
+export type LatencyDataPoint = FromSchema<typeof latencyDataPointSchema>
+
+const requestDataPointSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    date: { type: 'string', format: 'date-time' },
+    count: { type: 'number' },
+    rps: { type: 'number' }
+  },
+  required: ['date', 'count', 'rps']
+} as const
+export type RequestDataPoint = FromSchema<typeof requestDataPointSchema>
+
+const kafkaDataPointSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    date: { type: 'string', format: 'date-time' },
+    producers: { type: 'number' },
+    producedMessages: { type: 'number' },
+    consumers: { type: 'number' },
+    consumersStreams: { type: 'number' },
+    consumersTopics: { type: 'number' },
+    consumedMessages: { type: 'number' },
+    hooksMessagesInFlight: { type: 'number' },
+    hooksDlqMessagesTotal: { type: 'number' },
+  },
+  required: [
+    'date',
+    'producers',
+    'producedMessages',
+    'consumers',
+    'consumersStreams',
+    'consumersTopics',
+    'consumedMessages',
+    'hooksMessagesInFlight',
+    'hooksDlqMessagesTotal',
+  ]
+} as const
+export type KafkaDataPoint = FromSchema<typeof kafkaDataPointSchema>
+
+const undiciDataPointSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    date: { type: 'string', format: 'date-time' },
+    idleSockets: { type: 'number' },
+    openSockets: { type: 'number' },
+    pendingRequests: { type: 'number' },
+    queuedRequests: { type: 'number' },
+    activeRequests: { type: 'number' },
+    sizeRequests: { type: 'number' },
+  },
+  required: [
+    'date',
+    'idleSockets',
+    'openSockets',
+    'pendingRequests',
+    'queuedRequests',
+    'activeRequests',
+    'sizeRequests',
+  ]
+} as const
+export type UndiciDataPoint = FromSchema<typeof undiciDataPointSchema>
+
+const websocketDataPointSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    date: { type: 'string', format: 'date-time' },
+    connections: { type: 'number' }
+  },
+  required: [
+    'date',
+    'connections'
+  ]
+} as const
+export type WebsocketDataPoint = FromSchema<typeof websocketDataPointSchema>
+
+const nodejsDataPointSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    date: { type: 'string', format: 'date-time' },
+    resources: { type: 'number' }
+  },
+  required: [
+    'date',
+    'resources'
+  ]
+} as const
+export type NodejsDataPoint = FromSchema<typeof nodejsDataPointSchema>
+
+export const requiredMetricKeys = ['dataMem', 'dataCpu', 'dataKafka', 'dataReq', 'dataLatency', 'dataUndici', 'dataWebsocket', 'dataNodejs'] as const
+export const metricResponseSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    dataMem: { type: 'array', items: memoryDataPointSchema },
+    dataCpu: { type: 'array', items: cpuDataPointSchema },
+    dataLatency: { type: 'array', items: latencyDataPointSchema },
+    dataReq: { type: 'array', items: requestDataPointSchema },
+    dataKafka: { type: 'array', items: kafkaDataPointSchema },
+    dataUndici: { type: 'array', items: undiciDataPointSchema },
+    dataWebsocket: { type: 'array', items: websocketDataPointSchema },
+    dataNodejs: { type: 'array', items: nodejsDataPointSchema }
+  },
+  required: requiredMetricKeys
+} as const
+export type MetricsResponse = FromSchema<typeof metricResponseSchema>
+export type SingleMetricResponse = {
+  [K in keyof MetricsResponse]: MetricsResponse[K] extends (infer T)[] ? T : never
+}
+
+export const pidParamSchema = { type: 'object', additionalProperties: false, properties: { pid: { type: 'number' } }, required: ['pid'] } as const
+export type PidParam = FromSchema<typeof pidParamSchema>
+
+export const profileSchema = { type: 'string', enum: ['cpu', 'heap'] } as const
+export type Profile = FromSchema<typeof profileSchema>
+
+export const selectableRuntimeSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    pid: {
+      type: 'integer'
+    },
+    cwd: {
+      type: 'string'
+    },
+    argv: {
+      type: 'array',
+      items: {
+        type: 'string'
+      }
+    },
+    uptimeSeconds: {
+      type: 'number'
+    },
+    execPath: {
+      type: 'string'
+    },
+    nodeVersion: {
+      type: 'string'
+    },
+    projectDir: {
+      type: 'string'
+    },
+    packageName: {
+      type: 'string'
+    },
+    packageVersion: {
+      type: 'string'
+    },
+    url: {
+      type: 'string'
+    },
+    platformaticVersion: {
+      type: 'string'
+    },
+    selected: {
+      type: 'boolean'
+    }
+  },
+  required: [
+    'pid',
+    'cwd',
+    'argv',
+    'uptimeSeconds',
+    'execPath',
+    'nodeVersion',
+    'projectDir',
+    'packageName',
+    'packageVersion',
+    'url',
+    'platformaticVersion',
+    'selected'
+  ]
+} as const
+export type SelectableRuntime = FromSchema<typeof selectableRuntimeSchema>

package/web/backend/utils/bytes.ts

@@ -0,0 +1 @@
+export const bytesToMB = (bytes: number): number => Number((bytes / (1024 * 1024)).toFixed(2))

package/web/backend/utils/client.openapi.ts

@@ -0,0 +1,29 @@
+import fs from 'fs/promises'
+import { create } from '@platformatic/service'
+import { join, resolve } from 'node:path'
+
+const __dirname = import.meta.dirname
+
+async function clientOpenapi () {
+  const basePath = resolve(__dirname, join('..', 'platformatic.json'))
+  const server = await create(basePath, {
+    service: { openapi: true },
+    watch: false,
+    plugins: {
+      paths: [
+        {
+          path: resolve(__dirname, join('..', 'plugins')),
+          encapsulate: false
+        },
+        resolve(__dirname, join('..', 'routes'))
+      ]
+    }
+  })
+
+  await server.start({})
+  const { body } = await server.getApplication().inject('/documentation/json')
+  await fs.writeFile('openapi.json', body)
+  await server.close()
+}
+
+clientOpenapi()

package/web/backend/utils/constants.ts

@@ -0,0 +1,4 @@
+export const MS_WAITING = 1000
+
+// This is to avoid the mapped metrics array from growing indefinitely (and therefore a memory leak): store the last 10 minutes of metrics (1# * 10m * 60s).
+export const MAX_STORED_METRICS = 600

package/web/backend/utils/metrics-helpers.ts

@@ -0,0 +1,89 @@
+import type { CpuDataPoint, KafkaDataPoint, LatencyDataPoint, MemoryDataPoint, MetricsResponse, NodejsDataPoint, RequestDataPoint, SingleMetricResponse, UndiciDataPoint, WebsocketDataPoint } from '../schemas/index.ts'
+import { MAX_STORED_METRICS } from './constants.ts'
+
+export type MappedMetrics = Record<number, {
+  aggregated: MetricsResponse,
+  services: Record<string, Record<'all' | number, MetricsResponse>>
+}>
+
+export const websocketMetricMap = {
+  active_ws_composer_connections: 'connections',
+} as const
+export const isWebsocketMetricName = (metricName: string): metricName is keyof typeof websocketMetricMap => metricName in websocketMetricMap
+
+export const nodejsMetricMap = {
+  active_resources_event_loop: 'resources'
+} as const
+export const isNodejsMetricName = (metricName: string): metricName is keyof typeof nodejsMetricMap => metricName in nodejsMetricMap
+
+export const undiciMetricMap = {
+  http_client_stats_free: 'idleSockets',
+  http_client_stats_connected: 'openSockets',
+  http_client_stats_pending: 'pendingRequests',
+  http_client_stats_queued: 'queuedRequests',
+  http_client_stats_running: 'activeRequests',
+  http_client_stats_size: 'sizeRequests'
+} as const
+export const isUndiciMetricName = (metricName: string): metricName is keyof typeof undiciMetricMap => metricName in undiciMetricMap
+
+export const kafkaMetricMap = {
+  kafka_producers: 'producers',
+  kafka_produced_messages: 'producedMessages',
+  kafka_consumers: 'consumers',
+  kafka_consumers_streams: 'consumersStreams',
+  kafka_consumers_topics: 'consumersTopics',
+  kafka_consumed_messages: 'consumedMessages',
+  kafka_hooks_messages_in_flight: 'hooksMessagesInFlight',
+  kafka_hooks_dlq_messages_total: 'hooksDlqMessagesTotal'
+} as const
+export const isKafkaMetricName = (metricName: string): metricName is keyof typeof kafkaMetricMap => metricName in kafkaMetricMap
+
+export const addMetricDataPoint = <T extends MemoryDataPoint | CpuDataPoint | LatencyDataPoint | RequestDataPoint | KafkaDataPoint | UndiciDataPoint | WebsocketDataPoint | NodejsDataPoint>(metrics: T[], dataPoint: T) => {
+  if (metrics.length >= MAX_STORED_METRICS) {
+    metrics.shift()
+  }
+  metrics.push(dataPoint)
+}
+
+export const initMetricsObject = (date: string): SingleMetricResponse => ({ dataMem: initMemData(date), dataCpu: initCpuData(date), dataKafka: initKafkaData(date), dataReq: initReqData(date), dataLatency: initLatencyData(date), dataUndici: initUndiciData(date), dataWebsocket: initWebsocketData(date), dataNodejs: initNodejsData(date) })
+
+export const initMetricsResponse = (date?: string, length?: number): MetricsResponse => {
+  const isArray = date && length
+  return {
+    dataCpu: isArray ? Array.from({ length }, () => initCpuData(date)) : [],
+    dataLatency: isArray ? Array.from({ length }, () => initLatencyData(date)) : [],
+    dataMem: isArray ? Array.from({ length }, () => initMemData(date)) : [],
+    dataReq: isArray ? Array.from({ length }, () => initReqData(date)) : [],
+    dataKafka: isArray ? Array.from({ length }, () => initKafkaData(date)) : [],
+    dataUndici: isArray ? Array.from({ length }, () => initUndiciData(date)) : [],
+    dataWebsocket: isArray ? Array.from({ length }, () => initWebsocketData(date)) : [],
+    dataNodejs: isArray ? Array.from({ length }, () => initNodejsData(date)) : [],
+  }
+}
+
+const initMemData = (date: string): MemoryDataPoint => ({ date, rss: 0, totalHeap: 0, usedHeap: 0, newSpace: 0, oldSpace: 0 })
+const initCpuData = (date: string): CpuDataPoint => ({ date, cpu: 0, eventLoop: 0 })
+const initLatencyData = (date: string): LatencyDataPoint => ({ date, p90: 0, p95: 0, p99: 0 })
+const initReqData = (date: string): RequestDataPoint => ({ date, count: 0, rps: 0, })
+const initKafkaData = (date: string): KafkaDataPoint => ({ date, consumedMessages: 0, consumers: 0, consumersStreams: 0, consumersTopics: 0, hooksDlqMessagesTotal: 0, hooksMessagesInFlight: 0, producedMessages: 0, producers: 0 })
+const initUndiciData = (date: string): UndiciDataPoint => ({ date, activeRequests: 0, idleSockets: 0, openSockets: 0, pendingRequests: 0, queuedRequests: 0, sizeRequests: 0 })
+const initWebsocketData = (date: string): WebsocketDataPoint => ({ date, connections: 0 })
+const initNodejsData = (date: string): NodejsDataPoint => ({ date, resources: 0 })
+
+export const initServiceMetrics = ({ areMultipleWorkersEnabled, metrics, pid, serviceId, workers }: {
+  metrics: MappedMetrics,
+  pid: number,
+  serviceId: string,
+  workers: number,
+  areMultipleWorkersEnabled: boolean
+}): void => {
+  if (!metrics[pid].services[serviceId]) {
+    metrics[pid].services[serviceId] = { all: initMetricsResponse() }
+
+    if (areMultipleWorkersEnabled) {
+      for (let i = 0; i < workers; i++) {
+        metrics[pid].services[serviceId][i] = initMetricsResponse()
+      }
+    }
+  }
+}