@platformatic/watt-extra 0.1.0

package/lib/wattpro.js

@@ -0,0 +1,580 @@
+import { readFile } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import { createRequire } from 'node:module'
+import { loadConfiguration } from '@platformatic/runtime'
+
+const require = createRequire(import.meta.url)
+
+// Simple replacement for ensureLoggableError
+function ensureLoggableError (err) {
+  if (!err) return err
+  if (typeof err === 'string') return new Error(err)
+  if (err instanceof Error) return err
+  return new Error(String(err))
+}
+
+async function restartRuntime (runtime) {
+  runtime.logger.info('Received SIGUSR2, restarting all services ...')
+
+  try {
+    await runtime.restart()
+  } catch (err) {
+    runtime.logger.error(
+      { err: ensureLoggableError(err) },
+      'Failed to restart services.'
+    )
+  }
+}
+
+class WattPro {
+  #env
+  #logger
+  #require
+  #appDir
+  #applicationName
+  #instanceId
+  #instanceConfig
+  #originalConfig
+  #config
+  #sharedContext
+
+  constructor (app) {
+    this.#env = app.env
+    this.#logger = app.log
+    this.#appDir = app.env.PLT_APP_DIR
+    this.#applicationName = app.applicationName || app.env.PLT_APP_NAME
+    this.#require = createRequire(join(this.#appDir, 'package.json'))
+    this.#instanceId = app.instanceId
+    this.runtime = null
+    this.#sharedContext = {}
+    this.#instanceConfig = app.instanceConfig
+  }
+
+  async spawn () {
+    try {
+      this.runtime = await this.#createRuntime()
+      this.#logger.info('Starting runtime -WATT')
+      await this.runtime.start()
+      await this.updateSharedContext(this.#sharedContext)
+      this.#logger.info('Runtime started')
+    } catch (err) {
+      this.#logger.error(
+        { err: ensureLoggableError(err) },
+        'Failed to start runtime'
+      )
+      throw err
+    }
+  }
+
+  async close () {
+    if (this.runtime) {
+      const runtime = this.runtime
+      this.runtime = null
+
+      this.#logger.info('Closing runtime')
+      await runtime.close()
+      this.#logger.info('Runtime closed')
+    }
+  }
+
+  async applyIccConfigUpdates (config) {
+    this.#logger.info({ config }, 'Applying ICC config updates')
+
+    if (this.#instanceConfig) {
+      this.#instanceConfig.config = config
+    }
+
+    if (config.httpCacheConfig) {
+      try {
+        const undiciConfig = await this.#getUndiciConfig()
+        await this.runtime.updateUndiciInterceptors?.(undiciConfig)
+      } catch (err) {
+        this.#logger.error({ err }, 'Failed to update undici interceptors')
+      }
+    }
+
+    if (config.resources?.services && config.resources.services.length > 0) {
+      const resourceUpdates = config.resources.services.map((service) => ({
+        service: service.name,
+        workers: service.threads,
+        health: {
+          maxHeapTotal: `${service.heap}MB`,
+        },
+      }))
+
+      try {
+        await this.runtime.updateServicesResources(resourceUpdates)
+        this.#logger.info(
+          { resourceUpdates },
+          'Successfully updated service resources'
+        )
+      } catch (err) {
+        this.#logger.error(
+          { err, resourceUpdates },
+          'Failed to update service resources'
+        )
+      }
+    }
+  }
+
+  async updateSharedContext (context) {
+    this.#sharedContext = context
+    await this.runtime?.updateSharedContext?.({ context })
+  }
+
+  async #loadAppConfig () {
+    this.#logger.info('Loading app config')
+    try {
+      const config = await loadConfiguration(this.#appDir)
+      return config
+    } catch (err) {
+      this.#logger.error(err, 'Failed to load app config')
+      throw new Error('Failed to load app config.', { cause: err })
+    }
+  }
+
+  async #createRuntime () {
+    this.#logger.info('Creating runtime')
+    const { Runtime } = this.#require('@platformatic/runtime')
+
+    this.#config = await this.#loadAppConfig()
+
+    this.#logger.info('Patching runtime config')
+
+    this.#originalConfig = structuredClone(this.#config)
+
+    if (this.#config) {
+      this.#patchRuntimeConfig(this.#config)
+    }
+
+    this.#logger.info('Building runtime')
+
+    const runtime = new Runtime(this.#config, { isProduction: true })
+
+    /* c8 ignore next 3 */
+    const restartListener = restartRuntime.bind(null, runtime)
+    process.on('SIGUSR2', restartListener)
+    runtime.on('closed', () => {
+      process.removeListener('SIGUSR2', restartListener)
+    })
+
+    await this.#configureServices(runtime)
+
+    try {
+      await runtime.init()
+    } catch (e) {
+      await runtime.close()
+      throw e
+    }
+
+    return runtime
+  }
+
+  #patchRuntimeConfig (config) {
+    this.#configureRuntime(config)
+    this.#configureTelemetry(config)
+    this.#configureHttpCaching(config)
+    this.#configureHealth(config)
+    this.#configureSystemResources(config)
+    this.#configureScheduler(config)
+  }
+
+  #configureRuntime (config) {
+    const { https, ...serverConfig } = config.server ?? {}
+    config.server = {
+      ...serverConfig,
+      hostname: this.#env.PLT_APP_HOSTNAME || serverConfig.hostname,
+      port: this.#env.PLT_APP_PORT || serverConfig.port,
+    }
+
+    config.hotReload = false
+    config.restartOnError = 1000
+    config.metrics = {
+      server: 'hide',
+      defaultMetrics: {
+        enabled: true,
+      },
+      hostname: this.#env.PLT_APP_HOSTNAME || '127.0.0.1',
+      port: this.#env.PLT_METRICS_PORT || 9090,
+      labels: {
+        serviceId: 'main',
+        applicationId: this.#instanceConfig.applicationId,
+        instanceId: this.#instanceId,
+      },
+    }
+
+    if (this.#env.PLT_DISABLE_FLAMEGRAPHS !== true) {
+      if (config.preload === undefined) {
+        config.preload = []
+      }
+      const pprofPath = require.resolve('@platformatic/wattpm-pprof-capture')
+      config.preload.push(pprofPath)
+    }
+
+    this.#configureUndici(config)
+    config.managementApi = true
+  }
+
+  #getUndiciConfig () {
+    const config = this.#config
+
+    const undiciConfig = structuredClone(this.#originalConfig.undici ?? {})
+
+    if (undiciConfig.interceptors === undefined) {
+      undiciConfig.interceptors = []
+    }
+
+    const enableSlicerInterceptor =
+      this.#instanceConfig?.enableSlicerInterceptor ?? false
+    if (enableSlicerInterceptor) {
+      const slicerInterceptorConfig = this.#getSlicerInterceptorConfig(config)
+      if (slicerInterceptorConfig) {
+        undiciConfig.interceptors.push(slicerInterceptorConfig)
+      }
+    }
+
+    const enableTrafficInterceptor =
+      this.#instanceConfig?.enableTrafficInterceptor ?? false
+    if (enableTrafficInterceptor) {
+      const trafficInterceptorConfig =
+        this.#getTrafficInterceptorConfig()
+      if (trafficInterceptorConfig) {
+        undiciConfig.interceptors.push(trafficInterceptorConfig)
+      }
+    }
+
+    return undiciConfig
+  }
+
+  #configureUndici (config) {
+    config.undici = this.#getUndiciConfig(config)
+  }
+
+  #getTrafficInterceptorConfig () {
+    if (!this.#instanceConfig?.iccServices?.trafficante?.url) {
+      return
+    }
+    const { origin: trafficanteOrigin, pathname: trafficantePath } = new URL(
+      this.#instanceConfig.iccServices.trafficante.url
+    )
+    return {
+      module: this.#require.resolve(
+        'undici-traffic-interceptor'
+      ),
+      options: {
+        labels: {
+          applicationId: this.#instanceConfig.applicationId,
+        },
+        bloomFilter: {
+          size: 100000,
+          errorRate: 0.01,
+        },
+        maxResponseSize: 5 * 1024 * 1024, // 5MB
+        trafficInspectorOptions: {
+          url: trafficanteOrigin,
+          pathSendBody: join(trafficantePath, '/requests'),
+          pathSendMeta: join(trafficantePath, '/requests/hash'),
+        },
+        matchingDomains: [this.#env.PLT_APP_INTERNAL_SUB_DOMAIN],
+      },
+    }
+  }
+
+  #getSlicerInterceptorConfig (config) {
+    // We need to initialize the slicer interceptor even if there is no cache config
+    // to be able to update the onfiguration at runtime
+    const defaultCacheConfig = {
+      rules: [
+        {
+          routeToMatch: 'http://plt.slicer.default/',
+          headers: {},
+        },
+      ],
+    }
+
+    // This is the cache config from ICC
+    const httpCacheConfig =
+      this.#instanceConfig?.config?.httpCacheConfig ?? defaultCacheConfig
+    let autoGeneratedConfig = null
+    if (httpCacheConfig) {
+      try {
+        autoGeneratedConfig = httpCacheConfig
+      } catch (e) {
+        this.#logger.error(
+          { err: ensureLoggableError(e) },
+          'Failed to parse auto generated cache config'
+        )
+      }
+    }
+
+    let userConfig = null
+    // This is the user config from the environment variable
+    if (this.#env.PLT_CACHE_CONFIG) {
+      try {
+        userConfig = JSON.parse(this.#env.PLT_CACHE_CONFIG)
+      } catch (e) {
+        this.#logger.error(
+          { err: ensureLoggableError(e) },
+          'Failed to parse user cache config'
+        )
+      }
+    }
+
+    if (!userConfig && !autoGeneratedConfig) return null
+
+    let cacheConfig = userConfig ?? autoGeneratedConfig
+    if (autoGeneratedConfig && userConfig) {
+      cacheConfig = this.#mergeCacheConfigs(autoGeneratedConfig, userConfig)
+    }
+
+    const cacheTagsHeader = this.#getCacheTagsHeader(config)
+
+    for (const rule of cacheConfig.rules ?? []) {
+      if (rule.cacheTags) {
+        if (!rule.headers) {
+          rule.headers = {}
+        }
+        rule.headers[cacheTagsHeader] = rule.cacheTags
+        delete rule.cacheTags
+      }
+    }
+
+    return {
+      module: this.#require.resolve('@platformatic/slicer-interceptor'),
+      options: cacheConfig,
+    }
+  }
+
+  #mergeCacheConfigs (autoGeneratedConfig, userConfig) {
+    const mergedConfig = { ...userConfig }
+
+    for (const rule of autoGeneratedConfig.rules ?? []) {
+      const ruleIndex = mergedConfig.rules.findIndex(
+        (r) => r.routeToMatch === rule.routeToMatch
+      )
+
+      if (ruleIndex === -1) {
+        mergedConfig.rules.push(rule)
+      }
+    }
+
+    return mergedConfig
+  }
+
+  #configureTelemetry (config) {
+    const enableOpenTelemetry =
+      !!this.#instanceConfig?.enableOpenTelemetry &&
+      !!this.#instanceConfig?.iccServices?.riskEngine?.url
+
+    // We need to always set an opentelemetry config to pass a telemetry
+    // applicationName to render a taxonomy diagram
+    config.telemetry = config.telemetry ?? {
+      enabled: enableOpenTelemetry,
+      applicationName: `${this.#applicationName}`,
+      skip: [
+        { method: 'GET', path: '/documentation' },
+        { method: 'GET', path: '/documentation/json' },
+      ],
+      exporter: {
+        type: 'otlp',
+        options: {
+          url:
+            this.#instanceConfig?.iccServices?.riskEngine?.url + '/v1/traces',
+          headers: {
+            'x-platformatic-application-id': this.#instanceConfig.applicationId,
+          },
+          keepAlive: true,
+          httpAgentOptions: {
+            rejectUnauthorized: false,
+          },
+        },
+      },
+    }
+  }
+
+  #configureHttpCaching (config) {
+    const cacheTagsHeader = this.#getCacheTagsHeader(config)
+    const httpCache = this.#instanceConfig?.httpCache?.clientOpts
+
+    if (!httpCache?.host) {
+      this.#logger.warn(
+        'Missing required environment variables for Redis cache, not setting up HTTP cache'
+      )
+      return
+    }
+
+    config.httpCache = {
+      ...config.httpCache,
+      cacheTagsHeader,
+      store: this.#require.resolve('@platformatic/undici-cache-redis'),
+      clientOpts: httpCache,
+    }
+  }
+
+  #configureHealth (config) {
+    config.health = {
+      ...config.health,
+      enabled: true,
+      interval: 1000,
+      maxUnhealthyChecks: 30,
+    }
+  }
+
+  #configureScheduler (config) {
+    // Disable all watt schedules. We do that because
+    // we will create/update them in ICC, not on watt in memory
+    if (config.scheduler) {
+      config.scheduler = config.scheduler.map((scheduler) => ({
+        ...scheduler,
+        enabled: false,
+      }))
+    }
+  }
+
+  #configureSystemResources (config) {
+    if (!this.#instanceConfig) {
+      return
+    }
+    // Set system wide resources
+    const resources = this.#instanceConfig?.config?.resources
+    if (!resources) {
+      return
+    }
+
+    const { threads, heap } = resources
+
+    if (threads > 0) {
+      config.workers = threads
+    }
+
+    if (heap > 0) {
+      config.health ??= {}
+      config.health.maxHeapTotal = heap * 1024 * 1024
+    }
+
+    // In v3 we renamed services to applications, so we support both (note that this is coming from icc)
+    const res = resources.services || resources.applications
+
+    // Set services resources
+    for (const application of config.applications ?? []) {
+      let applicationResources = res?.find((s) => s.name === application.id)
+
+      if (!applicationResources) {
+        applicationResources = {
+          threads,
+          heap,
+        }
+      }
+      application.workers = applicationResources.threads
+      application.health ??= {}
+      application.health.maxHeapTotal = applicationResources.heap * 1024 * 1024
+    }
+  }
+
+  async #configureServices (runtime) {
+    if (typeof runtime.setApplicationConfigPatch !== 'function') {
+      return
+    }
+
+    const config = runtime.getRuntimeConfig(true)
+
+    for (const app of config.applications ?? []) {
+      if (app.type === 'next') {
+        await this.#configureNextService(runtime, app)
+      } else if (
+        [
+          '@platformatic/service',
+          '@platformatic/composer',
+          '@platformatic/db',
+        ].includes(app.type)
+      ) {
+        await this.#configurePlatformaticServices(runtime, app)
+      }
+    }
+  }
+
+  async #configureNextService (runtime, service) {
+    let nextSchema
+
+    try {
+      const nextPackage = createRequire(
+        resolve(service.path, 'index.js')
+      ).resolve('@platformatic/next')
+      nextSchema = JSON.parse(
+        await readFile(resolve(nextPackage, '../schema.json'), 'utf8')
+      )
+    } catch (e) {
+      this.#logger.error(
+        { err: ensureLoggableError(e) },
+        `Failed to load @platformatic/next schema for service ${service.id}`
+      )
+      throw e
+    }
+
+    const patches = []
+
+    if ('cache' in nextSchema.properties) {
+      const httpCache = this.#instanceConfig?.httpCache?.clientOpts || {}
+      const { keyPrefix, host, port, username, password } = httpCache
+
+      if (!keyPrefix || !host || !port) {
+        this.#logger.warn(
+          'Missing required environment variables for Redis cache, not setting up HTTP next cache'
+        )
+      } else {
+        patches.push({
+          op: 'add',
+          path: '/cache',
+          value: {
+            adapter: 'valkey',
+            url: `valkey://${username}:${password}@${host}:${port}`,
+            prefix: keyPrefix,
+            maxTTL: 604800, // 86400 * 7
+          },
+        })
+      }
+    }
+
+    // Add trailingSlash true to Next entrypoints that support it
+    // This is technically useless as Next.js will manage it at build time, but we keep it
+    // in case in the future they compare build and production next.config.js
+    if (
+      service.entrypoint &&
+      nextSchema.properties.next?.properties.trailingSlash?.type === 'boolean'
+    ) {
+      patches.push({ op: 'add', path: '/next/trailingSlash', value: true })
+    }
+
+    if (patches.length) {
+      this.#patchService(runtime, service.id, patches)
+    }
+  }
+
+  async #configurePlatformaticServices (runtime, app) {
+    if (app.entrypoint) {
+      const config = app
+      const patches = [{ op: 'add', path: '/server/trustProxy', value: true }]
+
+      if (!config.server) {
+        patches.unshift({ op: 'add', path: '/server', value: {} })
+      }
+
+      patches.push({ op: 'remove', path: '/server/https' })
+
+      this.#patchService(runtime, app.id, patches)
+    }
+  }
+
+  async #patchService (runtime, id, patches) {
+    this.#logger.info({ patches }, `Applying patches to service ${id} ...`)
+    runtime.setApplicationConfigPatch(id, patches)
+  }
+
+  #getCacheTagsHeader (config) {
+    const customCacheTagsHeader = config.httpCache?.cacheTagsHeader
+    const defaultCacheTagsHeader = this.#env.PLT_DEFAULT_CACHE_TAGS_HEADER
+    return customCacheTagsHeader ?? defaultCacheTagsHeader
+  }
+}
+
+export default WattPro

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@platformatic/watt-extra",
+  "version": "0.1.0",
+  "description": "The Platformatic runtime manager",
+  "type": "module",
+  "scripts": {
+    "lint": "eslint",
+    "test": "pnpm run lint && borp -c 1 --timeout=180000 ./test/*.test.js",
+    "start": "node index.js"
+  },
+  "bin": {
+    "watt-extra": "./cli.js"
+  },
+  "devDependencies": {
+    "@fastify/websocket": "^11.1.0",
+    "@platformatic/composer": "^3.0.1",
+    "@platformatic/next": "^3.0.1",
+    "@platformatic/node": "^3.0.1",
+    "@platformatic/service": "^3.0.1",
+    "borp": "^0.20.0",
+    "eslint": "9",
+    "fastify": "^5.4.0",
+    "fastify-plugin": "^5.0.1",
+    "neostandard": "^0.12.0",
+    "next": "^15.3.4",
+    "platformatic": "^3.0.1",
+    "pprof-format": "^2.1.0",
+    "why-is-node-running": "^2.3.0"
+  },
+  "dependencies": {
+    "@datadog/pprof": "^5.9.0",
+    "@fastify/error": "^4.2.0",
+    "@platformatic/runtime": "^3.0.1",
+    "@platformatic/slicer-interceptor": "^0.3.0",
+    "@platformatic/undici-cache-redis": "^0.7.1",
+    "@platformatic/wattpm-pprof-capture": "^3.0.1",
+    "undici-traffic-interceptor": "^0.1.4",
+    "avvio": "^9.1.0",
+    "chalk": "^4.1.2",
+    "commist": "^3.2.0",
+    "env-schema": "^6.0.1",
+    "execa": "^9.6.0",
+    "help-me": "^5.0.0",
+    "minimist": "^1.2.8",
+    "pino": "^9.7.0",
+    "pino-pretty": "^13.0.0",
+    "undici": "^7.11.0",
+    "ws": "^8.18.3"
+  }
+}