@platformatic/runtime 3.53.0-alpha.0 → 3.54.0

package/config.d.ts

@@ -493,6 +493,10 @@ export type PlatformaticRuntimeConfig = {
           additionalProperties?: never;
           [k: string]: unknown;
         };
+    /**
+     * Enable the OpenTelemetry diagnostic logger. Diagnostic messages are forwarded to the Platformatic global logger using the current logger level.
+     */
+    diagLogger?: boolean | string;
   };
   verticalScaler?: {
     enabled?: boolean;

package/lib/runtime.js

@@ -77,6 +77,8 @@ const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
 const kHeapCheckCounter = Symbol('plt.runtime.worker.heapCheckCounter')
 const kLastHeapStats = Symbol('plt.runtime.worker.lastHeapStats')
 const kHealthITCTimeoutMs = 5000
+const kApplicationRestartsMetricName = 'platformatic_application_restarts_total'
+const kApplicationRestartsMetricHelp = 'Total number of restarts triggered by the runtime for an application.'
 
 const MAX_LISTENERS_COUNT = 100
 
@@ -133,6 +135,7 @@ export class Runtime extends EventEmitter {
 
   #applicationsConfigsPatches
   #applications
+  #applicationRestartCounts
   #workers
   #workersBroadcastChannel
   #workerITCHandlers
@@ -158,6 +161,7 @@ export class Runtime extends EventEmitter {
     this.#isProduction = this.#context.isProduction ?? this.#context.production ?? false
     this.#concurrency = Math.max(1, config.startupConcurrency ?? this.#context.concurrency ?? DEFAULT_CONCURRENCY)
     this.#applications = new Map()
+    this.#applicationRestartCounts = new Map()
     this.#workers = new RoundRobinMap()
     this.#url = undefined
     this.#entrypointPort = undefined
@@ -501,6 +505,7 @@ export class Runtime extends EventEmitter {
       }
 
       this.#applications.set(application.id, application)
+      this.#applicationRestartCounts.set(application.id, this.#applicationRestartCounts.get(application.id) ?? 0)
       setupInvocations.push([application])
       toStart.push(application.id)
     }
@@ -542,6 +547,7 @@ export class Runtime extends EventEmitter {
     for (const application of applications) {
       this.#dynamicWorkersScaler?.remove(application)
       this.#applications.delete(application)
+      this.#applicationRestartCounts.delete(application)
     }
 
     for (const application of applications) {
@@ -712,6 +718,7 @@ export class Runtime extends EventEmitter {
         await this.#replaceWorker(config, applicationConfig, workersCount, id, i, worker, true)
       }
 
+      this.#incrementApplicationRestartCount(id)
       this.emitAndNotify('application:restarted', id)
     } finally {
       this.#restartingApplications.delete(id)
@@ -1151,6 +1158,10 @@ export class Runtime extends EventEmitter {
   async getMetrics (format = 'json') {
     let metrics = null
 
+    const applicationRestartMetrics = format === 'json'
+      ? this.#getApplicationRestartMetricsJson()
+      : this.#formatApplicationRestartMetricsText()
+
     // Get process-level metrics once from main thread registry (if available)
     let processMetricsJson = null
     if (this.#processMetricsRegistry) {
@@ -1216,9 +1227,68 @@ export class Runtime extends EventEmitter {
       }
     }
 
+    if (metrics !== null && applicationRestartMetrics.length > 0) {
+      metrics = format === 'json'
+        ? [...applicationRestartMetrics, ...metrics]
+        : applicationRestartMetrics + metrics
+    }
+
     return { metrics }
   }
 
+  #incrementApplicationRestartCount (applicationId) {
+    this.#applicationRestartCounts.set(applicationId, (this.#applicationRestartCounts.get(applicationId) ?? 0) + 1)
+  }
+
+  #getApplicationRestartMetricLabels (applicationId) {
+    return {
+      ...this.#config.metrics?.labels,
+      [this.#metricsLabelName]: applicationId
+    }
+  }
+
+  #getApplicationRestartMetricsJson () {
+    const metrics = []
+
+    for (const applicationId of this.#applications.keys()) {
+      metrics.push({
+        name: kApplicationRestartsMetricName,
+        help: kApplicationRestartsMetricHelp,
+        type: 'counter',
+        aggregator: 'sum',
+        values: [{
+          value: this.#applicationRestartCounts.get(applicationId) ?? 0,
+          labels: this.#getApplicationRestartMetricLabels(applicationId),
+          metricName: kApplicationRestartsMetricName
+        }]
+      })
+    }
+
+    return metrics
+  }
+
+  #formatApplicationRestartMetricsText () {
+    let output = ''
+
+    for (const applicationId of this.#applications.keys()) {
+      const labelParts = []
+      const labels = this.#getApplicationRestartMetricLabels(applicationId)
+
+      for (const [key, val] of Object.entries(labels)) {
+        const escapedVal = String(val).replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n')
+        labelParts.push(`${key}="${escapedVal}"`)
+      }
+
+      const labelStr = labelParts.length > 0 ? `{${labelParts.join(',')}}` : ''
+
+      output += `# HELP ${kApplicationRestartsMetricName} ${kApplicationRestartsMetricHelp}\n`
+      output += `# TYPE ${kApplicationRestartsMetricName} counter\n`
+      output += `${kApplicationRestartsMetricName}${labelStr} ${this.#applicationRestartCounts.get(applicationId) ?? 0}\n`
+    }
+
+    return output
+  }
+
   // Apply labels to process metrics and push to output array (for JSON format)
   #applyLabelsToMetrics (processMetrics, labels, outputArray) {
     for (let i = 0; i < processMetrics.length; i++) {
@@ -2152,6 +2222,7 @@ export class Runtime extends EventEmitter {
             )
 
             await this.#replaceWorker(config, applicationConfig, workersCount, id, index, worker)
+            this.#incrementApplicationRestartCount(id)
           } catch (e) {
             this.logger.error(
               `Cannot replace the ${errorLabel}. Forcefully terminating it ...`
@@ -2477,6 +2548,7 @@ export class Runtime extends EventEmitter {
         try {
           await this.#setupWorker(config, applicationConfig, workersCount, id, index)
           await this.#startWorker(config, applicationConfig, workersCount, id, index, silent, bootstrapAttempt)
+          this.#incrementApplicationRestartCount(id)
 
           this.logger.info(
             `The ${this.#workerExtendedLabel(id, index, workersCount)} has been successfully restarted ...`

package/lib/worker/itc.js

@@ -1,5 +1,5 @@
 import { ensureLoggableError, executeInParallel, executeWithTimeout, kTimeout } from '@platformatic/foundation'
-import { ITC } from '@platformatic/itc'
+import { initializeITCTelemetry, ITC } from '@platformatic/itc'
 import { Unpromise } from '@watchable/unpromise'
 import { once } from 'node:events'
 import { createRequire } from 'node:module'
@@ -147,7 +147,9 @@ export async function waitEventFromITC (worker, event) {
   return safeHandleInITC(worker, () => once(worker[kITC], event))
 }
 
-export function setupITC (controller, application, dispatcher, sharedContext) {
+export async function setupITC (controller, application, dispatcher, sharedContext) {
+  await initializeITCTelemetry()
+
   const logger = globalThis.platformatic.logger
   const messaging = new MessagingITC(controller.applicationConfig.id, workerData.config, logger)
 

package/lib/worker/main.js

@@ -271,7 +271,7 @@ async function main () {
   }
 
   // Setup interaction with parent port
-  const itc = setupITC(controller, applicationConfig, threadDispatcher, sharedContext)
+  const itc = await setupITC(controller, applicationConfig, threadDispatcher, sharedContext)
   globalThis[kITC] = itc
   globalThis.platformatic.itc = itc
 

package/lib/worker/messaging.js

@@ -1,5 +1,15 @@
 import { ensureLoggableError, executeWithTimeout, kTimeout } from '@platformatic/foundation'
-import { errors, generateRequest, generateResponse, ITC, parseRequest, sanitize } from '@platformatic/itc'
+import {
+  errors,
+  generateRequest,
+  generateResponse,
+  ITC,
+  parseRequest,
+  sanitize,
+  startOutgoingMessagingSpan,
+  startOutgoingMessagingSpanSync,
+  traceIncomingMessagingHandler
+} from '@platformatic/itc'
 import { MessagingError } from '../errors.js'
 import { RoundRobinMap } from './round-robin-map.js'
 import { kITC, kWorkersBroadcast } from './symbols.js'
@@ -50,69 +60,106 @@ export class MessagingITC extends ITC {
   handle (message, handler) {
     if (typeof message === 'object') {
       for (const [name, fn] of Object.entries(message)) {
-        super.handle(name, fn)
+        super.handle(name, this.#wrapHandler(name, fn))
       }
     } else {
-      super.handle(message, handler)
+      super.handle(message, this.#wrapHandler(message, handler))
     }
   }
 
   async send (application, name, message, options) {
-    // Get the next worker for the application
-    const worker = this.#workers.next(application)
+    const telemetry = startOutgoingMessagingSpan('send', this.#id, application, name, {
+      telemetryContext: options?.telemetryContext,
+      telemetryMetadata: options?.telemetryMetadata
+    })
+    let error = null
 
-    if (!worker) {
-      throw new MessagingError(application, 'No workers available')
-    }
+    try {
+      // Get the next worker for the application
+      const worker = this.#workers.next(application)
 
-    if (!worker.channel) {
-      // Use twice the value here as a fallback measure. The target handler in the main thread is forwarding
-      // the request to the worker, using executeWithTimeout with the user set timeout value.
-      const channel = await executeWithTimeout(
-        globalThis[kITC].send('getWorkerMessagingChannel', {
-          id: this.#id,
-          application: worker.application,
-          worker: worker.worker
-        }),
-        this.#timeout * 2
-      )
+      if (!worker) {
+        throw new MessagingError(application, 'No workers available')
+      }
+
+      if (!worker.channel) {
+        // Use twice the value here as a fallback measure. The target handler in the main thread is forwarding
+        // the request to the worker, using executeWithTimeout with the user set timeout value.
+        const channel = await executeWithTimeout(
+          globalThis[kITC].send('getWorkerMessagingChannel', {
+            id: this.#id,
+            application: worker.application,
+            worker: worker.worker
+          }),
+          this.#timeout * 2
+        )
+
+        /* c8 ignore next 3 - Hard to test */
+        if (channel === kTimeout) {
+          throw new MessagingError(application, 'Timeout while waiting for a communication channel.')
+        }
+
+        worker.channel = channel
+        this.#setupChannel(channel)
 
-      /* c8 ignore next 3 - Hard to test */
-      if (channel === kTimeout) {
-        throw new MessagingError(application, 'Timeout while waiting for a communication channel.')
+        channel[kPendingResponses] = new Map()
+        channel.on('close', this.#handlePendingResponse.bind(this, channel))
       }
 
-      worker.channel = channel
-      this.#setupChannel(channel)
+      const context = { ...options }
+      context.channel = worker.channel
+      context.application = worker.application
+      context.trackResponse = true
 
-      channel[kPendingResponses] = new Map()
-      channel.on('close', this.#handlePendingResponse.bind(this, channel))
-    }
+      if (options?.meta || telemetry?.meta) {
+        context.meta = { ...options?.meta, ...telemetry?.meta }
+      }
 
-    const context = { ...options }
-    context.channel = worker.channel
-    context.application = worker.application
-    context.trackResponse = true
+      const send = () => executeWithTimeout(super.send(name, message, context), this.#timeout)
+      const response = telemetry ? await telemetry.run(send) : await send()
 
-    const response = await executeWithTimeout(super.send(name, message, context), this.#timeout)
+      if (response === kTimeout) {
+        throw new MessagingError(application, 'Timeout while waiting for a response.')
+      }
 
-    if (response === kTimeout) {
-      throw new MessagingError(application, 'Timeout while waiting for a response.')
+      return response
+    } catch (err) {
+      error = err
+      throw err
+    } finally {
+      telemetry?.end(error)
     }
-
-    return response
   }
 
-  notify (application, name, message) {
-    const request = generateRequest(name, message)
+  notify (application, name, message, options) {
+    const telemetry = startOutgoingMessagingSpanSync('notify', this.#id, application, name, {
+      telemetryContext: options?.telemetryContext,
+      telemetryMetadata: options?.telemetryMetadata
+    })
+    const metadata = options?.meta || telemetry?.meta ? { ...options?.meta, ...telemetry?.meta } : undefined
+    const request = generateRequest(name, message, metadata)
+
+    let error = null
 
-    let channel = this.#notificationsChannels.get(application)
-    if (!channel) {
-      channel = new BroadcastChannel(`plt.messaging.notifications-${application}`)
-      this.#notificationsChannels.set(application, channel)
-    }
+    try {
+      let channel = this.#notificationsChannels.get(application)
+      if (!channel) {
+        channel = new BroadcastChannel(`plt.messaging.notifications-${application}`)
+        this.#notificationsChannels.set(application, channel)
+      }
 
-    channel.postMessage(sanitize(request))
+      const postMessage = () => channel.postMessage(sanitize(request))
+      if (telemetry) {
+        telemetry.run(postMessage)
+      } else {
+        postMessage()
+      }
+    } catch (err) {
+      error = err
+      throw err
+    } finally {
+      telemetry?.end(error)
+    }
   }
 
   async addSource (channel) {
@@ -165,7 +212,7 @@ export class MessagingITC extends ITC {
   #setupChannel (channel) {
     // Setup the message for processing
     channel.on('message', event => {
-      this.#listener(event, { channel })
+      this.#listener(event, { channel, message: event })
     })
   }
 
@@ -207,12 +254,16 @@ export class MessagingITC extends ITC {
         throw new errors.HandlerNotFoundError(request.name)
       }
 
-      await handler(request.data)
+      await handler(request.data, { application: this.#id, message: request, notification: true })
     } catch (error) {
       this.#logger.error({ error }, `"Handler for the "${request.name}" message failed.`)
     }
   }
 
+  #wrapHandler (messageName, handler) {
+    return (data, context) => traceIncomingMessagingHandler(this.#id, messageName, handler, data, context)
+  }
+
   #handlePendingResponse (channel) {
     for (const { application, request } of channel[kPendingResponses].values()) {
       this._emitResponse(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/runtime",
-  "version": "3.53.0-alpha.0",
+  "version": "3.54.0",
   "description": "",
   "main": "index.js",
   "type": "module",
@@ -36,14 +36,14 @@
     "typescript": "^5.5.4",
     "undici-oidc-interceptor": "^0.5.0",
     "why-is-node-running": "^2.2.2",
-    "@platformatic/composer": "3.53.0-alpha.0",
-    "@platformatic/gateway": "3.53.0-alpha.0",
-    "@platformatic/db": "3.53.0-alpha.0",
-    "@platformatic/service": "3.53.0-alpha.0",
-    "@platformatic/sql-graphql": "3.53.0-alpha.0",
-    "@platformatic/node": "3.53.0-alpha.0",
-    "@platformatic/sql-mapper": "3.53.0-alpha.0",
-    "@platformatic/wattpm-pprof-capture": "3.53.0-alpha.0"
+    "@platformatic/db": "3.54.0",
+    "@platformatic/gateway": "3.54.0",
+    "@platformatic/node": "3.54.0",
+    "@platformatic/service": "3.54.0",
+    "@platformatic/composer": "3.54.0",
+    "@platformatic/sql-mapper": "3.54.0",
+    "@platformatic/sql-graphql": "3.54.0",
+    "@platformatic/wattpm-pprof-capture": "3.54.0"
   },
   "dependencies": {
     "@fastify/accepts": "^5.0.0",
@@ -65,7 +65,7 @@
     "help-me": "^5.0.0",
     "minimist": "^1.2.8",
     "pino": "^10.1.0",
-    "pino-opentelemetry-transport": "^2.0.0",
+    "pino-opentelemetry-transport": "^3.0.0",
     "pino-pretty": "^13.0.0",
     "semgrator": "^0.3.0",
     "sonic-boom": "^4.2.0",
@@ -73,12 +73,12 @@
     "undici": "^7.0.0",
     "undici-thread-interceptor": "^1.3.1",
     "ws": "^8.16.0",
-    "@platformatic/foundation": "3.53.0-alpha.0",
-    "@platformatic/basic": "3.53.0-alpha.0",
-    "@platformatic/generators": "3.53.0-alpha.0",
-    "@platformatic/metrics": "3.53.0-alpha.0",
-    "@platformatic/itc": "3.53.0-alpha.0",
-    "@platformatic/telemetry": "3.53.0-alpha.0"
+    "@platformatic/basic": "3.54.0",
+    "@platformatic/foundation": "3.54.0",
+    "@platformatic/itc": "3.54.0",
+    "@platformatic/metrics": "3.54.0",
+    "@platformatic/generators": "3.54.0",
+    "@platformatic/telemetry": "3.54.0"
   },
   "engines": {
     "node": ">=22.19.0"

package/schema.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.53.0-alpha.0.json",
+  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.54.0.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Platformatic Runtime Config",
   "type": "object",
@@ -2595,6 +2595,17 @@
               }
             }
           ]
+        },
+        "diagLogger": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "string"
+            }
+          ],
+          "description": "Enable the OpenTelemetry diagnostic logger. Diagnostic messages are forwarded to the Platformatic global logger using the current logger level."
         }
       },
       "required": [