@vtex/api 6.51.1-beta → 6.52.0

package/CHANGELOG.md

@@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [6.52.0]
+### Added
+- Cluster-wide Prometheus `/metrics` aggregation. Each cluster worker keeps its own
+  prom-client registry, so serving `/metrics` from a single round-robin selected
+  worker exposed only that worker's counters; Prometheus read the per-scrape braid
+  of independent monotonic counters as counter resets and inflated `rate()` /
+  `increase()` on `runtime_http_*` counters by orders of magnitude. In multi-worker
+  mode the worker answering a scrape now asks the master for a merged, monotonic
+  view built from every worker's registry over the existing cluster IPC (via
+  prom-client's `AggregatorRegistry`), with a bounded timeout and a local-registry
+  fallback. Single-worker mode (`workers === 1`, includes `LINKED`) is unchanged.
+  Backport of #667 to the 6.x line.
+### Fixed
+- `runtime_http_*` metrics no longer emit samples without a `handler` label. Requests
+  that never reach a named handler (unmatched paths, replica-level rate limit
+  rejections, errors before the route pipeline) were counted with
+  `handler: undefined`; Node's cluster IPC serializes worker registries as JSON, which
+  drops `undefined` values, so the aggregated `/metrics` exposed a second, unnamed
+  series that Prometheus reads as `handler=""`. Those requests are now labelled
+  `handler="undefined"` — the same value prom-client rendered locally before cluster
+  aggregation — keeping dashboards and alerts that filter on it working. Backport of
+  #673 to the 6.x line.
+- `/_status` requests are now reported as `handler="builtin:status-track"`, matching
+  the other builtin handlers, instead of falling into the unnamed bucket.
+
 ## [6.51.0] - 2026-06-23
 ### Added
 - Base `IOClients` getter `janusCatalogSystem` (Janus Catalog) and

package/lib/service/master.d.ts

@@ -1,2 +1,4 @@
+import { Worker } from 'cluster';
 import { ServiceJSON } from './worker/runtime/typings';
+export declare const onMessage: (worker: Worker, message: any) => void;
 export declare const startMaster: (service: ServiceJSON) => void;

package/lib/service/master.js

@@ -3,11 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.startMaster = void 0;
+exports.startMaster = exports.onMessage = void 0;
 const cluster_1 = __importDefault(require("cluster"));
 const os_1 = require("os");
 const constants_1 = require("../constants");
 const logger_1 = require("./logger");
+const clusterMetricsAggregator_1 = require("./metrics/clusterMetricsAggregator");
 const listeners_1 = require("./worker/listeners");
 const statusTrack_1 = require("./worker/runtime/statusTrack");
 let handledSignal;
@@ -19,7 +20,12 @@ const onMessage = (worker, message) => {
         (0, statusTrack_1.trackStatus)();
         (0, statusTrack_1.broadcastStatusTrack)();
     }
-    else {
+    else if ((0, clusterMetricsAggregator_1.isAggMetricsRequest)(message)) {
+        (0, clusterMetricsAggregator_1.handleWorkerMetricsRequest)(worker, message);
+    }
+    else if (!(0, clusterMetricsAggregator_1.isPromClientMessage)(message)) {
+        // prom-client's own cluster messages are handled by its cluster listener;
+        // anything else that reaches here is genuinely unexpected.
         listeners_1.logger.warn({
             content: message,
             message: 'Worker sent message',
@@ -27,6 +33,7 @@ const onMessage = (worker, message) => {
         });
     }
 };
+exports.onMessage = onMessage;
 const onExit = (worker, code, signal) => {
     if (!constants_1.LINKED && worker.exitedAfterDisconnect === false) {
         listeners_1.logger.error({
@@ -70,6 +77,11 @@ const startMaster = (service) => {
     if (service.deterministicVary) {
         process.env.DETERMINISTIC_VARY = 'true';
     }
+    // Set up the master-side Prometheus aggregator so workers can request a
+    // merged, monotonic /metrics view across the whole cluster over IPC.
+    if (numWorkers > 1) {
+        (0, clusterMetricsAggregator_1.initMasterAggregatorRegistry)();
+    }
     // Setup dubugger
     if (constants_1.LINKED) {
         cluster_1.default.setupMaster({ inspectPort: constants_1.INSPECT_DEBUGGER_PORT });
@@ -82,7 +94,7 @@ const startMaster = (service) => {
     }
     cluster_1.default.on('online', onOnline);
     cluster_1.default.on('exit', onExit);
-    cluster_1.default.on('message', onMessage);
+    cluster_1.default.on('message', exports.onMessage);
     process.on('SIGINT', handleSignal(SIGINT_TIMEOUT_S));
     process.on('SIGTERM', handleSignal(shutdownTimeout));
 };

package/lib/service/metrics/clusterMetricsAggregator.d.ts

@@ -0,0 +1,72 @@
+import { Worker } from 'cluster';
+import { AggregatorRegistry } from 'prom-client';
+/**
+ * Cluster-wide Prometheus metrics aggregation.
+ *
+ * In multi-worker mode each cluster worker keeps its own prom-client default
+ * registry. Serving `/metrics` from a single (round-robin selected) worker
+ * therefore exposes only that worker's local counters, which makes Prometheus
+ * treat the per-scrape braid of independent counters as counter resets and
+ * inflates `rate()`/`increase()` by orders of magnitude.
+ *
+ * This module uses prom-client's `AggregatorRegistry` cluster IPC protocol so
+ * that the worker answering a scrape asks the master for a merged, monotonic
+ * view built from every worker's registry in one pass (no double-counting of
+ * the answering worker).
+ */
+/** Tagged IPC message: worker -> master, "please build the cluster aggregate". */
+export declare const AGG_METRICS_REQ = "vtex-api:aggMetricsReq";
+/** Tagged IPC message: master -> worker, carries the aggregate (or an error). */
+export declare const AGG_METRICS_RES = "vtex-api:aggMetricsRes";
+export interface AggMetricsReqMessage {
+    type: typeof AGG_METRICS_REQ;
+    id: number;
+}
+export interface AggMetricsResMessage {
+    type: typeof AGG_METRICS_RES;
+    id: number;
+    body?: string;
+    error?: string;
+}
+export declare const isAggMetricsRequest: (message: any) => message is AggMetricsReqMessage;
+export declare const isAggMetricsResponse: (message: any) => message is AggMetricsResMessage;
+/**
+ * prom-client's cluster protocol emits its own tagged IPC messages
+ * (`prom-client:getMetricsReq` / `prom-client:getMetricsRes`). They are handled
+ * by prom-client's own cluster listeners, so our `onMessage` handlers must
+ * ignore them instead of warning about "unknown" messages.
+ */
+export declare const isPromClientMessage: (message: any) => boolean;
+/**
+ * Constructs the master-side `AggregatorRegistry` (idempotent). Its constructor
+ * installs prom-client's master `cluster.on('message')` collector, which is
+ * what makes `clusterMetrics()` work. Must be called in the master before any
+ * worker can answer a scrape.
+ */
+export declare const initMasterAggregatorRegistry: () => AggregatorRegistry;
+/**
+ * Handles an `AGG_METRICS_REQ` from a worker: builds the cluster aggregate and
+ * replies to that worker only. prom-client fans the request out to *all*
+ * connected workers (including the requester), so the requester is one of the N
+ * merged sources and is never double-counted.
+ */
+export declare const handleWorkerMetricsRequest: (worker: Worker, message: AggMetricsReqMessage) => Promise<void>;
+/**
+ * Constructs the worker-side `AggregatorRegistry` (idempotent). Its constructor
+ * installs prom-client's worker `process.on('message')` responder that answers
+ * the master's `getMetricsReq` with this worker's registry JSON.
+ */
+export declare const ensureWorkerAggregatorRegistry: () => AggregatorRegistry;
+/**
+ * Requests the cluster aggregate from the master over IPC and resolves with the
+ * merged exposition string. On timeout or any error it falls back to this
+ * worker's local `register.metrics()` so `/metrics` always answers promptly.
+ */
+export declare const requestAggregatedMetrics: () => Promise<string>;
+/**
+ * Resolves the pending `requestAggregatedMetrics` promise correlated by id when
+ * the master replies. On error it falls back to the local registry.
+ */
+export declare const handleMasterMetricsResponse: (message: AggMetricsResMessage) => void;
+/** Test-only helper to reset module state between test cases. */
+export declare const __resetForTests: () => void;

package/lib/service/metrics/clusterMetricsAggregator.js

@@ -0,0 +1,181 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.__resetForTests = exports.handleMasterMetricsResponse = exports.requestAggregatedMetrics = exports.ensureWorkerAggregatorRegistry = exports.handleWorkerMetricsRequest = exports.initMasterAggregatorRegistry = exports.isPromClientMessage = exports.isAggMetricsResponse = exports.isAggMetricsRequest = exports.AGG_METRICS_RES = exports.AGG_METRICS_REQ = void 0;
+const prom_client_1 = require("prom-client");
+const listeners_1 = require("../worker/listeners");
+/**
+ * Cluster-wide Prometheus metrics aggregation.
+ *
+ * In multi-worker mode each cluster worker keeps its own prom-client default
+ * registry. Serving `/metrics` from a single (round-robin selected) worker
+ * therefore exposes only that worker's local counters, which makes Prometheus
+ * treat the per-scrape braid of independent counters as counter resets and
+ * inflates `rate()`/`increase()` by orders of magnitude.
+ *
+ * This module uses prom-client's `AggregatorRegistry` cluster IPC protocol so
+ * that the worker answering a scrape asks the master for a merged, monotonic
+ * view built from every worker's registry in one pass (no double-counting of
+ * the answering worker).
+ */
+/** Tagged IPC message: worker -> master, "please build the cluster aggregate". */
+exports.AGG_METRICS_REQ = 'vtex-api:aggMetricsReq';
+/** Tagged IPC message: master -> worker, carries the aggregate (or an error). */
+exports.AGG_METRICS_RES = 'vtex-api:aggMetricsRes';
+/**
+ * Upper bound for a worker waiting on the master aggregate. prom-client's own
+ * `clusterMetrics()` has a 5s timeout collecting worker registries; we guard
+ * slightly above that and fall back to the local registry on expiry so that
+ * `/metrics` never hangs or returns empty.
+ */
+const AGG_METRICS_TIMEOUT_MS = 6000;
+const isAggMetricsRequest = (message) => (message === null || message === void 0 ? void 0 : message.type) === exports.AGG_METRICS_REQ && typeof (message === null || message === void 0 ? void 0 : message.id) === 'number';
+exports.isAggMetricsRequest = isAggMetricsRequest;
+const isAggMetricsResponse = (message) => (message === null || message === void 0 ? void 0 : message.type) === exports.AGG_METRICS_RES && typeof (message === null || message === void 0 ? void 0 : message.id) === 'number';
+exports.isAggMetricsResponse = isAggMetricsResponse;
+/**
+ * prom-client's cluster protocol emits its own tagged IPC messages
+ * (`prom-client:getMetricsReq` / `prom-client:getMetricsRes`). They are handled
+ * by prom-client's own cluster listeners, so our `onMessage` handlers must
+ * ignore them instead of warning about "unknown" messages.
+ */
+const isPromClientMessage = (message) => typeof (message === null || message === void 0 ? void 0 : message.type) === 'string' && message.type.startsWith('prom-client:');
+exports.isPromClientMessage = isPromClientMessage;
+// ---------------------------------------------------------------------------
+// Master side
+// ---------------------------------------------------------------------------
+let masterAggregatorRegistry;
+/**
+ * Constructs the master-side `AggregatorRegistry` (idempotent). Its constructor
+ * installs prom-client's master `cluster.on('message')` collector, which is
+ * what makes `clusterMetrics()` work. Must be called in the master before any
+ * worker can answer a scrape.
+ */
+const initMasterAggregatorRegistry = () => {
+    if (!masterAggregatorRegistry) {
+        masterAggregatorRegistry = new prom_client_1.AggregatorRegistry();
+    }
+    return masterAggregatorRegistry;
+};
+exports.initMasterAggregatorRegistry = initMasterAggregatorRegistry;
+/**
+ * Handles an `AGG_METRICS_REQ` from a worker: builds the cluster aggregate and
+ * replies to that worker only. prom-client fans the request out to *all*
+ * connected workers (including the requester), so the requester is one of the N
+ * merged sources and is never double-counted.
+ */
+const handleWorkerMetricsRequest = async (worker, message) => {
+    var _a, _b;
+    const aggregator = (0, exports.initMasterAggregatorRegistry)();
+    let response;
+    try {
+        const body = await aggregator.clusterMetrics();
+        response = { type: exports.AGG_METRICS_RES, id: message.id, body };
+    }
+    catch (err) {
+        response = { type: exports.AGG_METRICS_RES, id: message.id, error: (_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : String(err) };
+    }
+    // `worker.send` can throw (e.g. the worker disconnected between the request
+    // and the reply). This handler is invoked without `await` from the master's
+    // message listener, so a thrown send would surface as an unhandled rejection
+    // in the master process. Swallow and log it instead.
+    try {
+        worker.send(response);
+    }
+    catch (err) {
+        listeners_1.logger.warn({
+            content: (_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : String(err),
+            message: 'Failed to send aggregated cluster metrics to worker',
+            pid: process.pid,
+            workerId: worker.id,
+        });
+    }
+};
+exports.handleWorkerMetricsRequest = handleWorkerMetricsRequest;
+// ---------------------------------------------------------------------------
+// Worker side
+// ---------------------------------------------------------------------------
+let workerAggregatorRegistry;
+const pendingRequests = new Map();
+let requestCounter = 0;
+/**
+ * Constructs the worker-side `AggregatorRegistry` (idempotent). Its constructor
+ * installs prom-client's worker `process.on('message')` responder that answers
+ * the master's `getMetricsReq` with this worker's registry JSON.
+ */
+const ensureWorkerAggregatorRegistry = () => {
+    if (!workerAggregatorRegistry) {
+        workerAggregatorRegistry = new prom_client_1.AggregatorRegistry();
+    }
+    return workerAggregatorRegistry;
+};
+exports.ensureWorkerAggregatorRegistry = ensureWorkerAggregatorRegistry;
+/**
+ * Requests the cluster aggregate from the master over IPC and resolves with the
+ * merged exposition string. On timeout or any error it falls back to this
+ * worker's local `register.metrics()` so `/metrics` always answers promptly.
+ */
+const requestAggregatedMetrics = async () => {
+    if (typeof process.send !== 'function') {
+        return prom_client_1.register.metrics();
+    }
+    const id = requestCounter++;
+    return new Promise((resolve) => {
+        var _a;
+        const timer = setTimeout(() => {
+            pendingRequests.delete(id);
+            listeners_1.logger.warn({
+                message: 'Timed out waiting for aggregated cluster metrics; falling back to local registry',
+                pid: process.pid,
+            });
+            prom_client_1.register.metrics().then(resolve).catch(() => resolve(''));
+        }, AGG_METRICS_TIMEOUT_MS);
+        pendingRequests.set(id, { resolve, timer });
+        try {
+            process.send({ type: exports.AGG_METRICS_REQ, id });
+        }
+        catch (err) {
+            clearTimeout(timer);
+            pendingRequests.delete(id);
+            listeners_1.logger.warn({
+                content: (_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : String(err),
+                message: 'Failed to request aggregated cluster metrics; falling back to local registry',
+                pid: process.pid,
+            });
+            prom_client_1.register.metrics().then(resolve).catch(() => resolve(''));
+        }
+    });
+};
+exports.requestAggregatedMetrics = requestAggregatedMetrics;
+/**
+ * Resolves the pending `requestAggregatedMetrics` promise correlated by id when
+ * the master replies. On error it falls back to the local registry.
+ */
+const handleMasterMetricsResponse = (message) => {
+    const pending = pendingRequests.get(message.id);
+    if (!pending) {
+        return;
+    }
+    pendingRequests.delete(message.id);
+    clearTimeout(pending.timer);
+    if (message.error != null || message.body == null) {
+        listeners_1.logger.warn({
+            content: message.error,
+            message: 'Master failed to aggregate cluster metrics; falling back to local registry',
+            pid: process.pid,
+        });
+        prom_client_1.register.metrics().then(pending.resolve).catch(() => pending.resolve(''));
+        return;
+    }
+    pending.resolve(message.body);
+};
+exports.handleMasterMetricsResponse = handleMasterMetricsResponse;
+/** Test-only helper to reset module state between test cases. */
+// tslint:disable-next-line:variable-name
+const __resetForTests = () => {
+    masterAggregatorRegistry = undefined;
+    workerAggregatorRegistry = undefined;
+    pendingRequests.forEach((p) => clearTimeout(p.timer));
+    pendingRequests.clear();
+    requestCounter = 0;
+};
+exports.__resetForTests = __resetForTests;

package/lib/service/metrics/requestHandlerLabel.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Label value reported for requests that never reached a named handler: unmatched
+ * paths (Koa answers its default 404), requests rejected before the route pipeline
+ * (replica-level rate limiter, errors in compress/recorder), and handlers that
+ * don't set `ctx.requestHandlerName`.
+ *
+ * The value must be a non-empty string. `ctx.requestHandlerName` is `undefined`
+ * for those requests, and Node's cluster IPC serializes each worker's registry as
+ * JSON, which drops properties whose value is `undefined`. The sample then reaches
+ * the aggregated `/metrics` with the `handler` label missing altogether, which
+ * Prometheus reads as `handler=""` — a distinct, unnamed series that shows up as
+ * an extra "Value" line in dashboards.
+ *
+ * `'undefined'` is deliberate rather than a nicer word: prom-client's local
+ * exposition already rendered `handler: undefined` as `handler="undefined"` before
+ * the cluster aggregation was introduced, so keeping that value makes the
+ * aggregated output match the historical series identity and keeps existing
+ * dashboards, filters and alerts (e.g. `handler!~"builtin:.*|undefined"`) working.
+ */
+export declare const UNNAMED_REQUEST_HANDLER = "undefined";
+/**
+ * Resolves the `handler` label value for a request, falling back to
+ * {@link UNNAMED_REQUEST_HANDLER} when the pipeline never named the handler.
+ * Empty strings fall back too, so the label is never emitted empty.
+ */
+export declare const requestHandlerLabel: (requestHandlerName?: string) => string;

package/lib/service/metrics/requestHandlerLabel.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.requestHandlerLabel = exports.UNNAMED_REQUEST_HANDLER = void 0;
+/**
+ * Label value reported for requests that never reached a named handler: unmatched
+ * paths (Koa answers its default 404), requests rejected before the route pipeline
+ * (replica-level rate limiter, errors in compress/recorder), and handlers that
+ * don't set `ctx.requestHandlerName`.
+ *
+ * The value must be a non-empty string. `ctx.requestHandlerName` is `undefined`
+ * for those requests, and Node's cluster IPC serializes each worker's registry as
+ * JSON, which drops properties whose value is `undefined`. The sample then reaches
+ * the aggregated `/metrics` with the `handler` label missing altogether, which
+ * Prometheus reads as `handler=""` — a distinct, unnamed series that shows up as
+ * an extra "Value" line in dashboards.
+ *
+ * `'undefined'` is deliberate rather than a nicer word: prom-client's local
+ * exposition already rendered `handler: undefined` as `handler="undefined"` before
+ * the cluster aggregation was introduced, so keeping that value makes the
+ * aggregated output match the historical series identity and keeps existing
+ * dashboards, filters and alerts (e.g. `handler!~"builtin:.*|undefined"`) working.
+ */
+exports.UNNAMED_REQUEST_HANDLER = 'undefined';
+/**
+ * Resolves the `handler` label value for a request, falling back to
+ * {@link UNNAMED_REQUEST_HANDLER} when the pipeline never named the handler.
+ * Empty strings fall back too, so the label is never emitted empty.
+ */
+const requestHandlerLabel = (requestHandlerName) => requestHandlerName || exports.UNNAMED_REQUEST_HANDLER;
+exports.requestHandlerLabel = requestHandlerLabel;

package/lib/service/metrics/requestMetricsMiddleware.js

@@ -4,6 +4,7 @@ exports.addRequestMetricsMiddleware = void 0;
 const stream_1 = require("stream");
 const utils_1 = require("../../utils");
 const instruments_1 = require("../tracing/metrics/instruments");
+const requestHandlerLabel_1 = require("./requestHandlerLabel");
 const addRequestMetricsMiddleware = () => {
     const concurrentRequests = (0, instruments_1.createConcurrentRequestsInstrument)();
     const requestTimings = (0, instruments_1.createRequestsTimingsInstrument)();
@@ -13,7 +14,7 @@ const addRequestMetricsMiddleware = () => {
     return async function addRequestMetrics(ctx, next) {
         const start = process.hrtime();
         concurrentRequests.inc(1);
-        ctx.req.once('aborted', () => abortedRequests.inc({ ["handler" /* RequestsMetricLabels.REQUEST_HANDLER */]: ctx.requestHandlerName }, 1));
+        ctx.req.once('aborted', () => abortedRequests.inc({ ["handler" /* RequestsMetricLabels.REQUEST_HANDLER */]: (0, requestHandlerLabel_1.requestHandlerLabel)(ctx.requestHandlerName) }, 1));
         let responseClosed = false;
         ctx.res.once('close', () => (responseClosed = true));
         try {
@@ -22,15 +23,15 @@ const addRequestMetricsMiddleware = () => {
         finally {
             const responseLength = ctx.response.length;
             if (responseLength) {
-                responseSizes.observe({ ["handler" /* RequestsMetricLabels.REQUEST_HANDLER */]: ctx.requestHandlerName }, responseLength);
+                responseSizes.observe({ ["handler" /* RequestsMetricLabels.REQUEST_HANDLER */]: (0, requestHandlerLabel_1.requestHandlerLabel)(ctx.requestHandlerName) }, responseLength);
             }
             totalRequests.inc({
-                ["handler" /* RequestsMetricLabels.REQUEST_HANDLER */]: ctx.requestHandlerName,
+                ["handler" /* RequestsMetricLabels.REQUEST_HANDLER */]: (0, requestHandlerLabel_1.requestHandlerLabel)(ctx.requestHandlerName),
                 ["status_code" /* RequestsMetricLabels.STATUS_CODE */]: ctx.response.status,
             }, 1);
             const onResFinished = () => {
                 requestTimings.observe({
-                    ["handler" /* RequestsMetricLabels.REQUEST_HANDLER */]: ctx.requestHandlerName,
+                    ["handler" /* RequestsMetricLabels.REQUEST_HANDLER */]: (0, requestHandlerLabel_1.requestHandlerLabel)(ctx.requestHandlerName),
                 }, (0, utils_1.hrToMillisFloat)(process.hrtime(start)));
                 concurrentRequests.dec(1);
             };

package/lib/service/worker/index.d.ts

@@ -1,6 +1,7 @@
 import Koa from 'koa';
 import { MetricsAccumulator } from '../../metrics/MetricsAccumulator';
 import { ServiceJSON } from './runtime/typings';
+export declare const onMessage: (service: ServiceJSON) => (message: any) => void;
 export declare const startWorker: (serviceJSON: ServiceJSON) => Koa<Koa.DefaultState, Koa.DefaultContext>;
 declare global {
     namespace NodeJS {

package/lib/service/worker/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.startWorker = void 0;
+exports.startWorker = exports.onMessage = void 0;
 const http_1 = require("http");
 const koa_1 = __importDefault(require("koa"));
 const koa_compress_1 = __importDefault(require("koa-compress"));
@@ -14,6 +14,7 @@ const MetricsAccumulator_1 = require("../../metrics/MetricsAccumulator");
 const loaders_1 = require("../loaders");
 const console_1 = require("../logger/console");
 const loggerTypes_1 = require("../logger/loggerTypes");
+const clusterMetricsAggregator_1 = require("../metrics/clusterMetricsAggregator");
 const requestMetricsMiddleware_1 = require("../metrics/requestMetricsMiddleware");
 const TracerSingleton_1 = require("../tracing/TracerSingleton");
 const tracingMiddlewares_1 = require("../tracing/tracingMiddlewares");
@@ -54,7 +55,12 @@ const onMessage = (service) => (message) => {
     else if ((0, statusTrack_1.isStatusTrack)(message)) {
         (0, statusTrack_1.trackStatus)();
     }
-    else {
+    else if ((0, clusterMetricsAggregator_1.isAggMetricsResponse)(message)) {
+        (0, clusterMetricsAggregator_1.handleMasterMetricsResponse)(message);
+    }
+    else if (!(0, clusterMetricsAggregator_1.isPromClientMessage)(message)) {
+        // prom-client's own cluster messages are handled by its worker listener;
+        // anything else that reaches here is genuinely unexpected.
         listeners_1.logger.warn({
             content: message,
             message: 'Master sent message',
@@ -62,6 +68,7 @@ const onMessage = (service) => (message) => {
         });
     }
 };
+exports.onMessage = onMessage;
 const createAppHttpHandlers = ({ config: { routes, clients } }, serviceJSON, globalLimiter) => {
     if (routes && clients) {
         return Object.keys(routes).reduce((acc, routeId) => {
@@ -153,11 +160,16 @@ const startWorker = (serviceJSON) => {
     var _a, _b;
     (0, listeners_1.addProcessListeners)();
     const tracer = TracerSingleton_1.TracerSingleton.getTracer();
+    // In multi-worker mode install prom-client's worker-side cluster responder so
+    // the master can collect this worker's registry for the aggregated /metrics.
+    if (serviceJSON.workers > 1) {
+        (0, clusterMetricsAggregator_1.ensureWorkerAggregatorRegistry)();
+    }
     const app = new koa_1.default();
     app.proxy = true;
     app
         .use(error_1.error)
-        .use((0, middlewares_1.prometheusLoggerMiddleware)())
+        .use((0, middlewares_1.prometheusLoggerMiddleware)(serviceJSON.workers))
         .use((0, tracingMiddlewares_1.addTracingMiddleware)(tracer))
         .use((0, requestMetricsMiddleware_1.addRequestMetricsMiddleware)())
         .use((0, middlewares_1.addMetricsLoggerMiddleware)())
@@ -190,7 +202,7 @@ const startWorker = (serviceJSON) => {
         const privateHandlersRouter = routerFromPrivateHttpHandlers(httpHandlers.pvt);
         app.use(privateHandlersRouter.routes());
     }
-    process.on('message', onMessage(serviceJSON));
+    process.on('message', (0, exports.onMessage)(serviceJSON));
     return app;
 };
 exports.startWorker = startWorker;

package/lib/service/worker/runtime/builtIn/middlewares.d.ts

@@ -1,4 +1,4 @@
 import { ServiceContext } from '../typings';
 export declare function recorderMiddleware(ctx: ServiceContext, next: () => Promise<void>): Promise<void>;
 export declare const addMetricsLoggerMiddleware: () => (ctx: ServiceContext, next: () => Promise<void>) => Promise<void>;
-export declare const prometheusLoggerMiddleware: () => (ctx: ServiceContext, next: () => Promise<void>) => Promise<void>;
+export declare const prometheusLoggerMiddleware: (workers?: number) => (ctx: ServiceContext, next: () => Promise<void>) => Promise<void>;

package/lib/service/worker/runtime/builtIn/middlewares.js

@@ -4,6 +4,7 @@ exports.prometheusLoggerMiddleware = exports.addMetricsLoggerMiddleware = export
 const prom_client_1 = require("prom-client");
 const constants_1 = require("../../../../constants");
 const metricsLogger_1 = require("../../../logger/metricsLogger");
+const clusterMetricsAggregator_1 = require("../../../metrics/clusterMetricsAggregator");
 const EventLoopLagMeasurer_1 = require("../../../tracing/metrics/measurers/EventLoopLagMeasurer");
 const recorder_1 = require("../utils/recorder");
 async function recorderMiddleware(ctx, next) {
@@ -22,10 +23,14 @@ const addMetricsLoggerMiddleware = () => {
     };
 };
 exports.addMetricsLoggerMiddleware = addMetricsLoggerMiddleware;
-const prometheusLoggerMiddleware = () => {
+const prometheusLoggerMiddleware = (workers = 1) => {
     (0, prom_client_1.collectDefaultMetrics)();
     const eventLoopLagMeasurer = new EventLoopLagMeasurer_1.EventLoopLagMeasurer();
     eventLoopLagMeasurer.start();
+    // In multi-worker mode each worker holds only its own registry, so /metrics
+    // must serve the cluster-wide aggregate requested from the master over IPC.
+    // In single-worker mode (LINKED / workers:1) the local registry is complete.
+    const isMultiWorker = workers > 1;
     return async (ctx, next) => {
         if (ctx.request.path !== '/metrics') {
             return next();
@@ -36,7 +41,7 @@ const prometheusLoggerMiddleware = () => {
         }
         await eventLoopLagMeasurer.updateInstrumentsAndReset();
         ctx.set('Content-Type', prom_client_1.register.contentType);
-        ctx.body = await prom_client_1.register.metrics();
+        ctx.body = isMultiWorker ? await (0, clusterMetricsAggregator_1.requestAggregatedMetrics)() : await prom_client_1.register.metrics();
         ctx.status = 200;
     };
 };

package/lib/service/worker/runtime/statusTrack.js

@@ -14,7 +14,10 @@ const isStatusTrackBroadcast = (message) => message === BROADCAST_STATUS_TRACK;
 exports.isStatusTrackBroadcast = isStatusTrackBroadcast;
 const statusTrackHandler = async (ctx) => {
     var _a, _b, _c;
-    (_b = (_a = ctx.tracing) === null || _a === void 0 ? void 0 : _a.currentSpan) === null || _b === void 0 ? void 0 : _b.setOperationName('builtin:status-track');
+    // Parity with the other builtin handlers: name the request so its samples don't
+    // land in the catch-all `handler="undefined"` bucket.
+    ctx.requestHandlerName = 'builtin:status-track';
+    (_b = (_a = ctx.tracing) === null || _a === void 0 ? void 0 : _a.currentSpan) === null || _b === void 0 ? void 0 : _b.setOperationName(ctx.requestHandlerName);
     if (!constants_1.LINKED) {
         (_c = process.send) === null || _c === void 0 ? void 0 : _c.call(process, BROADCAST_STATUS_TRACK);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vtex/api",
-  "version": "6.51.1-beta",
+  "version": "6.52.0",
   "description": "VTEX I/O API client",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",