@kb-labs/gateway-app 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/.kb/database/kb.sqlite-shm +0 -0
  2. package/.kb/database/kb.sqlite-wal +0 -0
  3. package/package.json +49 -0
  4. package/src/__tests__/auth-routes.test.ts +279 -0
  5. package/src/__tests__/execute-routes.test.ts +408 -0
  6. package/src/__tests__/execution-registry.test.ts +218 -0
  7. package/src/__tests__/health.test.ts +215 -0
  8. package/src/__tests__/live-gateway.e2e.test.ts +648 -0
  9. package/src/__tests__/llm-gateway.test.ts +361 -0
  10. package/src/__tests__/observability-collector.test.ts +59 -0
  11. package/src/__tests__/platform-api.test.ts +317 -0
  12. package/src/__tests__/registry.test.ts +546 -0
  13. package/src/__tests__/retry-executor.test.ts +244 -0
  14. package/src/__tests__/server.integration.test.ts +417 -0
  15. package/src/__tests__/subscription-registry.test.ts +308 -0
  16. package/src/__tests__/telemetry-ingest.test.ts +309 -0
  17. package/src/__tests__/tokens.test.ts +83 -0
  18. package/src/__tests__/ws-client-connect.e2e.test.ts +381 -0
  19. package/src/__tests__/ws-handshake.e2e.test.ts +288 -0
  20. package/src/auth/middleware.ts +50 -0
  21. package/src/auth/routes.ts +57 -0
  22. package/src/auth/tokens.ts +41 -0
  23. package/src/bootstrap.ts +98 -0
  24. package/src/clients/subscription-registry.ts +137 -0
  25. package/src/clients/ws-handler.ts +196 -0
  26. package/src/config.ts +20 -0
  27. package/src/docs/routes.ts +70 -0
  28. package/src/execute/errors.ts +21 -0
  29. package/src/execute/execution-registry.ts +84 -0
  30. package/src/execute/retry-executor.ts +159 -0
  31. package/src/execute/routes.ts +239 -0
  32. package/src/hosts/dispatcher.ts +2 -0
  33. package/src/hosts/registry.ts +305 -0
  34. package/src/hosts/ws-handler.ts +445 -0
  35. package/src/index.ts +7 -0
  36. package/src/llm/routes.ts +343 -0
  37. package/src/manifest.ts +21 -0
  38. package/src/observability/collector.ts +346 -0
  39. package/src/platform/routes.ts +195 -0
  40. package/src/server.ts +447 -0
  41. package/src/telemetry/routes.ts +89 -0
  42. package/src/ws/gateway-ws.ts +73 -0
  43. package/tsconfig.build.json +15 -0
  44. package/tsconfig.json +10 -0
  45. package/tsup.config.ts +8 -0
  46. package/vitest.config.ts +23 -0
@@ -0,0 +1,343 @@
1
+ /**
2
+ * @module AI Gateway — OpenAI-compatible LLM endpoint.
3
+ *
4
+ * Exposes `POST /api/v1/llm/chat/completions` using OpenAI ChatCompletion format.
5
+ * Model field is a tier abstraction (small/medium/large) — never a literal model name.
6
+ * All calls go through the platform LLM chain: QueuedLLM → AnalyticsLLM → LLMRouter → adapter.
7
+ */
8
+ import type { FastifyInstance } from 'fastify';
9
+ import { platform } from '@kb-labs/core-runtime';
10
+ import type {
11
+ ILLM,
12
+ ILogger,
13
+ LLMMessage,
14
+ LLMToolCallOptions,
15
+ LLMTool,
16
+ ILLMRouter,
17
+ LLMTier,
18
+ } from '@kb-labs/core-platform';
19
+ import {
20
+ ChatCompletionRequestSchema,
21
+ type ChatCompletionRequest,
22
+ type ChatCompletionResponse,
23
+ type ChatCompletionChunk,
24
+ } from '@kb-labs/gateway-contracts';
25
+
26
+ // ── Tier resolution ───────────────────────────────────────────────────────
27
+
28
+ function isLLMRouter(llm: ILLM): llm is ILLM & ILLMRouter {
29
+ return typeof (llm as any).resolveAdapter === 'function';
30
+ }
31
+
32
+ /**
33
+ * Resolve tier to a concrete ILLM adapter via LLMRouter.
34
+ * If no router (single-adapter setup), returns platform.llm directly.
35
+ */
36
+ async function resolveLLMForTier(tier: LLMTier): Promise<ILLM | undefined> {
37
+ const llm = platform.llm;
38
+ if (!llm) {return undefined;}
39
+
40
+ if (isLLMRouter(llm)) {
41
+ const binding = await llm.resolveAdapter({ tier });
42
+ return binding.adapter;
43
+ }
44
+
45
+ // Single adapter — ignore tier, return as-is
46
+ return llm;
47
+ }
48
+
49
+ // ── Route registration ────────────────────────────────────────────────────
50
+
51
+ /**
52
+ * Register AI Gateway routes on the given Fastify scope.
53
+ * The scope is expected to have the auth middleware already applied.
54
+ */
55
+ export function registerLLMGatewayRoutes(app: FastifyInstance, logger: ILogger): void {
56
+ // hide: true — can stream SSE (text/event-stream), incompatible with OpenAPI response schema
57
+ app.post('/llm/v1/chat/completions', { schema: { tags: ['LLM'], summary: 'OpenAI-compatible chat completions', hide: true } }, async (request, reply) => {
58
+ const auth = request.authContext;
59
+ if (!auth) {
60
+ return reply.code(401).send({ error: 'Unauthorized' });
61
+ }
62
+
63
+ const parsed = ChatCompletionRequestSchema.safeParse(request.body);
64
+ if (!parsed.success) {
65
+ return reply.code(400).send({
66
+ error: {
67
+ message: 'Bad Request',
68
+ type: 'invalid_request_error',
69
+ code: null,
70
+ param: null,
71
+ },
72
+ issues: parsed.error.issues,
73
+ });
74
+ }
75
+
76
+ const req = parsed.data;
77
+ const requestId = `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
78
+
79
+ // Resolve tier to LLM adapter
80
+ const llm = await resolveLLMForTier(req.model);
81
+ if (!llm) {
82
+ return reply.code(503).send({
83
+ error: {
84
+ message: `LLM not available for tier "${req.model}"`,
85
+ type: 'server_error',
86
+ code: null,
87
+ param: null,
88
+ },
89
+ });
90
+ }
91
+
92
+ logger.info('AI Gateway request', {
93
+ requestId,
94
+ tier: req.model,
95
+ stream: req.stream,
96
+ messageCount: req.messages.length,
97
+ hasTools: !!req.tools?.length,
98
+ tenantId: auth.namespaceId,
99
+ });
100
+
101
+ try {
102
+ if (req.stream) {
103
+ return await handleStreamingRequest(reply, llm, req, requestId, logger);
104
+ }
105
+ return await handleCompletionRequest(reply, llm, req, requestId);
106
+ } catch (err) {
107
+ const error = err instanceof Error ? err : new Error(String(err));
108
+ logger.error('AI Gateway error', error, { requestId, tier: req.model });
109
+ return reply.code(500).send({
110
+ error: {
111
+ message: 'Internal server error',
112
+ type: 'server_error',
113
+ code: null,
114
+ param: null,
115
+ },
116
+ });
117
+ }
118
+ });
119
+ }
120
+
121
+ // ── Non-streaming handler ─────────────────────────────────────────────────
122
+
123
+ async function handleCompletionRequest(
124
+ reply: any,
125
+ llm: ILLM,
126
+ req: ChatCompletionRequest,
127
+ requestId: string,
128
+ ) {
129
+ const messages = toILLMMessages(req);
130
+ const startTime = Date.now();
131
+ const hasTools = req.tools && req.tools.length > 0;
132
+
133
+ if (hasTools && llm.chatWithTools) {
134
+ const tools = toILLMTools(req.tools!);
135
+ const options: LLMToolCallOptions = {
136
+ temperature: req.temperature,
137
+ maxTokens: req.max_tokens,
138
+ stop: normalizeStop(req.stop),
139
+ tools,
140
+ toolChoice: req.tool_choice as LLMToolCallOptions['toolChoice'],
141
+ };
142
+
143
+ const result = await llm.chatWithTools(messages, options);
144
+
145
+ const toolCalls = result.toolCalls?.map((tc) => ({
146
+ id: tc.id,
147
+ type: 'function' as const,
148
+ function: {
149
+ name: tc.name,
150
+ arguments: typeof tc.input === 'string' ? tc.input : JSON.stringify(tc.input),
151
+ },
152
+ }));
153
+
154
+ const finishReason =
155
+ result.stopReason === 'tool_use'
156
+ ? ('tool_calls' as const)
157
+ : result.stopReason === 'max_tokens'
158
+ ? ('length' as const)
159
+ : ('stop' as const);
160
+
161
+ const response: ChatCompletionResponse = {
162
+ id: requestId,
163
+ object: 'chat.completion',
164
+ created: Math.floor(startTime / 1000),
165
+ model: req.model,
166
+ choices: [
167
+ {
168
+ index: 0,
169
+ message: {
170
+ role: 'assistant',
171
+ content: result.content || null,
172
+ ...(toolCalls?.length ? { tool_calls: toolCalls } : {}),
173
+ },
174
+ finish_reason: finishReason,
175
+ },
176
+ ],
177
+ usage: {
178
+ prompt_tokens: result.usage.promptTokens,
179
+ completion_tokens: result.usage.completionTokens,
180
+ total_tokens: result.usage.promptTokens + result.usage.completionTokens,
181
+ },
182
+ };
183
+
184
+ return reply.code(200).send(response);
185
+ }
186
+
187
+ // No tools — use simple complete()
188
+ const systemPrompt = messages.find((m) => m.role === 'system')?.content;
189
+ const userMessages = messages.filter((m) => m.role !== 'system');
190
+ const prompt = userMessages.map((m) => m.content).join('\n\n');
191
+
192
+ const result = await llm.complete(prompt, {
193
+ systemPrompt,
194
+ temperature: req.temperature,
195
+ maxTokens: req.max_tokens,
196
+ stop: normalizeStop(req.stop),
197
+ });
198
+
199
+ const response: ChatCompletionResponse = {
200
+ id: requestId,
201
+ object: 'chat.completion',
202
+ created: Math.floor(startTime / 1000),
203
+ model: req.model,
204
+ choices: [
205
+ {
206
+ index: 0,
207
+ message: { role: 'assistant', content: result.content },
208
+ finish_reason: 'stop',
209
+ },
210
+ ],
211
+ usage: {
212
+ prompt_tokens: result.usage.promptTokens,
213
+ completion_tokens: result.usage.completionTokens,
214
+ total_tokens: result.usage.promptTokens + result.usage.completionTokens,
215
+ },
216
+ };
217
+
218
+ return reply.code(200).send(response);
219
+ }
220
+
221
+ // ── Streaming handler ─────────────────────────────────────────────────────
222
+
223
+ async function handleStreamingRequest(
224
+ reply: any,
225
+ llm: ILLM,
226
+ req: ChatCompletionRequest,
227
+ requestId: string,
228
+ logger: ILogger,
229
+ ) {
230
+ const created = Math.floor(Date.now() / 1000);
231
+
232
+ reply.raw.writeHead(200, {
233
+ 'Content-Type': 'text/event-stream',
234
+ 'Cache-Control': 'no-cache',
235
+ 'Connection': 'keep-alive',
236
+ 'X-Request-Id': requestId,
237
+ });
238
+ reply.raw.flushHeaders();
239
+
240
+ const writeChunk = (chunk: ChatCompletionChunk): void => {
241
+ if (!reply.raw.writableEnded) {
242
+ reply.raw.write(`data: ${JSON.stringify(chunk)}\n\n`);
243
+ }
244
+ };
245
+
246
+ const writeDone = (): void => {
247
+ if (!reply.raw.writableEnded) {
248
+ reply.raw.write('data: [DONE]\n\n');
249
+ reply.raw.end();
250
+ }
251
+ };
252
+
253
+ try {
254
+ // Initial role chunk
255
+ writeChunk({
256
+ id: requestId,
257
+ object: 'chat.completion.chunk',
258
+ created,
259
+ model: req.model,
260
+ choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }],
261
+ });
262
+
263
+ // Stream text from LLM
264
+ const systemPrompt = req.messages.find((m) => m.role === 'system')?.content;
265
+ const userMessages = req.messages.filter((m) => m.role !== 'system');
266
+ const prompt = userMessages.map((m) => m.content).join('\n\n');
267
+
268
+ for await (const text of llm.stream(prompt, {
269
+ systemPrompt,
270
+ temperature: req.temperature,
271
+ maxTokens: req.max_tokens,
272
+ stop: normalizeStop(req.stop),
273
+ })) {
274
+ writeChunk({
275
+ id: requestId,
276
+ object: 'chat.completion.chunk',
277
+ created,
278
+ model: req.model,
279
+ choices: [{ index: 0, delta: { content: text }, finish_reason: null }],
280
+ });
281
+ }
282
+
283
+ // Finish chunk
284
+ writeChunk({
285
+ id: requestId,
286
+ object: 'chat.completion.chunk',
287
+ created,
288
+ model: req.model,
289
+ choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
290
+ });
291
+
292
+ writeDone();
293
+ } catch (err) {
294
+ const error = err instanceof Error ? err : new Error(String(err));
295
+ logger.error('AI Gateway stream error', error, { requestId });
296
+ if (!reply.raw.writableEnded) {
297
+ reply.raw.write(
298
+ `data: ${JSON.stringify({ error: { message: error.message, type: 'server_error' } })}\n\n`,
299
+ );
300
+ reply.raw.end();
301
+ }
302
+ }
303
+
304
+ return reply;
305
+ }
306
+
307
+ // ── Helpers ───────────────────────────────────────────────────────────────
308
+
309
+ function toILLMMessages(req: ChatCompletionRequest): LLMMessage[] {
310
+ return req.messages.map((m) => {
311
+ const msg: LLMMessage = { role: m.role, content: m.content };
312
+ if (m.tool_call_id) {msg.toolCallId = m.tool_call_id;}
313
+ if (m.tool_calls) {
314
+ msg.toolCalls = m.tool_calls.map((tc) => ({
315
+ id: tc.id,
316
+ name: tc.function.name,
317
+ input: safeJsonParse(tc.function.arguments),
318
+ }));
319
+ }
320
+ return msg;
321
+ });
322
+ }
323
+
324
+ function toILLMTools(tools: NonNullable<ChatCompletionRequest['tools']>): LLMTool[] {
325
+ return tools.map((t) => ({
326
+ name: t.function.name,
327
+ description: t.function.description ?? '',
328
+ inputSchema: t.function.parameters ?? {},
329
+ }));
330
+ }
331
+
332
+ function normalizeStop(stop: string | string[] | undefined): string[] | undefined {
333
+ if (!stop) {return undefined;}
334
+ return Array.isArray(stop) ? stop : [stop];
335
+ }
336
+
337
+ function safeJsonParse(str: string): unknown {
338
+ try {
339
+ return JSON.parse(str);
340
+ } catch {
341
+ return str;
342
+ }
343
+ }
@@ -0,0 +1,21 @@
1
+ import type { ServiceManifest } from '@kb-labs/plugin-contracts';
2
+
3
+ export const manifest: ServiceManifest = {
4
+ schema: 'kb.service/1',
5
+ id: 'gateway',
6
+ name: 'Gateway',
7
+ version: '1.0.0',
8
+ description: 'Central router — aggregates REST API, Workflow, Marketplace',
9
+ runtime: {
10
+ entry: 'dist/index.js',
11
+ port: 4000,
12
+ healthCheck: '/health',
13
+ },
14
+ dependsOn: ['rest', 'workflow'],
15
+ env: {
16
+ PORT: { description: 'HTTP port', default: '4000' },
17
+ NODE_ENV: { description: 'Environment mode', default: 'development' },
18
+ },
19
+ };
20
+
21
+ export default manifest;
@@ -0,0 +1,346 @@
1
+ import type { FastifyInstance, FastifyReply, FastifyRequest, HookHandlerDoneFunction } from 'fastify';
2
+ import type { GatewayConfig } from '@kb-labs/gateway-contracts';
3
+ import {
4
+ OBSERVABILITY_CONTRACT_VERSION,
5
+ OBSERVABILITY_SCHEMA,
6
+ CANONICAL_OBSERVABILITY_METRICS,
7
+ type ServiceHealthStatus,
8
+ type ObservabilityCheck,
9
+ } from '@kb-labs/core-contracts';
10
+ import type { ServiceObservabilityDescribe, ServiceObservabilityHealth } from '@kb-labs/core-contracts';
11
+ import {
12
+ createServiceObservabilityDescribe,
13
+ createServiceObservabilityHealth,
14
+ OperationMetricsTracker,
15
+ } from '@kb-labs/shared-http';
16
+ import { hostname } from 'node:os';
17
+ import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
18
+
19
+ declare module 'fastify' {
20
+ interface FastifyRequest {
21
+ kbMetricsStart?: number;
22
+ }
23
+ }
24
+
25
+ type RouteStats = {
26
+ count: number;
27
+ totalDurationMs: number;
28
+ maxDurationMs: number;
29
+ errorCount: number;
30
+ };
31
+
32
+ function normalizeRoute(route: string | undefined): string {
33
+ if (!route) {
34
+ return 'unknown';
35
+ }
36
+ return route
37
+ .split('?')[0]!
38
+ .replace(/\/[0-9a-fA-F-]{6,}/g, '/:id');
39
+ }
40
+
41
+ function metricLine(name: string, value: number, labels?: Record<string, string>): string {
42
+ if (!labels || Object.keys(labels).length === 0) {
43
+ return `${name} ${value}`;
44
+ }
45
+ const pairs = Object.entries(labels).map(([key, labelValue]) => `${key}="${labelValue.replace(/"/g, '\\"')}"`);
46
+ return `${name}{${pairs.join(',')}} ${value}`;
47
+ }
48
+
49
+ export class GatewayObservabilityCollector {
50
+ private readonly instanceId = `${hostname()}:${process.pid}`;
51
+ private readonly eventLoop = monitorEventLoopDelay({ resolution: 20 });
52
+ private readonly routeStats = new Map<string, RouteStats>();
53
+ private readonly operationMetrics = new OperationMetricsTracker();
54
+ private readonly startedAt = Date.now();
55
+ private readonly dependencies;
56
+ private lastCpuUsage = process.cpuUsage();
57
+ private lastCpuTime = Date.now();
58
+ private intervalId: NodeJS.Timeout | null = null;
59
+ private activeOperations = 0;
60
+ private requestsTotal = 0;
61
+ private errorsTotal = 0;
62
+ private lastSnapshot = {
63
+ cpuPercent: 0,
64
+ rssBytes: process.memoryUsage().rss,
65
+ heapUsedBytes: process.memoryUsage().heapUsed,
66
+ eventLoopLagMs: 0,
67
+ };
68
+
69
+ constructor(private readonly config: GatewayConfig) {
70
+ this.dependencies = Object.keys(this.config.upstreams).map((serviceId) => ({
71
+ serviceId,
72
+ required: false,
73
+ description: 'Gateway upstream',
74
+ }));
75
+ }
76
+
77
+ register(server: unknown): void {
78
+ const hookServer = server as FastifyInstance;
79
+ this.eventLoop.enable();
80
+ this.intervalId = setInterval(() => this.captureRuntimeSnapshot(), 10_000);
81
+ this.captureRuntimeSnapshot();
82
+
83
+ hookServer.addHook('onRequest', (request: FastifyRequest, _reply: FastifyReply, done: HookHandlerDoneFunction) => {
84
+ request.kbMetricsStart = performance.now();
85
+ this.activeOperations += 1;
86
+ done();
87
+ });
88
+
89
+ hookServer.addHook('onResponse', (request: FastifyRequest, reply: FastifyReply, done: HookHandlerDoneFunction) => {
90
+ const started = request.kbMetricsStart ?? performance.now();
91
+ const durationMs = Math.max(performance.now() - started, 0);
92
+ const route = `${request.method.toUpperCase()} ${normalizeRoute(request.routeOptions?.url ?? request.url)}`;
93
+ const stats = this.routeStats.get(route) ?? {
94
+ count: 0,
95
+ totalDurationMs: 0,
96
+ maxDurationMs: 0,
97
+ errorCount: 0,
98
+ };
99
+
100
+ stats.count += 1;
101
+ stats.totalDurationMs += durationMs;
102
+ stats.maxDurationMs = Math.max(stats.maxDurationMs, durationMs);
103
+ if (reply.statusCode >= 400) {
104
+ stats.errorCount += 1;
105
+ this.errorsTotal += 1;
106
+ }
107
+ this.routeStats.set(route, stats);
108
+ this.requestsTotal += 1;
109
+ this.activeOperations = Math.max(0, this.activeOperations - 1);
110
+ done();
111
+ });
112
+
113
+ hookServer.addHook('onClose', (_instance: unknown, done: HookHandlerDoneFunction) => {
114
+ if (this.intervalId) {
115
+ clearInterval(this.intervalId);
116
+ this.intervalId = null;
117
+ }
118
+ this.eventLoop.disable();
119
+ done();
120
+ });
121
+ }
122
+
123
+ buildDescribe(): ServiceObservabilityDescribe {
124
+ return createServiceObservabilityDescribe({
125
+ schema: OBSERVABILITY_SCHEMA,
126
+ contractVersion: OBSERVABILITY_CONTRACT_VERSION,
127
+ serviceId: 'gateway',
128
+ instanceId: this.instanceId,
129
+ serviceType: 'gateway',
130
+ version: '1.0.0',
131
+ environment: process.env.NODE_ENV ?? 'development',
132
+ startedAt: new Date(this.startedAt).toISOString(),
133
+ dependencies: this.dependencies,
134
+ metricsEndpoint: '/metrics',
135
+ healthEndpoint: '/observability/health',
136
+ logsSource: 'gateway',
137
+ capabilities: ['httpMetrics', 'eventLoopMetrics', 'operationMetrics', 'logCorrelation'],
138
+ metricFamilies: [...CANONICAL_OBSERVABILITY_METRICS],
139
+ });
140
+ }
141
+
142
+ buildHealth(input: {
143
+ status: ServiceHealthStatus;
144
+ adapterChecks: Array<{ id: string; available: boolean; latencyMs?: number }>;
145
+ upstreamChecks: Array<{ id: string; status: string; latencyMs?: number }>;
146
+ }): ServiceObservabilityHealth {
147
+ const checks: ObservabilityCheck[] = [
148
+ ...input.adapterChecks.map((entry): ObservabilityCheck => ({
149
+ id: `adapter:${entry.id}`,
150
+ status: entry.available ? 'ok' : 'warn',
151
+ latencyMs: entry.latencyMs,
152
+ message: entry.available ? 'Adapter available' : 'Adapter unavailable',
153
+ })),
154
+ ...input.upstreamChecks.map((entry): ObservabilityCheck => ({
155
+ id: `upstream:${entry.id}`,
156
+ status: entry.status === 'up' ? 'ok' : 'warn',
157
+ latencyMs: entry.latencyMs,
158
+ message: entry.status === 'up' ? 'Upstream healthy' : 'Upstream unavailable',
159
+ })),
160
+ ];
161
+
162
+ const topOperations = mergeTopOperations(
163
+ Array.from(this.routeStats.entries())
164
+ .sort((a, b) => b[1].count - a[1].count || b[1].maxDurationMs - a[1].maxDurationMs)
165
+ .slice(0, 5)
166
+ .map(([operation, stats]) => ({
167
+ operation: `http.${operation}`,
168
+ count: stats.count,
169
+ avgDurationMs: stats.count > 0 ? stats.totalDurationMs / stats.count : 0,
170
+ maxDurationMs: stats.maxDurationMs,
171
+ errorCount: stats.errorCount,
172
+ })),
173
+ this.operationMetrics.getTopOperations(),
174
+ );
175
+
176
+ return createServiceObservabilityHealth({
177
+ schema: OBSERVABILITY_SCHEMA,
178
+ contractVersion: OBSERVABILITY_CONTRACT_VERSION,
179
+ serviceId: 'gateway',
180
+ instanceId: this.instanceId,
181
+ observedAt: new Date().toISOString(),
182
+ status: input.status,
183
+ uptimeSec: Math.floor((Date.now() - this.startedAt) / 1000),
184
+ metricsEndpoint: '/metrics',
185
+ logsSource: 'gateway',
186
+ capabilities: ['httpMetrics', 'eventLoopMetrics', 'operationMetrics', 'logCorrelation'],
187
+ checks,
188
+ snapshot: {
189
+ cpuPercent: this.lastSnapshot.cpuPercent,
190
+ rssBytes: this.lastSnapshot.rssBytes,
191
+ heapUsedBytes: this.lastSnapshot.heapUsedBytes,
192
+ eventLoopLagMs: this.lastSnapshot.eventLoopLagMs,
193
+ activeOperations: this.activeOperations,
194
+ },
195
+ topOperations,
196
+ state: input.status === 'healthy' ? 'active' : input.status === 'degraded' ? 'partial_observability' : 'insufficient_data',
197
+ meta: {
198
+ requestsTotal: this.requestsTotal,
199
+ errorsTotal: this.errorsTotal,
200
+ },
201
+ });
202
+ }
203
+
204
+ async renderPrometheusMetrics(healthStatus: ServiceHealthStatus): Promise<string> {
205
+ this.captureRuntimeSnapshot();
206
+
207
+ const lines = [
208
+ '# HELP process_cpu_percent Current process CPU usage percentage',
209
+ '# TYPE process_cpu_percent gauge',
210
+ metricLine('process_cpu_percent', this.lastSnapshot.cpuPercent),
211
+ '# HELP process_rss_bytes Current process resident set size in bytes',
212
+ '# TYPE process_rss_bytes gauge',
213
+ metricLine('process_rss_bytes', this.lastSnapshot.rssBytes),
214
+ '# HELP process_heap_used_bytes Current process heap used in bytes',
215
+ '# TYPE process_heap_used_bytes gauge',
216
+ metricLine('process_heap_used_bytes', this.lastSnapshot.heapUsedBytes),
217
+ '# HELP process_event_loop_lag_ms Current event loop lag in milliseconds',
218
+ '# TYPE process_event_loop_lag_ms gauge',
219
+ metricLine('process_event_loop_lag_ms', this.lastSnapshot.eventLoopLagMs),
220
+ '# HELP service_health_status Service health status (2=healthy, 1=degraded, 0=unhealthy)',
221
+ '# TYPE service_health_status gauge',
222
+ metricLine('service_health_status', healthStatus === 'healthy' ? 2 : healthStatus === 'degraded' ? 1 : 0),
223
+ '# HELP service_restarts_total Service restart counter within current process lifetime',
224
+ '# TYPE service_restarts_total gauge',
225
+ metricLine('service_restarts_total', 0),
226
+ '# HELP service_active_operations Current number of active operations',
227
+ '# TYPE service_active_operations gauge',
228
+ metricLine('service_active_operations', this.activeOperations),
229
+ '# HELP http_requests_total Total number of HTTP requests',
230
+ '# TYPE http_requests_total counter',
231
+ '# HELP http_errors_total Total number of HTTP errors (4xx, 5xx)',
232
+ '# TYPE http_errors_total counter',
233
+ '# HELP http_request_duration_ms Total duration of HTTP requests grouped by route',
234
+ '# TYPE http_request_duration_ms summary',
235
+ '# HELP service_operation_total Total number of service operations',
236
+ '# TYPE service_operation_total counter',
237
+ '# HELP service_operation_duration_ms Total duration of service operations grouped by route',
238
+ '# TYPE service_operation_duration_ms summary',
239
+ ];
240
+
241
+ for (const [route, stats] of this.routeStats.entries()) {
242
+ const status = stats.errorCount > 0 ? 'error' : 'ok';
243
+ lines.push(metricLine('http_requests_total', stats.count, { route }));
244
+ lines.push(metricLine('http_errors_total', stats.errorCount, { route }));
245
+ lines.push(metricLine('http_request_duration_ms', Number(stats.totalDurationMs.toFixed(2)), { route }));
246
+ lines.push(metricLine('service_operation_total', stats.count, { operation: `http.${route}`, status }));
247
+ lines.push(metricLine('service_operation_duration_ms', Number(stats.totalDurationMs.toFixed(2)), { operation: `http.${route}`, status }));
248
+ }
249
+
250
+ lines.push(...this.operationMetrics.getMetricLines());
251
+
252
+ return `${lines.join('\n')}\n`;
253
+ }
254
+
255
+ recordOperation(operation: string, durationMs = 0, status: 'ok' | 'error' = 'ok'): void {
256
+ this.operationMetrics.recordOperation(operation, durationMs, status);
257
+ }
258
+
259
+ observeOperation<T>(operation: string, work: () => T | Promise<T>): Promise<T> {
260
+ return this.operationMetrics.observeOperation(operation, work);
261
+ }
262
+
263
+ private captureRuntimeSnapshot(): void {
264
+ const currentUsage = process.cpuUsage(this.lastCpuUsage);
265
+ const currentTime = Date.now();
266
+ const deltaTime = Math.max(currentTime - this.lastCpuTime, 1);
267
+
268
+ this.lastCpuUsage = process.cpuUsage();
269
+ this.lastCpuTime = currentTime;
270
+
271
+ const cpuTimeMs = (currentUsage.user + currentUsage.system) / 1000;
272
+ const memory = process.memoryUsage();
273
+ const eventLoopLagMs = Number((this.eventLoop.mean / 1_000_000).toFixed(2));
274
+
275
+ this.lastSnapshot = {
276
+ cpuPercent: Number(Math.min((cpuTimeMs / deltaTime) * 100, 100).toFixed(2)),
277
+ rssBytes: memory.rss,
278
+ heapUsedBytes: memory.heapUsed,
279
+ eventLoopLagMs: Number.isFinite(eventLoopLagMs) ? eventLoopLagMs : 0,
280
+ };
281
+
282
+ this.eventLoop.reset();
283
+ }
284
+ }
285
+
286
+ function mergeTopOperations(
287
+ httpOperations: Array<{
288
+ operation: string;
289
+ count: number;
290
+ avgDurationMs: number;
291
+ maxDurationMs: number;
292
+ errorCount: number;
293
+ }>,
294
+ domainOperations: Array<{
295
+ operation: string;
296
+ count?: number;
297
+ avgDurationMs?: number;
298
+ maxDurationMs?: number;
299
+ errorCount?: number;
300
+ }>,
301
+ limit = 5,
302
+ ) {
303
+ const merged = new Map<string, {
304
+ operation: string;
305
+ count?: number;
306
+ avgDurationMs?: number;
307
+ maxDurationMs?: number;
308
+ errorCount?: number;
309
+ }>();
310
+
311
+ for (const item of [...httpOperations, ...domainOperations]) {
312
+ const existing = merged.get(item.operation);
313
+ if (!existing) {
314
+ merged.set(item.operation, { ...item });
315
+ continue;
316
+ }
317
+
318
+ const count = (existing.count ?? 0) + (item.count ?? 0);
319
+ const totalDurationMs =
320
+ (existing.avgDurationMs ?? 0) * (existing.count ?? 0) +
321
+ (item.avgDurationMs ?? 0) * (item.count ?? 0);
322
+
323
+ merged.set(item.operation, {
324
+ operation: item.operation,
325
+ count,
326
+ avgDurationMs: count > 0 ? totalDurationMs / count : 0,
327
+ maxDurationMs: Math.max(existing.maxDurationMs ?? 0, item.maxDurationMs ?? 0),
328
+ errorCount: (existing.errorCount ?? 0) + (item.errorCount ?? 0),
329
+ });
330
+ }
331
+
332
+ const ranked = Array.from(merged.values())
333
+ .sort((a, b) => (b.count ?? 0) - (a.count ?? 0) || (b.maxDurationMs ?? 0) - (a.maxDurationMs ?? 0))
334
+ const sliced = ranked.slice(0, limit);
335
+
336
+ if (domainOperations.length === 0 || sliced.some((item) => !item.operation.startsWith('http.'))) {
337
+ return sliced;
338
+ }
339
+
340
+ const firstDomainOperation = ranked.find((item) => !item.operation.startsWith('http.'));
341
+ if (!firstDomainOperation) {
342
+ return sliced;
343
+ }
344
+
345
+ return [...sliced.slice(0, Math.max(0, limit - 1)), firstDomainOperation];
346
+ }