@kb-labs/rest-api-app 2.14.0 → 2.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -21,8 +21,6 @@ import { WORKFLOW_REDIS_CHANNEL } from '@kb-labs/workflow-constants';
21
21
  import { validatorCompiler, serializerCompiler } from 'fastify-type-provider-zod';
22
22
  import { z } from 'zod';
23
23
  import { errorEnvelopeSchema, ErrorCode, ErrorResponseSchema, JobsListResponseSchema, ListJobsQuerySchema, JobStatsResponseSchema, JobResponseSchema, JobActionResponseSchema } from '@kb-labs/rest-api-contracts';
24
- import { exec } from 'child_process';
25
- import { promisify } from 'util';
26
24
  import { readKbConfig } from '@kb-labs/core-config';
27
25
  import { AdapterNameSchema } from '@kb-labs/gateway-contracts';
28
26
  import fastifyCors from '@fastify/cors';
@@ -2900,261 +2898,15 @@ async function registerCacheRoutes(server, config, registry) {
2900
2898
  }
2901
2899
  });
2902
2900
  }
2903
- var DEFAULT_CONFIG = {
2904
- model: "gpt-4",
2905
- temperature: 0.3,
2906
- maxTokens: 1500,
2907
- debug: false
2908
- };
2909
- var IncidentAnalyzer = class {
2910
- config;
2911
- logger;
2912
- constructor(config = {}, logger = console) {
2913
- this.config = { ...DEFAULT_CONFIG, ...config };
2914
- this.logger = logger;
2915
- }
2916
- /**
2917
- * Analyze incident using LLM
2918
- *
2919
- * @param incident - Incident to analyze
2920
- * @returns AI analysis result
2921
- */
2922
- async analyze(incident) {
2923
- this.log("info", "Analyzing incident", { id: incident.id, type: incident.type });
2924
- const prompt = this.buildAnalysisPrompt(incident);
2925
- try {
2926
- const result = await platform.llm.complete(prompt, {
2927
- model: this.config.model,
2928
- temperature: this.config.temperature,
2929
- maxTokens: this.config.maxTokens,
2930
- systemPrompt: this.getSystemPrompt()
2931
- });
2932
- this.log("debug", "LLM response received", {
2933
- length: result.content.length,
2934
- tokensUsed: result.usage ? result.usage.promptTokens + result.usage.completionTokens : void 0
2935
- });
2936
- const analysis = this.parseAnalysisResponse(result.content);
2937
- this.log("info", "Incident analysis complete", {
2938
- id: incident.id,
2939
- rootCausesCount: analysis.rootCauses.length,
2940
- recommendationsCount: analysis.recommendations.length
2941
- });
2942
- return analysis;
2943
- } catch (error) {
2944
- this.log("error", "Incident analysis failed", {
2945
- id: incident.id,
2946
- error: error instanceof Error ? error.message : String(error)
2947
- });
2948
- return this.getFallbackAnalysis(incident);
2949
- }
2950
- }
2951
- /**
2952
- * Build analysis prompt from incident data
2953
- * @private
2954
- */
2955
- buildAnalysisPrompt(incident) {
2956
- let prompt = `# Incident Analysis Request
2957
-
2958
- `;
2959
- prompt += `## Incident Overview
2960
- `;
2961
- prompt += `- **ID**: ${incident.id}
2962
- `;
2963
- prompt += `- **Type**: ${incident.type}
2964
- `;
2965
- prompt += `- **Severity**: ${incident.severity}
2966
- `;
2967
- prompt += `- **Title**: ${incident.title}
2968
- `;
2969
- prompt += `- **Time**: ${new Date(incident.timestamp).toISOString()}
2970
-
2971
- `;
2972
- prompt += `## Description
2973
- ${incident.details}
2974
-
2975
- `;
2976
- if (incident.metadata && Object.keys(incident.metadata).length > 0) {
2977
- prompt += `## Metrics
2978
- `;
2979
- prompt += "```json\n";
2980
- prompt += JSON.stringify(incident.metadata, null, 2);
2981
- prompt += "\n```\n\n";
2982
- }
2983
- if (incident.relatedData?.logs && incident.relatedData.logs.sampleErrors.length > 0) {
2984
- prompt += `## Related Error Logs (${incident.relatedData.logs.errorCount} total)
2985
- `;
2986
- incident.relatedData.logs.sampleErrors.forEach((error, idx) => {
2987
- prompt += `${idx + 1}. ${error}
2988
- `;
2989
- });
2990
- prompt += "\n";
2991
- }
2992
- if (incident.relatedData?.timeline && incident.relatedData.timeline.length > 0) {
2993
- prompt += `## Event Timeline
2994
- `;
2995
- incident.relatedData.timeline.slice(0, 10).forEach((event) => {
2996
- const time = new Date(event.timestamp).toISOString();
2997
- prompt += `- [${time}] (${event.source}) ${event.event}
2998
- `;
2999
- });
3000
- prompt += "\n";
3001
- }
3002
- if (incident.relatedData?.metrics) {
3003
- prompt += `## System Metrics
3004
- `;
3005
- prompt += "```json\n";
3006
- prompt += JSON.stringify(incident.relatedData.metrics, null, 2);
3007
- prompt += "\n```\n\n";
3008
- }
3009
- prompt += `## Analysis Instructions
3010
- `;
3011
- prompt += `Please analyze this incident and provide:
3012
- `;
3013
- prompt += `1. A brief executive summary (2-3 sentences)
3014
- `;
3015
- prompt += `2. Root causes with confidence scores (0.0-1.0) and evidence
3016
- `;
3017
- prompt += `3. Patterns or trends you observe
3018
- `;
3019
- prompt += `4. Actionable recommendations for resolution and prevention
3020
-
3021
- `;
3022
- prompt += `Format your response as JSON:
3023
- `;
3024
- prompt += "```json\n";
3025
- prompt += `{
3026
- `;
3027
- prompt += ` "summary": "Executive summary here",
3028
- `;
3029
- prompt += ` "rootCauses": [
3030
- `;
3031
- prompt += ` {
3032
- `;
3033
- prompt += ` "factor": "Root cause description",
3034
- `;
3035
- prompt += ` "confidence": 0.85,
3036
- `;
3037
- prompt += ` "evidence": "Supporting evidence from logs/metrics"
3038
- `;
3039
- prompt += ` }
3040
- `;
3041
- prompt += ` ],
3042
- `;
3043
- prompt += ` "patterns": ["Pattern 1", "Pattern 2"],
3044
- `;
3045
- prompt += ` "recommendations": ["Recommendation 1", "Recommendation 2"]
3046
- `;
3047
- prompt += `}
3048
- `;
3049
- prompt += "```";
3050
- return prompt;
3051
- }
3052
- /**
3053
- * Get system prompt for incident analysis
3054
- * @private
3055
- */
3056
- getSystemPrompt() {
3057
- return `You are an expert SRE (Site Reliability Engineer) and incident response specialist.
3058
- Your role is to analyze system incidents, identify root causes, and provide actionable recommendations.
3059
-
3060
- Guidelines:
3061
- - Base your analysis on the actual data provided (logs, metrics, timeline)
3062
- - Provide confidence scores based on available evidence (0.0 = no evidence, 1.0 = certain)
3063
- - Focus on actionable recommendations that can prevent future incidents
3064
- - Be concise but thorough - this is for production systems
3065
- - Always respond with valid JSON matching the requested format`;
3066
- }
3067
- /**
3068
- * Parse LLM response into structured analysis
3069
- * @private
3070
- */
3071
- parseAnalysisResponse(response) {
3072
- try {
3073
- const jsonMatch = response.match(/```json\s*([\s\S]*?)\s*```/) ?? response.match(/```\s*([\s\S]*?)\s*```/) ?? [null, response];
3074
- const jsonStr = jsonMatch[1] || response;
3075
- const parsed = JSON.parse(jsonStr.trim());
3076
- return {
3077
- summary: parsed.summary || "No summary provided",
3078
- rootCauses: Array.isArray(parsed.rootCauses) ? parsed.rootCauses : [],
3079
- patterns: Array.isArray(parsed.patterns) ? parsed.patterns : [],
3080
- recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations : [],
3081
- analyzedAt: Date.now()
3082
- };
3083
- } catch (error) {
3084
- this.log("warn", "Failed to parse LLM response, using fallback", {
3085
- error: error instanceof Error ? error.message : String(error)
3086
- });
3087
- return {
3088
- summary: response.substring(0, 200),
3089
- rootCauses: [],
3090
- patterns: [],
3091
- recommendations: [],
3092
- analyzedAt: Date.now()
3093
- };
3094
- }
3095
- }
3096
- /**
3097
- * Get fallback analysis when LLM is unavailable
3098
- * @private
3099
- */
3100
- getFallbackAnalysis(incident) {
3101
- const errorCount = incident.relatedData?.logs?.errorCount ?? 0;
3102
- const sampleErrors = incident.relatedData?.logs?.sampleErrors ?? [];
3103
- return {
3104
- summary: `${incident.severity} ${incident.type} incident detected. ${errorCount} errors logged.`,
3105
- rootCauses: [
3106
- {
3107
- factor: incident.title,
3108
- confidence: 0.5,
3109
- evidence: incident.details
3110
- }
3111
- ],
3112
- patterns: sampleErrors.length > 0 ? [`Multiple error types detected: ${sampleErrors.length} unique errors`] : [],
3113
- recommendations: [
3114
- "Review related error logs for stack traces",
3115
- "Check system metrics during incident timeframe",
3116
- "Verify external dependencies are operational"
3117
- ],
3118
- analyzedAt: Date.now()
3119
- };
3120
- }
3121
- log(level, message, meta) {
3122
- if (level === "debug" && !this.config.debug) {
3123
- return;
3124
- }
3125
- const prefix = "[IncidentAnalyzer]";
3126
- if (this.logger[level]) {
3127
- if (meta) {
3128
- this.logger[level]({ ...meta }, `${prefix} ${message}`);
3129
- } else {
3130
- this.logger[level](`${prefix} ${message}`);
3131
- }
3132
- } else {
3133
- console.log(`${prefix} [${level}] ${message}`, meta ?? "");
3134
- }
3135
- }
3136
- };
3137
-
3138
- // src/routes/observability.ts
3139
- var execAsync = promisify(exec);
3140
- var DEVKIT_CACHE_KEY = "observability:devkit-health";
3141
- var DEVKIT_CACHE_TTL_MS = 10 * 60 * 1e3;
3142
- async function registerObservabilityRoutes(fastify, config, repoRoot, historicalMetrics, incidentStorage, platform19) {
2901
+ async function registerObservabilityRoutes(fastify, config, repoRoot, historicalMetrics, platform16) {
3143
2902
  const basePath = normalizeBasePath(config.basePath);
3144
2903
  const stateBrokerPaths = resolvePaths(basePath, "/observability/state-broker");
3145
- const devkitPaths = resolvePaths(basePath, "/observability/devkit");
3146
2904
  const systemMetricsPaths = resolvePaths(basePath, "/observability/system-metrics");
3147
2905
  const metricsHistoryPaths = resolvePaths(basePath, "/observability/metrics/history");
3148
2906
  const metricsHeatmapPaths = resolvePaths(basePath, "/observability/metrics/heatmap");
3149
- const incidentsListPaths = resolvePaths(basePath, "/observability/incidents");
3150
- const incidentsDetailPaths = resolvePaths(basePath, "/observability/incidents/:id");
3151
- const incidentsCreatePaths = resolvePaths(basePath, "/observability/incidents");
3152
- const incidentsHistoryPaths = resolvePaths(basePath, "/observability/incidents/history");
3153
- const incidentsResolvePaths = resolvePaths(basePath, "/observability/incidents/:id/resolve");
3154
- const incidentsAnalyzePaths = resolvePaths(basePath, "/observability/incidents/:id/analyze");
3155
2907
  const insightsChatPaths = resolvePaths(basePath, "/observability/insights/chat");
3156
2908
  for (const path3 of stateBrokerPaths) {
3157
- fastify.get(path3, async (_request, reply) => {
2909
+ fastify.get(path3, { schema: { tags: ["Observability"], summary: "State Broker statistics" } }, async (_request, reply) => {
3158
2910
  try {
3159
2911
  const stateBrokerUrl = process.env.KB_STATE_DAEMON_URL || "http://localhost:7777";
3160
2912
  fastify.log.debug({ url: stateBrokerUrl }, "Fetching State Broker stats");
@@ -3193,7 +2945,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3193
2945
  }
3194
2946
  };
3195
2947
  } catch (error) {
3196
- platform19?.logger.error("Failed to fetch State Broker stats", error instanceof Error ? error : new Error(String(error)));
2948
+ platform16?.logger.error("Failed to fetch State Broker stats", error instanceof Error ? error : new Error(String(error)));
3197
2949
  const isTimeout = error instanceof Error && error.name === "AbortError";
3198
2950
  return reply.code(503).send({
3199
2951
  ok: false,
@@ -3208,105 +2960,10 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3208
2960
  }
3209
2961
  });
3210
2962
  }
3211
- for (const path3 of devkitPaths) {
3212
- fastify.get(path3, async (_request, reply) => {
3213
- const now = Date.now();
3214
- if (platform19?.cache) {
3215
- try {
3216
- const cached = await platform19.cache.get(DEVKIT_CACHE_KEY);
3217
- if (cached && now < cached.expiresAt) {
3218
- const remainingTtl = Math.round((cached.expiresAt - now) / 1e3);
3219
- fastify.log.debug({ remainingTtl }, "Returning cached DevKit health from platform.cache");
3220
- return {
3221
- ok: true,
3222
- data: cached.data,
3223
- meta: {
3224
- source: "devkit-cli",
3225
- repoRoot,
3226
- cached: true,
3227
- cachedAt: cached.timestamp,
3228
- expiresAt: cached.expiresAt,
3229
- ttlSeconds: remainingTtl
3230
- }
3231
- };
3232
- }
3233
- } catch (cacheError) {
3234
- fastify.log.warn({ err: cacheError }, "Failed to read from platform.cache, proceeding without cache");
3235
- }
3236
- }
3237
- try {
3238
- fastify.log.debug({ cwd: repoRoot }, "Executing DevKit health check (cache miss)");
3239
- const { stdout, stderr } = await execAsync("npx kb-devkit-health --json", {
3240
- cwd: repoRoot,
3241
- timeout: 3e4,
3242
- // 30s timeout
3243
- env: {
3244
- ...process.env,
3245
- // Ensure DevKit runs in non-interactive mode
3246
- CI: "true"
3247
- }
3248
- });
3249
- if (stderr) {
3250
- fastify.log.warn({ stderr }, "DevKit health check produced warnings");
3251
- }
3252
- const health = JSON.parse(stdout);
3253
- const expiresAt = now + DEVKIT_CACHE_TTL_MS;
3254
- if (platform19?.cache) {
3255
- try {
3256
- await platform19.cache.set(DEVKIT_CACHE_KEY, {
3257
- data: health,
3258
- timestamp: now,
3259
- expiresAt
3260
- }, DEVKIT_CACHE_TTL_MS);
3261
- fastify.log.debug({
3262
- healthScore: health.healthScore,
3263
- grade: health.grade,
3264
- cachedUntil: new Date(expiresAt).toISOString()
3265
- }, "DevKit health check completed and cached in platform.cache");
3266
- } catch (cacheError) {
3267
- fastify.log.warn({ err: cacheError }, "Failed to write to platform.cache");
3268
- }
3269
- }
3270
- return {
3271
- ok: true,
3272
- data: health,
3273
- meta: {
3274
- source: "devkit-cli",
3275
- repoRoot,
3276
- command: "npx kb-devkit-health --json",
3277
- cached: false,
3278
- cachedAt: now,
3279
- expiresAt,
3280
- ttlSeconds: DEVKIT_CACHE_TTL_MS / 1e3
3281
- }
3282
- };
3283
- } catch (error) {
3284
- let partialData = null;
3285
- if (error && typeof error === "object" && "stdout" in error) {
3286
- try {
3287
- partialData = JSON.parse(error.stdout);
3288
- } catch {
3289
- partialData = null;
3290
- }
3291
- }
3292
- platform19?.logger.error("Failed to execute DevKit health check", error instanceof Error ? error : new Error(String(error)));
3293
- return reply.code(500).send({
3294
- ok: false,
3295
- error: {
3296
- code: "DEVKIT_ERROR",
3297
- message: error instanceof Error ? error.message : "Failed to execute DevKit health check",
3298
- details: {
3299
- partialData
3300
- }
3301
- }
3302
- });
3303
- }
3304
- });
3305
- }
3306
2963
  for (const path3 of systemMetricsPaths) {
3307
- fastify.get(path3, async (_request, reply) => {
2964
+ fastify.get(path3, { schema: { tags: ["Observability"], summary: "System resource metrics" } }, async (_request, reply) => {
3308
2965
  try {
3309
- if (!platform19?.cache) {
2966
+ if (!platform16?.cache) {
3310
2967
  return reply.code(503).send({
3311
2968
  ok: false,
3312
2969
  error: {
@@ -3318,17 +2975,17 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3318
2975
  fastify.log.debug("Fetching system metrics from all instances");
3319
2976
  const allMetrics = [];
3320
2977
  try {
3321
- if ("scan" in platform19.cache && typeof platform19.cache.scan === "function") {
3322
- const keys = await platform19.cache.scan("system-metrics:*");
2978
+ if ("scan" in platform16.cache && typeof platform16.cache.scan === "function") {
2979
+ const keys = await platform16.cache.scan("system-metrics:*");
3323
2980
  for (const key of keys) {
3324
- const metrics = await platform19.cache.get(key);
2981
+ const metrics = await platform16.cache.get(key);
3325
2982
  if (metrics) {
3326
2983
  allMetrics.push(metrics);
3327
2984
  }
3328
2985
  }
3329
2986
  } else {
3330
2987
  const currentInstanceId = hostname();
3331
- const metrics = await platform19.cache.get(`system-metrics:${currentInstanceId}`);
2988
+ const metrics = await platform16.cache.get(`system-metrics:${currentInstanceId}`);
3332
2989
  if (metrics) {
3333
2990
  allMetrics.push(metrics);
3334
2991
  }
@@ -3337,7 +2994,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3337
2994
  } catch (scanError) {
3338
2995
  fastify.log.warn({ err: scanError }, "Failed to scan platform.cache for system metrics");
3339
2996
  const currentInstanceId = hostname();
3340
- const metrics = await platform19.cache.get(`system-metrics:${currentInstanceId}`);
2997
+ const metrics = await platform16.cache.get(`system-metrics:${currentInstanceId}`);
3341
2998
  if (metrics) {
3342
2999
  allMetrics.push(metrics);
3343
3000
  }
@@ -3385,7 +3042,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3385
3042
  }
3386
3043
  };
3387
3044
  } catch (error) {
3388
- platform19?.logger.error("Failed to fetch system metrics", error instanceof Error ? error : new Error(String(error)));
3045
+ platform16?.logger.error("Failed to fetch system metrics", error instanceof Error ? error : new Error(String(error)));
3389
3046
  return reply.code(500).send({
3390
3047
  ok: false,
3391
3048
  error: {
@@ -3399,6 +3056,8 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3399
3056
  for (const path3 of metricsHistoryPaths) {
3400
3057
  fastify.get(path3, {
3401
3058
  schema: {
3059
+ tags: ["Observability"],
3060
+ summary: "Historical time-series metrics",
3402
3061
  querystring: {
3403
3062
  type: "object",
3404
3063
  properties: {
@@ -3481,7 +3140,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3481
3140
  }
3482
3141
  };
3483
3142
  } catch (error) {
3484
- platform19?.logger.error("Failed to query historical metrics", error instanceof Error ? error : new Error(String(error)), { query });
3143
+ platform16?.logger.error("Failed to query historical metrics", error instanceof Error ? error : new Error(String(error)), { query });
3485
3144
  return reply.code(500).send({
3486
3145
  ok: false,
3487
3146
  error: {
@@ -3495,6 +3154,8 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3495
3154
  for (const path3 of metricsHeatmapPaths) {
3496
3155
  fastify.get(path3, {
3497
3156
  schema: {
3157
+ tags: ["Observability"],
3158
+ summary: "Metrics heatmap data",
3498
3159
  querystring: {
3499
3160
  type: "object",
3500
3161
  properties: {
@@ -3570,7 +3231,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3570
3231
  }
3571
3232
  };
3572
3233
  } catch (error) {
3573
- platform19?.logger.error("Failed to query heatmap data", error instanceof Error ? error : new Error(String(error)), { query });
3234
+ platform16?.logger.error("Failed to query heatmap data", error instanceof Error ? error : new Error(String(error)), { query });
3574
3235
  return reply.code(500).send({
3575
3236
  ok: false,
3576
3237
  error: {
@@ -3581,310 +3242,11 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3581
3242
  }
3582
3243
  });
3583
3244
  }
3584
- for (const path3 of incidentsListPaths) {
3585
- fastify.get(path3, {
3586
- schema: {
3587
- querystring: {
3588
- type: "object",
3589
- properties: {
3590
- limit: { type: "integer", default: 50 },
3591
- severity: {
3592
- type: "string",
3593
- enum: ["critical", "warning", "info"]
3594
- },
3595
- type: {
3596
- type: "string",
3597
- enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
3598
- },
3599
- from: { type: "integer" },
3600
- to: { type: "integer" },
3601
- includeResolved: { type: "boolean", default: false }
3602
- }
3603
- }
3604
- }
3605
- }, async (request, reply) => {
3606
- if (!incidentStorage) {
3607
- return reply.code(503).send({
3608
- ok: false,
3609
- error: {
3610
- code: "INCIDENTS_NOT_CONFIGURED",
3611
- message: "Incident storage is not configured"
3612
- }
3613
- });
3614
- }
3615
- try {
3616
- const query = request.query;
3617
- const incidents = await incidentStorage.queryIncidents({
3618
- limit: query.limit ?? 50,
3619
- severity: query.severity,
3620
- type: query.type,
3621
- from: query.from,
3622
- to: query.to,
3623
- includeResolved: query.includeResolved ?? false
3624
- });
3625
- const stats = await incidentStorage.getStats();
3626
- return {
3627
- ok: true,
3628
- data: {
3629
- incidents,
3630
- summary: {
3631
- total: stats.total,
3632
- unresolved: stats.unresolved,
3633
- bySeverity: stats.bySeverity,
3634
- showing: incidents.length
3635
- }
3636
- }
3637
- };
3638
- } catch (error) {
3639
- fastify.log.error({ err: error }, "Failed to list incidents");
3640
- return reply.code(500).send({
3641
- ok: false,
3642
- error: {
3643
- code: "INCIDENTS_LIST_ERROR",
3644
- message: error instanceof Error ? error.message : "Failed to list incidents"
3645
- }
3646
- });
3647
- }
3648
- });
3649
- }
3650
- for (const path3 of incidentsDetailPaths) {
3651
- fastify.get(path3, {
3652
- schema: {
3653
- params: {
3654
- type: "object",
3655
- properties: {
3656
- id: { type: "string" }
3657
- },
3658
- required: ["id"]
3659
- }
3660
- }
3661
- }, async (request, reply) => {
3662
- if (!incidentStorage) {
3663
- return reply.code(503).send({
3664
- ok: false,
3665
- error: {
3666
- code: "INCIDENTS_NOT_CONFIGURED",
3667
- message: "Incident storage is not configured"
3668
- }
3669
- });
3670
- }
3671
- const { id } = request.params;
3672
- try {
3673
- const incident = await incidentStorage.getIncident(id);
3674
- if (!incident) {
3675
- return reply.code(404).send({
3676
- ok: false,
3677
- error: {
3678
- code: "INCIDENT_NOT_FOUND",
3679
- message: `Incident ${id} not found`
3680
- }
3681
- });
3682
- }
3683
- if (platform19?.analytics) {
3684
- platform19.analytics.track("incident.viewed", {
3685
- incidentId: id,
3686
- type: incident.type,
3687
- severity: incident.severity,
3688
- isResolved: !!incident.resolvedAt,
3689
- hasAIAnalysis: !!incident.aiAnalysis
3690
- }).catch(() => {
3691
- });
3692
- }
3693
- return {
3694
- ok: true,
3695
- data: incident
3696
- };
3697
- } catch (error) {
3698
- fastify.log.error({ err: error }, "Failed to get incident");
3699
- return reply.code(500).send({
3700
- ok: false,
3701
- error: {
3702
- code: "INCIDENT_GET_ERROR",
3703
- message: error instanceof Error ? error.message : "Failed to get incident"
3704
- }
3705
- });
3706
- }
3707
- });
3708
- }
3709
- for (const path3 of incidentsCreatePaths) {
3710
- fastify.post(path3, {
3711
- schema: {
3712
- body: {
3713
- type: "object",
3714
- properties: {
3715
- type: {
3716
- type: "string",
3717
- enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
3718
- },
3719
- severity: {
3720
- type: "string",
3721
- enum: ["critical", "warning", "info"]
3722
- },
3723
- title: { type: "string" },
3724
- details: { type: "string" },
3725
- rootCause: { type: "string" },
3726
- affectedServices: {
3727
- type: "array",
3728
- items: { type: "string" }
3729
- },
3730
- timestamp: { type: "number" },
3731
- metadata: { type: "object" }
3732
- },
3733
- required: ["type", "severity", "title", "details"]
3734
- }
3735
- }
3736
- }, async (request, reply) => {
3737
- if (!incidentStorage) {
3738
- return reply.code(503).send({
3739
- ok: false,
3740
- error: {
3741
- code: "INCIDENT_STORAGE_UNAVAILABLE",
3742
- message: "Incident storage is not initialized"
3743
- }
3744
- });
3745
- }
3746
- try {
3747
- const payload = request.body;
3748
- const incident = await incidentStorage.createIncident(payload);
3749
- return {
3750
- ok: true,
3751
- data: incident,
3752
- meta: {
3753
- source: "incident-storage"
3754
- }
3755
- };
3756
- } catch (error) {
3757
- platform19?.logger.error("Failed to create incident", error instanceof Error ? error : new Error(String(error)));
3758
- return reply.code(500).send({
3759
- ok: false,
3760
- error: {
3761
- code: "INCIDENT_CREATE_ERROR",
3762
- message: error instanceof Error ? error.message : "Failed to create incident"
3763
- }
3764
- });
3765
- }
3766
- });
3767
- }
3768
- for (const path3 of incidentsHistoryPaths) {
3769
- fastify.get(path3, {
3770
- schema: {
3771
- querystring: {
3772
- type: "object",
3773
- properties: {
3774
- limit: { type: "integer", minimum: 1, maximum: 500 },
3775
- severity: { type: "string", enum: ["critical", "warning", "info"] },
3776
- type: {
3777
- type: "string",
3778
- enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
3779
- },
3780
- from: { type: "integer" },
3781
- to: { type: "integer" },
3782
- includeResolved: { type: "boolean" }
3783
- }
3784
- }
3785
- }
3786
- }, async (request, reply) => {
3787
- if (!incidentStorage) {
3788
- return reply.code(503).send({
3789
- ok: false,
3790
- error: {
3791
- code: "INCIDENT_STORAGE_UNAVAILABLE",
3792
- message: "Incident storage is not initialized"
3793
- }
3794
- });
3795
- }
3796
- try {
3797
- const query = request.query;
3798
- const incidents = await incidentStorage.queryIncidents({
3799
- limit: query.limit ?? 50,
3800
- severity: query.severity,
3801
- type: query.type,
3802
- from: query.from,
3803
- to: query.to,
3804
- includeResolved: query.includeResolved ?? false
3805
- });
3806
- return {
3807
- ok: true,
3808
- data: incidents,
3809
- meta: {
3810
- source: "incident-storage",
3811
- count: incidents.length
3812
- }
3813
- };
3814
- } catch (error) {
3815
- platform19?.logger.error("Failed to query incidents", error instanceof Error ? error : new Error(String(error)));
3816
- return reply.code(500).send({
3817
- ok: false,
3818
- error: {
3819
- code: "INCIDENT_QUERY_ERROR",
3820
- message: error instanceof Error ? error.message : "Failed to query incidents"
3821
- }
3822
- });
3823
- }
3824
- });
3825
- }
3826
- for (const path3 of incidentsResolvePaths) {
3827
- fastify.post(path3, {
3828
- schema: {
3829
- params: {
3830
- type: "object",
3831
- properties: {
3832
- id: { type: "string" }
3833
- },
3834
- required: ["id"]
3835
- },
3836
- body: {
3837
- type: "object",
3838
- properties: {
3839
- resolutionNotes: { type: "string" }
3840
- }
3841
- }
3842
- }
3843
- }, async (request, reply) => {
3844
- if (!incidentStorage) {
3845
- return reply.code(503).send({
3846
- ok: false,
3847
- error: {
3848
- code: "INCIDENT_STORAGE_UNAVAILABLE",
3849
- message: "Incident storage is not initialized"
3850
- }
3851
- });
3852
- }
3853
- try {
3854
- const { id } = request.params;
3855
- const body = request.body;
3856
- const incident = await incidentStorage.resolveIncident(id, body.resolutionNotes);
3857
- if (!incident) {
3858
- return reply.code(404).send({
3859
- ok: false,
3860
- error: {
3861
- code: "INCIDENT_NOT_FOUND",
3862
- message: `Incident with id ${id} not found`
3863
- }
3864
- });
3865
- }
3866
- return {
3867
- ok: true,
3868
- data: incident,
3869
- meta: {
3870
- source: "incident-storage"
3871
- }
3872
- };
3873
- } catch (error) {
3874
- platform19?.logger.error("Failed to resolve incident", error instanceof Error ? error : new Error(String(error)));
3875
- return reply.code(500).send({
3876
- ok: false,
3877
- error: {
3878
- code: "INCIDENT_RESOLVE_ERROR",
3879
- message: error instanceof Error ? error.message : "Failed to resolve incident"
3880
- }
3881
- });
3882
- }
3883
- });
3884
- }
3885
3245
  for (const path3 of insightsChatPaths) {
3886
3246
  fastify.post(path3, {
3887
3247
  schema: {
3248
+ tags: ["Observability"],
3249
+ summary: "AI-powered observability insights chat",
3888
3250
  body: {
3889
3251
  type: "object",
3890
3252
  properties: {
@@ -3893,7 +3255,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3893
3255
  type: "object",
3894
3256
  properties: {
3895
3257
  includeMetrics: { type: "boolean" },
3896
- includeIncidents: { type: "boolean" },
3897
3258
  includeHistory: { type: "boolean" },
3898
3259
  timeRange: { type: "string", enum: ["1h", "6h", "24h", "7d"] },
3899
3260
  plugins: { type: "array", items: { type: "string" } }
@@ -3904,7 +3265,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3904
3265
  }
3905
3266
  }
3906
3267
  }, async (request, reply) => {
3907
- if (!platform19?.llm) {
3268
+ if (!platform16?.llm) {
3908
3269
  return reply.code(503).send({
3909
3270
  ok: false,
3910
3271
  error: {
@@ -3917,7 +3278,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3917
3278
  const body = request.body;
3918
3279
  const contextConfig = {
3919
3280
  includeMetrics: body.context?.includeMetrics ?? true,
3920
- includeIncidents: body.context?.includeIncidents ?? true,
3921
3281
  includeHistory: body.context?.includeHistory ?? true,
3922
3282
  timeRange: body.context?.timeRange ?? "24h",
3923
3283
  plugins: body.context?.plugins ?? []
@@ -3958,26 +3318,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3958
3318
  fastify.log.warn({ err: metricsError }, "Failed to build metrics context for insights");
3959
3319
  }
3960
3320
  }
3961
- if (contextConfig.includeIncidents && incidentStorage) {
3962
- try {
3963
- const incidents = await incidentStorage.queryIncidents({ limit: 10 });
3964
- if (incidents.length > 0) {
3965
- contextText += `
3966
- ## Recent Incidents (${incidents.length})
3967
- `;
3968
- for (const incident of incidents) {
3969
- contextText += `- [${incident.severity.toUpperCase()}] ${incident.title}
3970
- `;
3971
- if (incident.details) {
3972
- contextText += ` Details: ${incident.details.slice(0, 100)}${incident.details.length > 100 ? "..." : ""}
3973
- `;
3974
- }
3975
- }
3976
- }
3977
- } catch (incidentError) {
3978
- fastify.log.warn({ err: incidentError }, "Failed to fetch incidents for insights context");
3979
- }
3980
- }
3981
3321
  const prompt = `You are an AI assistant analyzing a software platform's observability data.
3982
3322
 
3983
3323
  ${contextText}
@@ -3991,15 +3331,15 @@ Provide a clear, actionable response based on the data above. Include:
3991
3331
 
3992
3332
  Be concise but thorough. Use markdown formatting.`;
3993
3333
  fastify.log.debug({ question: body.question, contextLength: contextText.length }, "Calling LLM for insights");
3994
- const result = await platform19.llm.complete(prompt, {
3334
+ const result = await platform16.llm.complete(prompt, {
3995
3335
  systemPrompt: "You are a DevOps and SRE expert assistant. Analyze system metrics and provide actionable insights. Be concise, technical, and helpful.",
3996
3336
  temperature: 0.7,
3997
3337
  maxTokens: 1e3
3998
3338
  });
3999
3339
  const totalTokens = result.usage.promptTokens + result.usage.completionTokens;
4000
3340
  fastify.log.debug({ tokensUsed: totalTokens }, "LLM response received for insights");
4001
- if (platform19.analytics) {
4002
- platform19.analytics.track("ai_insights.chat", {
3341
+ if (platform16.analytics) {
3342
+ platform16.analytics.track("ai_insights.chat", {
4003
3343
  questionLength: body.question.length,
4004
3344
  contextIncluded: Object.keys(contextConfig).filter((k) => contextConfig[k]),
4005
3345
  timeRange: contextConfig.timeRange,
@@ -4026,246 +3366,27 @@ Be concise but thorough. Use markdown formatting.`;
4026
3366
  source: "llm-insights",
4027
3367
  model: result.model
4028
3368
  }
4029
- };
4030
- } catch (error) {
4031
- platform19?.logger.error("Failed to generate insights", error instanceof Error ? error : new Error(String(error)));
4032
- if (platform19?.analytics) {
4033
- platform19.analytics.track("ai_insights.error", {
4034
- error: error instanceof Error ? error.message : "Unknown error",
4035
- questionLength: request.body?.question?.length ?? 0
4036
- }).catch(() => {
4037
- });
4038
- }
4039
- return reply.code(500).send({
4040
- ok: false,
4041
- error: {
4042
- code: "INSIGHTS_ERROR",
4043
- message: error instanceof Error ? error.message : "Failed to generate insights"
4044
- }
4045
- });
4046
- }
4047
- });
4048
- }
4049
- for (const path3 of incidentsAnalyzePaths) {
4050
- fastify.post(path3, async (request, reply) => {
4051
- if (!incidentStorage) {
4052
- return reply.code(503).send({
4053
- ok: false,
4054
- error: {
4055
- code: "INCIDENTS_NOT_CONFIGURED",
4056
- message: "Incident storage is not configured"
4057
- }
4058
- });
4059
- }
4060
- const { id } = request.params;
4061
- try {
4062
- const incident = await incidentStorage.getIncident(id);
4063
- if (!incident) {
4064
- return reply.code(404).send({
4065
- ok: false,
4066
- error: {
4067
- code: "INCIDENT_NOT_FOUND",
4068
- message: `Incident ${id} not found`
4069
- }
4070
- });
4071
- }
4072
- if (incident.aiAnalysis && incident.aiAnalyzedAt) {
4073
- const ageMs = Date.now() - incident.aiAnalyzedAt;
4074
- const ageMinutes = Math.floor(ageMs / 6e4);
4075
- if (ageMs < 60 * 60 * 1e3) {
4076
- return {
4077
- ok: true,
4078
- data: {
4079
- ...incident.aiAnalysis,
4080
- cached: true,
4081
- analyzedAt: incident.aiAnalyzedAt,
4082
- ageMinutes
4083
- }
4084
- };
4085
- }
4086
- }
4087
- const analyzer = new IncidentAnalyzer(
4088
- {
4089
- debug: process.env.NODE_ENV !== "production"
4090
- },
4091
- fastify.log
4092
- );
4093
- const analysis = await analyzer.analyze(incident);
4094
- await incidentStorage.updateAIAnalysis(id, analysis);
4095
- if (platform19?.analytics) {
4096
- platform19.analytics.track("incident.analyzed", {
4097
- incidentId: id,
4098
- type: incident.type,
4099
- severity: incident.severity,
4100
- rootCausesCount: analysis.rootCauses.length,
4101
- recommendationsCount: analysis.recommendations.length
4102
- }).catch(() => {
4103
- });
4104
- }
4105
- return {
4106
- ok: true,
4107
- data: {
4108
- ...analysis,
4109
- cached: false
4110
- }
4111
- };
4112
- } catch (error) {
4113
- fastify.log.error({ err: error }, "Failed to analyze incident");
4114
- if (platform19?.analytics) {
4115
- platform19.analytics.track("incident.analysis_error", {
4116
- incidentId: id,
4117
- error: error instanceof Error ? error.message : "Unknown error"
4118
- }).catch(() => {
4119
- });
4120
- }
4121
- return reply.code(500).send({
4122
- ok: false,
4123
- error: {
4124
- code: "ANALYSIS_ERROR",
4125
- message: error instanceof Error ? error.message : "Failed to analyze incident"
4126
- }
4127
- });
4128
- }
4129
- });
4130
- }
4131
- const testCreateIncidentPaths = resolvePaths(basePath, "/test/create-incident");
4132
- for (const path3 of testCreateIncidentPaths) {
4133
- fastify.post(path3, async (request, reply) => {
4134
- if (!incidentStorage) {
4135
- return reply.code(500).send({
4136
- ok: false,
4137
- error: { code: "INCIDENT_STORAGE_NOT_INITIALIZED", message: "Incident storage not initialized" }
4138
- });
4139
- }
4140
- try {
4141
- const {
4142
- type = "custom",
4143
- severity = "warning",
4144
- title = "Test Incident",
4145
- details,
4146
- relatedData
4147
- } = request.body;
4148
- const incident = await incidentStorage.createIncident({
4149
- type,
4150
- severity,
4151
- title,
4152
- details: details || "This is a test incident created manually for testing purposes.",
4153
- timestamp: Date.now(),
4154
- metadata: {
4155
- testMode: true,
4156
- createdVia: "test-endpoint"
4157
- },
4158
- relatedData: relatedData || {
4159
- timeline: [
4160
- {
4161
- timestamp: Date.now(),
4162
- event: "Test incident created via /test/create-incident",
4163
- source: "manual"
4164
- }
4165
- ]
4166
- }
4167
- });
4168
- fastify.log.info({ id: incident.id }, "Test incident created");
4169
- return { ok: true, data: incident };
4170
- } catch (error) {
4171
- fastify.log.error({ err: error }, "Failed to create test incident");
4172
- return reply.code(500).send({
4173
- ok: false,
4174
- error: {
4175
- code: "TEST_INCIDENT_CREATION_FAILED",
4176
- message: error instanceof Error ? error.message : "Failed to create test incident"
4177
- }
4178
- });
4179
- }
4180
- });
4181
- }
4182
- const testTriggerErrorsPaths = resolvePaths(basePath, "/test/trigger-errors");
4183
- for (const path3 of testTriggerErrorsPaths) {
4184
- fastify.get(path3, async (request, reply) => {
4185
- const { count = 10 } = request.query;
4186
- const errorCount = Math.min(Math.max(1, parseInt(count, 10) || 10), 100);
4187
- fastify.log.info(`Triggering ${errorCount} test errors`);
4188
- for (let i = 0; i < errorCount; i++) {
4189
- platform19?.logger.error(`Test error ${i + 1}/${errorCount}`, void 0, {
4190
- testMode: true,
4191
- errorNumber: i + 1,
4192
- totalErrors: errorCount
4193
- });
4194
- if (i === errorCount - 1) {
4195
- return reply.code(500).send({
4196
- ok: false,
4197
- error: {
4198
- code: "TEST_ERROR_TRIGGERED",
4199
- message: `Generated ${errorCount} test errors. Check /observability/incidents in ~30 seconds for auto-created incident.`
4200
- }
3369
+ };
3370
+ } catch (error) {
3371
+ platform16?.logger.error("Failed to generate insights", error instanceof Error ? error : new Error(String(error)));
3372
+ if (platform16?.analytics) {
3373
+ platform16.analytics.track("ai_insights.error", {
3374
+ error: error instanceof Error ? error.message : "Unknown error",
3375
+ questionLength: request.body?.question?.length ?? 0
3376
+ }).catch(() => {
4201
3377
  });
4202
3378
  }
4203
- }
4204
- return {
4205
- ok: true,
4206
- message: `Triggered ${errorCount} test errors. Incident should be auto-created in next detection cycle (~30s).`
4207
- };
4208
- });
4209
- }
4210
- const testSimulateLatencyPaths = resolvePaths(basePath, "/test/simulate-latency");
4211
- for (const path3 of testSimulateLatencyPaths) {
4212
- fastify.get(path3, async (request, reply) => {
4213
- const { delay = 2e3 } = request.query;
4214
- const delayMs = Math.min(Math.max(100, parseInt(delay, 10) || 2e3), 1e4);
4215
- fastify.log.info(`Simulating ${delayMs}ms latency`);
4216
- await new Promise((resolve3) => {
4217
- setTimeout(resolve3, delayMs);
4218
- });
4219
- return {
4220
- ok: true,
4221
- message: `Simulated ${delayMs}ms latency. Call this endpoint multiple times to trigger latency_spike incident.`,
4222
- actualDelay: delayMs
4223
- };
4224
- });
4225
- }
4226
- const testBulkErrorsPaths = resolvePaths(basePath, "/test/bulk-errors");
4227
- for (const path3 of testBulkErrorsPaths) {
4228
- fastify.post(path3, async (request, reply) => {
4229
- const { successCount = 10, errorCount = 50 } = request.body;
4230
- fastify.log.info({ successCount, errorCount }, "Generating bulk requests for incident testing");
4231
- for (let i = 0; i < successCount; i++) {
4232
- platform19?.logger.info(`Bulk test - success ${i + 1}/${successCount}`);
4233
- }
4234
- const testEndpoints = [
4235
- "POST /api/v1/test/endpoint-a",
4236
- "GET /api/v1/test/endpoint-b",
4237
- "PUT /api/v1/test/endpoint-c",
4238
- "DELETE /api/v1/test/endpoint-d",
4239
- "POST /api/v1/test/endpoint-e"
4240
- ];
4241
- for (let i = 0; i < errorCount; i++) {
4242
- const endpoint = testEndpoints[i % testEndpoints.length];
4243
- const errorTypes = [
4244
- "Connection timeout",
4245
- "Validation failed",
4246
- "Database query error",
4247
- "Authentication failed",
4248
- "Rate limit exceeded"
4249
- ];
4250
- const errorType = errorTypes[i % errorTypes.length];
4251
- platform19?.logger.error(`${errorType}: ${endpoint}`, void 0, {
4252
- testMode: true,
4253
- bulkTest: true,
4254
- errorIndex: i,
4255
- endpoint
3379
+ return reply.code(500).send({
3380
+ ok: false,
3381
+ error: {
3382
+ code: "INSIGHTS_ERROR",
3383
+ message: error instanceof Error ? error.message : "Failed to generate insights"
3384
+ }
4256
3385
  });
4257
3386
  }
4258
- const totalRequests = successCount + errorCount;
4259
- const errorRate = errorCount / totalRequests * 100;
4260
- return {
4261
- ok: true,
4262
- message: `Generated ${totalRequests} requests (${successCount} success, ${errorCount} errors)`,
4263
- errorRate: `${errorRate.toFixed(1)}%`,
4264
- note: "Incident should be auto-created in next detection cycle (~30s)"
4265
- };
4266
3387
  });
4267
3388
  }
4268
- fastify.log.info("Observability routes registered (including test endpoints)");
3389
+ fastify.log.info("Observability routes registered");
4269
3390
  }
4270
3391
  async function registerAnalyticsRoutes(fastify, config) {
4271
3392
  const basePath = normalizeBasePath(config.basePath);
@@ -5598,7 +4719,7 @@ function generateFallbackSummary(stats, logs) {
5598
4719
  return summary;
5599
4720
  }
5600
4721
  async function registerLogRoutes(server, config, eventHub) {
5601
- server.get("/api/v1/logs", async (request, reply) => {
4722
+ server.get("/api/v1/logs", { schema: { tags: ["Logs"], summary: "Query logs with filters" } }, async (request, reply) => {
5602
4723
  try {
5603
4724
  const limit = request.query.limit ? parseInt(request.query.limit, 10) : 100;
5604
4725
  const offset = request.query.offset ? parseInt(request.query.offset, 10) : 0;
@@ -5656,6 +4777,7 @@ async function registerLogRoutes(server, config, eventHub) {
5656
4777
  });
5657
4778
  server.get(
5658
4779
  "/api/v1/logs/:id",
4780
+ { schema: { tags: ["Logs"], summary: "Get log entry by ID" } },
5659
4781
  async (request, reply) => {
5660
4782
  try {
5661
4783
  const log = await platform.logs.getById(request.params.id);
@@ -5689,6 +4811,7 @@ async function registerLogRoutes(server, config, eventHub) {
5689
4811
  );
5690
4812
  server.get(
5691
4813
  "/api/v1/logs/:id/related",
4814
+ { schema: { tags: ["Logs"], summary: "Get logs related to a specific entry" } },
5692
4815
  async (request, reply) => {
5693
4816
  try {
5694
4817
  const log = await platform.logs.getById(request.params.id);
@@ -5717,7 +4840,7 @@ async function registerLogRoutes(server, config, eventHub) {
5717
4840
  }
5718
4841
  }
5719
4842
  );
5720
- server.get("/api/v1/logs/stream", async (request, reply) => {
4843
+ server.get("/api/v1/logs/stream", { schema: { hide: true } }, async (request, reply) => {
5721
4844
  const caps = platform.logs.getCapabilities();
5722
4845
  if (!caps.hasStreaming) {
5723
4846
  return reply.code(503).send({
@@ -5775,7 +4898,7 @@ async function registerLogRoutes(server, config, eventHub) {
5775
4898
  await new Promise(() => {
5776
4899
  });
5777
4900
  });
5778
- server.get("/api/v1/logs/stats", async (request, reply) => {
4901
+ server.get("/api/v1/logs/stats", { schema: { tags: ["Logs"], summary: "Get log storage statistics" } }, async (request, reply) => {
5779
4902
  try {
5780
4903
  const stats = await platform.logs.getStats();
5781
4904
  const caps = platform.logs.getCapabilities();
@@ -5805,7 +4928,7 @@ async function registerLogRoutes(server, config, eventHub) {
5805
4928
  });
5806
4929
  }
5807
4930
  });
5808
- server.post("/api/v1/logs/summarize", async (request, reply) => {
4931
+ server.post("/api/v1/logs/summarize", { schema: { tags: ["Logs"], summary: "AI-powered log summarization" } }, async (request, reply) => {
5809
4932
  const { timeRange, filters, groupBy, question, includeContext } = request.body;
5810
4933
  try {
5811
4934
  const query = {
@@ -6197,7 +5320,7 @@ async function registerDebugRoutes(fastify, config) {
6197
5320
  }
6198
5321
 
6199
5322
  // src/services/historical-metrics.ts
6200
- var DEFAULT_CONFIG2 = {
5323
+ var DEFAULT_CONFIG = {
6201
5324
  intervalMs: 5e3,
6202
5325
  maxPoints: {
6203
5326
  "1m": 12,
@@ -6216,7 +5339,7 @@ var HistoricalMetricsCollector = class {
6216
5339
  startTimeMs = Date.now();
6217
5340
  constructor(cache, config = {}, logger = console) {
6218
5341
  this.cache = cache;
6219
- this.config = { ...DEFAULT_CONFIG2, ...config, maxPoints: { ...DEFAULT_CONFIG2.maxPoints, ...config.maxPoints } };
5342
+ this.config = { ...DEFAULT_CONFIG, ...config, maxPoints: { ...DEFAULT_CONFIG.maxPoints, ...config.maxPoints } };
6220
5343
  this.logger = logger;
6221
5344
  }
6222
5345
  /**
@@ -6516,754 +5639,6 @@ var HistoricalMetricsCollector = class {
6516
5639
  }
6517
5640
  }
6518
5641
  };
6519
- var DEFAULT_CONFIG3 = {
6520
- ttlMs: 30 * 24 * 60 * 60 * 1e3,
6521
- // 30 days (not used with SQLite, kept for compatibility)
6522
- maxIncidents: 1e3,
6523
- // Not enforced with SQLite, use retention policy instead
6524
- debug: false
6525
- };
6526
- var IncidentStorage = class {
6527
- db;
6528
- config;
6529
- logger;
6530
- constructor(db, config = {}, logger = console) {
6531
- this.db = db;
6532
- this.config = { ...DEFAULT_CONFIG3, ...config };
6533
- this.logger = logger;
6534
- }
6535
- /**
6536
- * Create a new incident record
6537
- */
6538
- async createIncident(payload) {
6539
- const id = `inc-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
6540
- const timestamp = payload.timestamp ?? Date.now();
6541
- if (!payload.type || !payload.severity || !payload.title) {
6542
- throw new Error("Incident must have type, severity, and title");
6543
- }
6544
- const relatedLogsCount = payload.relatedData?.logs?.errorCount ?? 0;
6545
- const relatedLogsSample = payload.relatedData?.logs ? JSON.stringify(payload.relatedData.logs) : null;
6546
- const relatedMetrics = payload.relatedData?.metrics ? JSON.stringify(payload.relatedData.metrics) : null;
6547
- const timeline = payload.relatedData?.timeline ? JSON.stringify(payload.relatedData.timeline) : null;
6548
- await this.db.query(
6549
- `INSERT INTO incidents (
6550
- id, type, severity, title, details, timestamp,
6551
- resolved_at, resolution_notes,
6552
- affected_services, metadata,
6553
- related_logs_count, related_logs_sample, related_metrics, timeline
6554
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
6555
- [
6556
- id,
6557
- payload.type,
6558
- payload.severity,
6559
- payload.title,
6560
- payload.details,
6561
- timestamp,
6562
- payload.resolvedAt ?? null,
6563
- payload.resolutionNotes ?? null,
6564
- payload.affectedServices ? JSON.stringify(payload.affectedServices) : null,
6565
- payload.metadata ? JSON.stringify(payload.metadata) : null,
6566
- relatedLogsCount,
6567
- relatedLogsSample,
6568
- relatedMetrics,
6569
- timeline
6570
- ]
6571
- );
6572
- this.log("info", "Incident created", {
6573
- id,
6574
- type: payload.type,
6575
- severity: payload.severity
6576
- });
6577
- const created = await this.getIncident(id);
6578
- if (!created) {
6579
- throw new Error("Failed to retrieve created incident");
6580
- }
6581
- return created;
6582
- }
6583
- /**
6584
- * Get incident by ID
6585
- */
6586
- async getIncident(id) {
6587
- const result = await this.db.query(
6588
- "SELECT * FROM incidents WHERE id = ?",
6589
- [id]
6590
- );
6591
- if (result.rows.length === 0) {
6592
- return null;
6593
- }
6594
- return this.rowToIncident(result.rows[0]);
6595
- }
6596
- /**
6597
- * Query incidents with filters
6598
- */
6599
- async queryIncidents(options = {}) {
6600
- const {
6601
- limit = 50,
6602
- severity,
6603
- type,
6604
- from,
6605
- to,
6606
- includeResolved = false
6607
- } = options;
6608
- const conditions = [];
6609
- const params = [];
6610
- if (severity) {
6611
- const severityList = Array.isArray(severity) ? severity : [severity];
6612
- conditions.push(`severity IN (${severityList.map(() => "?").join(", ")})`);
6613
- params.push(...severityList);
6614
- }
6615
- if (type) {
6616
- const typeList = Array.isArray(type) ? type : [type];
6617
- conditions.push(`type IN (${typeList.map(() => "?").join(", ")})`);
6618
- params.push(...typeList);
6619
- }
6620
- if (from) {
6621
- conditions.push("timestamp >= ?");
6622
- params.push(from);
6623
- }
6624
- if (to) {
6625
- conditions.push("timestamp <= ?");
6626
- params.push(to);
6627
- }
6628
- if (!includeResolved) {
6629
- conditions.push("resolved_at IS NULL");
6630
- }
6631
- const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
6632
- const sql = `
6633
- SELECT * FROM incidents
6634
- ${whereClause}
6635
- ORDER BY timestamp DESC
6636
- LIMIT ?
6637
- `;
6638
- const result = await this.db.query(sql, [...params, limit]);
6639
- return result.rows.map((row) => this.rowToIncident(row));
6640
- }
6641
- /**
6642
- * Resolve an incident
6643
- */
6644
- async resolveIncident(id, resolutionNotes) {
6645
- const resolvedAt = Date.now();
6646
- const result = await this.db.query(
6647
- `UPDATE incidents
6648
- SET resolved_at = ?, resolution_notes = ?
6649
- WHERE id = ?`,
6650
- [resolvedAt, resolutionNotes ?? null, id]
6651
- );
6652
- if (result.rowCount === 0) {
6653
- return null;
6654
- }
6655
- this.log("info", "Incident resolved", { id, resolvedAt });
6656
- const incident = await this.getIncident(id);
6657
- if (incident && platform.analytics) {
6658
- const durationMs = resolvedAt - incident.timestamp;
6659
- const durationMinutes = Math.floor(durationMs / 6e4);
6660
- platform.analytics.track("incident.resolved", {
6661
- incidentId: id,
6662
- type: incident.type,
6663
- severity: incident.severity,
6664
- durationMs,
6665
- durationMinutes,
6666
- hasResolutionNotes: !!resolutionNotes,
6667
- wasAnalyzed: !!incident.aiAnalysis
6668
- }).catch(() => {
6669
- });
6670
- }
6671
- return incident;
6672
- }
6673
- /**
6674
- * Delete an incident
6675
- */
6676
- async deleteIncident(id) {
6677
- const incident = await this.getIncident(id);
6678
- const result = await this.db.query("DELETE FROM incidents WHERE id = ?", [id]);
6679
- const deleted = result.rowCount > 0;
6680
- if (deleted) {
6681
- this.log("info", "Incident deleted", { id });
6682
- if (incident && platform.analytics) {
6683
- platform.analytics.track("incident.deleted", {
6684
- incidentId: id,
6685
- type: incident.type,
6686
- severity: incident.severity,
6687
- wasResolved: !!incident.resolvedAt,
6688
- wasAnalyzed: !!incident.aiAnalysis
6689
- }).catch(() => {
6690
- });
6691
- }
6692
- }
6693
- return deleted;
6694
- }
6695
- /**
6696
- * Get statistics about stored incidents
6697
- */
6698
- async getStats() {
6699
- const totalResult = await this.db.query(`
6700
- SELECT
6701
- COUNT(*) as total,
6702
- MIN(timestamp) as oldest,
6703
- MAX(timestamp) as newest
6704
- FROM incidents
6705
- `);
6706
- const { total, oldest, newest } = totalResult.rows[0];
6707
- const severityResult = await this.db.query("SELECT severity, COUNT(*) as count FROM incidents GROUP BY severity");
6708
- const bySeverity = {
6709
- critical: 0,
6710
- warning: 0,
6711
- info: 0
6712
- };
6713
- for (const row of severityResult.rows) {
6714
- bySeverity[row.severity] = row.count;
6715
- }
6716
- const typeResult = await this.db.query("SELECT type, COUNT(*) as count FROM incidents GROUP BY type");
6717
- const byType = {};
6718
- for (const row of typeResult.rows) {
6719
- byType[row.type] = row.count;
6720
- }
6721
- const resolvedResult = await this.db.query(`
6722
- SELECT
6723
- SUM(CASE WHEN resolved_at IS NOT NULL THEN 1 ELSE 0 END) as resolved,
6724
- SUM(CASE WHEN resolved_at IS NULL THEN 1 ELSE 0 END) as unresolved
6725
- FROM incidents
6726
- `);
6727
- const { resolved, unresolved } = resolvedResult.rows[0];
6728
- return {
6729
- total,
6730
- bySeverity,
6731
- byType,
6732
- resolved: resolved ?? 0,
6733
- unresolved: unresolved ?? 0,
6734
- oldestTimestamp: oldest,
6735
- newestTimestamp: newest
6736
- };
6737
- }
6738
- /**
6739
- * Clear all incidents (admin function)
6740
- */
6741
- async clearAll() {
6742
- await this.db.query("DELETE FROM incidents");
6743
- this.log("warn", "All incidents cleared");
6744
- }
6745
- /**
6746
- * Update AI analysis for an incident
6747
- */
6748
- async updateAIAnalysis(id, analysis) {
6749
- await this.db.query(
6750
- `UPDATE incidents
6751
- SET ai_analysis = ?, ai_analyzed_at = ?
6752
- WHERE id = ?`,
6753
- [JSON.stringify(analysis), Date.now(), id]
6754
- );
6755
- this.log("debug", "AI analysis updated", { id });
6756
- }
6757
- /**
6758
- * Convert database row to Incident object
6759
- * @private
6760
- */
6761
- rowToIncident(row) {
6762
- const affectedServices = row.affected_services ? JSON.parse(row.affected_services) : void 0;
6763
- const metadata = row.metadata ? JSON.parse(row.metadata) : void 0;
6764
- const aiAnalysis = row.ai_analysis ? JSON.parse(row.ai_analysis) : void 0;
6765
- const relatedData = row.related_logs_sample || row.related_metrics || row.timeline ? {
6766
- logs: row.related_logs_sample ? JSON.parse(row.related_logs_sample) : void 0,
6767
- metrics: row.related_metrics ? JSON.parse(row.related_metrics) : void 0,
6768
- timeline: row.timeline ? JSON.parse(row.timeline) : void 0
6769
- } : void 0;
6770
- return {
6771
- id: row.id,
6772
- type: row.type,
6773
- severity: row.severity,
6774
- title: row.title,
6775
- details: row.details,
6776
- timestamp: row.timestamp,
6777
- resolvedAt: row.resolved_at ?? void 0,
6778
- resolutionNotes: row.resolution_notes ?? void 0,
6779
- affectedServices,
6780
- metadata,
6781
- relatedData,
6782
- aiAnalysis,
6783
- aiAnalyzedAt: row.ai_analyzed_at ?? void 0
6784
- };
6785
- }
6786
- log(level, message, meta) {
6787
- if (level === "debug" && !this.config.debug) {
6788
- return;
6789
- }
6790
- if (this.logger[level]) {
6791
- this.logger[level](`[IncidentStorage] ${message}`, meta);
6792
- } else {
6793
- console.log(`[IncidentStorage] [${level}] ${message}`, meta);
6794
- }
6795
- }
6796
- };
6797
- var DEFAULT_THRESHOLDS = {
6798
- errorRateWarning: 5,
6799
- errorRateCritical: 10,
6800
- latencyP99Warning: 2e3,
6801
- latencyP99Critical: 5e3,
6802
- latencyP95Warning: 1e3,
6803
- minRequestsForDetection: 10,
6804
- pluginErrorRateWarning: 10,
6805
- pluginErrorRateCritical: 25
6806
- };
6807
- var DEFAULT_CONFIG4 = {
6808
- intervalMs: 3e4,
6809
- // 30 seconds
6810
- thresholds: DEFAULT_THRESHOLDS,
6811
- cooldownMs: 5 * 60 * 1e3,
6812
- // 5 minutes
6813
- debug: false
6814
- };
6815
- var IncidentDetector = class {
6816
- incidentStorage;
6817
- config;
6818
- logger;
6819
- intervalHandle = null;
6820
- recentIncidents = [];
6821
- isRunning = false;
6822
- // Metrics history for "before" comparison (last 10 snapshots)
6823
- metricsHistory = [];
6824
- constructor(incidentStorage, config = {}, logger = console) {
6825
- this.incidentStorage = incidentStorage;
6826
- this.config = {
6827
- ...DEFAULT_CONFIG4,
6828
- ...config,
6829
- thresholds: { ...DEFAULT_THRESHOLDS, ...config.thresholds }
6830
- };
6831
- this.logger = logger;
6832
- }
6833
- /**
6834
- * Start automatic detection
6835
- */
6836
- start() {
6837
- if (this.isRunning) {
6838
- this.log("warn", "Detector already running");
6839
- return;
6840
- }
6841
- this.isRunning = true;
6842
- this.log("info", "Starting incident detector", {
6843
- intervalMs: this.config.intervalMs,
6844
- thresholds: this.config.thresholds
6845
- });
6846
- this.runDetection().catch((err) => {
6847
- this.log("error", "Initial detection failed", { error: err.message });
6848
- });
6849
- this.intervalHandle = setInterval(() => {
6850
- this.runDetection().catch((err) => {
6851
- this.log("error", "Detection cycle failed", { error: err.message });
6852
- });
6853
- }, this.config.intervalMs);
6854
- }
6855
- /**
6856
- * Stop automatic detection
6857
- */
6858
- stop() {
6859
- if (this.intervalHandle) {
6860
- clearInterval(this.intervalHandle);
6861
- this.intervalHandle = null;
6862
- }
6863
- this.isRunning = false;
6864
- this.log("info", "Incident detector stopped");
6865
- }
6866
- /**
6867
- * Run a single detection cycle
6868
- */
6869
- async runDetection() {
6870
- const metrics = metricsCollector.getMetrics();
6871
- const now = Date.now();
6872
- const totalErrors = (metrics.requests.clientErrors ?? 0) + (metrics.requests.serverErrors ?? 0);
6873
- const errorRate = metrics.requests.total > 0 ? totalErrors / metrics.requests.total * 100 : 0;
6874
- this.metricsHistory.push({
6875
- timestamp: now,
6876
- errorRate,
6877
- avgLatency: metrics.latency.average ?? 0,
6878
- totalRequests: metrics.requests.total
6879
- });
6880
- if (this.metricsHistory.length > 10) {
6881
- this.metricsHistory.shift();
6882
- }
6883
- this.recentIncidents = this.recentIncidents.filter(
6884
- (ri) => now - ri.timestamp < this.config.cooldownMs
6885
- );
6886
- if (metrics.requests.total < this.config.thresholds.minRequestsForDetection) {
6887
- this.log("debug", "Not enough requests for detection", {
6888
- total: metrics.requests.total,
6889
- min: this.config.thresholds.minRequestsForDetection
6890
- });
6891
- return;
6892
- }
6893
- await this.detectErrorRate(metrics, now);
6894
- await this.detectLatencyIssues(metrics, now);
6895
- await this.detectPluginFailures(metrics, now);
6896
- this.log("debug", "Detection cycle complete", {
6897
- recentIncidentsCount: this.recentIncidents.length
6898
- });
6899
- }
6900
- /**
6901
- * Detect high error rate
6902
- */
6903
- async detectErrorRate(metrics, now) {
6904
- const totalErrors = (metrics.requests.clientErrors ?? 0) + (metrics.requests.serverErrors ?? 0);
6905
- const errorRate = metrics.requests.total > 0 ? totalErrors / metrics.requests.total * 100 : 0;
6906
- const key = "error_rate";
6907
- if (this.isInCooldown(key)) {
6908
- return;
6909
- }
6910
- let severity = null;
6911
- let title = "";
6912
- let details = "";
6913
- if (errorRate >= this.config.thresholds.errorRateCritical) {
6914
- severity = "critical";
6915
- title = `Critical Error Rate: ${errorRate.toFixed(1)}%`;
6916
- details = `Error rate has exceeded critical threshold of ${this.config.thresholds.errorRateCritical}%. Current: ${errorRate.toFixed(2)}% (${totalErrors}/${metrics.requests.total} requests). Client errors: ${metrics.requests.clientErrors ?? 0}, Server errors: ${metrics.requests.serverErrors ?? 0}.`;
6917
- } else if (errorRate >= this.config.thresholds.errorRateWarning) {
6918
- severity = "warning";
6919
- title = `High Error Rate: ${errorRate.toFixed(1)}%`;
6920
- details = `Error rate has exceeded warning threshold of ${this.config.thresholds.errorRateWarning}%. Current: ${errorRate.toFixed(2)}% (${totalErrors}/${metrics.requests.total} requests).`;
6921
- }
6922
- if (severity) {
6923
- await this.createIncident({
6924
- type: "error_rate",
6925
- severity,
6926
- title,
6927
- details,
6928
- metadata: {
6929
- errorRate,
6930
- totalRequests: metrics.requests.total,
6931
- clientErrors: metrics.requests.clientErrors,
6932
- serverErrors: metrics.requests.serverErrors,
6933
- threshold: severity === "critical" ? this.config.thresholds.errorRateCritical : this.config.thresholds.errorRateWarning
6934
- }
6935
- }, key, now);
6936
- }
6937
- }
6938
- /**
6939
- * Detect latency spikes
6940
- */
6941
- async detectLatencyIssues(metrics, now) {
6942
- const avgLatency = metrics.latency.average ?? 0;
6943
- let p99 = avgLatency * 3;
6944
- let p95 = avgLatency * 2;
6945
- if (metrics.latency.histogram && metrics.latency.histogram.length > 0) {
6946
- const sorted = [...metrics.latency.histogram].sort((a, b) => b.max - a.max);
6947
- if (sorted.length > 0) {
6948
- p99 = sorted[0]?.max ?? p99;
6949
- p95 = sorted[Math.floor(sorted.length * 0.05)]?.max ?? p95;
6950
- }
6951
- }
6952
- const keyP99 = "latency_p99";
6953
- const keyP95 = "latency_p95";
6954
- if (!this.isInCooldown(keyP99)) {
6955
- let severity = null;
6956
- let title = "";
6957
- let details = "";
6958
- if (p99 >= this.config.thresholds.latencyP99Critical) {
6959
- severity = "critical";
6960
- title = `Critical P99 Latency: ${p99.toFixed(0)}ms`;
6961
- details = `P99 latency has exceeded critical threshold of ${this.config.thresholds.latencyP99Critical}ms. Current P99: ${p99.toFixed(0)}ms, Average: ${avgLatency.toFixed(0)}ms. This indicates severe performance degradation affecting tail latencies.`;
6962
- } else if (p99 >= this.config.thresholds.latencyP99Warning) {
6963
- severity = "warning";
6964
- title = `High P99 Latency: ${p99.toFixed(0)}ms`;
6965
- details = `P99 latency has exceeded warning threshold of ${this.config.thresholds.latencyP99Warning}ms. Current P99: ${p99.toFixed(0)}ms, Average: ${avgLatency.toFixed(0)}ms.`;
6966
- }
6967
- if (severity) {
6968
- await this.createIncident({
6969
- type: "latency_spike",
6970
- severity,
6971
- title,
6972
- details,
6973
- metadata: {
6974
- p99,
6975
- p95,
6976
- average: avgLatency,
6977
- threshold: severity === "critical" ? this.config.thresholds.latencyP99Critical : this.config.thresholds.latencyP99Warning
6978
- }
6979
- }, keyP99, now);
6980
- }
6981
- }
6982
- if (!this.isInCooldown(keyP95) && p95 >= this.config.thresholds.latencyP95Warning) {
6983
- await this.createIncident({
6984
- type: "latency_spike",
6985
- severity: "warning",
6986
- title: `Elevated P95 Latency: ${p95.toFixed(0)}ms`,
6987
- details: `P95 latency has exceeded threshold of ${this.config.thresholds.latencyP95Warning}ms. Current P95: ${p95.toFixed(0)}ms. Consider investigating slow endpoints.`,
6988
- metadata: {
6989
- p95,
6990
- average: avgLatency,
6991
- threshold: this.config.thresholds.latencyP95Warning
6992
- }
6993
- }, keyP95, now);
6994
- }
6995
- }
6996
- /**
6997
- * Detect plugin failures
6998
- */
6999
- async detectPluginFailures(metrics, now) {
7000
- if (!metrics.perPlugin || !Array.isArray(metrics.perPlugin)) {
7001
- return;
7002
- }
7003
- for (const plugin of metrics.perPlugin) {
7004
- if (!plugin.pluginId || plugin.requests < 5) {
7005
- continue;
7006
- }
7007
- const errorRate = plugin.requests > 0 ? (plugin.errors ?? 0) / plugin.requests * 100 : 0;
7008
- const key = `plugin:${plugin.pluginId}`;
7009
- if (this.isInCooldown(key)) {
7010
- continue;
7011
- }
7012
- let severity = null;
7013
- let title = "";
7014
- let details = "";
7015
- if (errorRate >= this.config.thresholds.pluginErrorRateCritical) {
7016
- severity = "critical";
7017
- title = `Plugin Failure: ${plugin.pluginId}`;
7018
- details = `Plugin ${plugin.pluginId} has critical error rate of ${errorRate.toFixed(1)}% (${plugin.errors ?? 0}/${plugin.requests} requests). This exceeds the critical threshold of ${this.config.thresholds.pluginErrorRateCritical}%.`;
7019
- } else if (errorRate >= this.config.thresholds.pluginErrorRateWarning) {
7020
- severity = "warning";
7021
- title = `Plugin Issues: ${plugin.pluginId}`;
7022
- details = `Plugin ${plugin.pluginId} has elevated error rate of ${errorRate.toFixed(1)}% (${plugin.errors ?? 0}/${plugin.requests} requests).`;
7023
- }
7024
- if (severity) {
7025
- await this.createIncident({
7026
- type: "plugin_failure",
7027
- severity,
7028
- title,
7029
- details,
7030
- affectedServices: [plugin.pluginId],
7031
- metadata: {
7032
- pluginId: plugin.pluginId,
7033
- errorRate,
7034
- requests: plugin.requests,
7035
- errors: plugin.errors,
7036
- avgLatency: plugin.latency?.average
7037
- }
7038
- }, key, now);
7039
- }
7040
- }
7041
- }
7042
- /**
7043
- * Check if incident key is in cooldown
7044
- */
7045
- isInCooldown(key) {
7046
- return this.recentIncidents.some((ri) => ri.key === key);
7047
- }
7048
- /**
7049
- * Gather related data (logs, metrics, timeline) for incident
7050
- * @private
7051
- */
7052
- async gatherRelatedData(incidentType, timeWindow = 5 * 60 * 1e3) {
7053
- const now = Date.now();
7054
- const from = now - timeWindow;
7055
- const relatedData = {
7056
- timeline: []
7057
- };
7058
- try {
7059
- const [errorLogsResult, fatalLogsResult] = await Promise.all([
7060
- platform.logs.query({ level: "error", from, to: now }, { limit: 50 }),
7061
- platform.logs.query({ level: "fatal", from, to: now }, { limit: 50 })
7062
- ]);
7063
- const allErrorLogs = [...errorLogsResult.logs, ...fatalLogsResult.logs];
7064
- if (allErrorLogs.length > 0) {
7065
- allErrorLogs.sort((a, b) => b.timestamp - a.timestamp);
7066
- const uniqueErrors = /* @__PURE__ */ new Set();
7067
- const endpointErrorCount = /* @__PURE__ */ new Map();
7068
- for (const log of allErrorLogs) {
7069
- let errorMsg = typeof log.message === "string" ? log.message : JSON.stringify(log.message);
7070
- if (log.err) {
7071
- const err = log.err;
7072
- const stack = err.stack ? `
7073
- ${err.stack.split("\n").slice(0, 3).join("\n")}` : "";
7074
- errorMsg = `${err.message || errorMsg}${stack}`;
7075
- if (log.plugin) {
7076
- errorMsg = `[${log.plugin}] ${errorMsg}`;
7077
- }
7078
- if (log.command) {
7079
- errorMsg += `
7080
- Command: ${log.command}`;
7081
- }
7082
- }
7083
- if (errorMsg && uniqueErrors.size < 5) {
7084
- uniqueErrors.add(errorMsg.substring(0, 500));
7085
- }
7086
- const endpoint = log.endpoint || log.url || "unknown";
7087
- const existing = endpointErrorCount.get(endpoint);
7088
- if (existing) {
7089
- existing.count++;
7090
- } else {
7091
- endpointErrorCount.set(endpoint, {
7092
- count: 1,
7093
- sample: errorMsg.substring(0, 200)
7094
- });
7095
- }
7096
- }
7097
- const topEndpoints = Array.from(endpointErrorCount.entries()).sort((a, b) => b[1].count - a[1].count).slice(0, 5).map(([endpoint, data]) => ({
7098
- endpoint,
7099
- count: data.count,
7100
- sample: data.sample
7101
- }));
7102
- relatedData.logs = {
7103
- errorCount: allErrorLogs.filter((l) => l.level === "error").length,
7104
- warnCount: 0,
7105
- // Could query warnings separately if needed
7106
- timeRange: [from, now],
7107
- sampleErrors: Array.from(uniqueErrors),
7108
- topEndpoints: topEndpoints.length > 0 ? topEndpoints : void 0
7109
- };
7110
- for (const log of allErrorLogs.slice(0, 10)) {
7111
- const msg = typeof log.message === "string" ? log.message : "Error occurred";
7112
- const plugin = log.plugin ? `[${log.plugin}] ` : "";
7113
- relatedData.timeline.push({
7114
- timestamp: log.timestamp,
7115
- event: `${plugin}${msg.substring(0, 100)}`,
7116
- source: "logs"
7117
- });
7118
- }
7119
- }
7120
- } catch (error) {
7121
- this.log("warn", "Failed to gather related logs", {
7122
- error: error instanceof Error ? error.message : String(error)
7123
- });
7124
- }
7125
- const currentMetrics = metricsCollector.getMetrics();
7126
- const duringMetrics = {
7127
- errorRate: currentMetrics.requests.total > 0 ? (currentMetrics.requests.clientErrors + currentMetrics.requests.serverErrors) / currentMetrics.requests.total * 100 : 0,
7128
- avgLatency: currentMetrics.latency.average ?? 0,
7129
- totalRequests: currentMetrics.requests.total,
7130
- totalErrors: currentMetrics.requests.clientErrors + currentMetrics.requests.serverErrors
7131
- };
7132
- let beforeMetrics;
7133
- if (this.metricsHistory.length >= 2) {
7134
- const beforeSnapshots = this.metricsHistory.slice(0, -1);
7135
- if (beforeSnapshots.length > 0) {
7136
- const avgErrorRate = beforeSnapshots.reduce((sum, s) => sum + s.errorRate, 0) / beforeSnapshots.length;
7137
- const avgLatency = beforeSnapshots.reduce((sum, s) => sum + s.avgLatency, 0) / beforeSnapshots.length;
7138
- const avgRequests = beforeSnapshots.reduce((sum, s) => sum + s.totalRequests, 0) / beforeSnapshots.length;
7139
- beforeMetrics = {
7140
- errorRate: avgErrorRate,
7141
- avgLatency,
7142
- totalRequests: avgRequests
7143
- };
7144
- this.log("debug", "Calculated before metrics", {
7145
- before: beforeMetrics,
7146
- during: duringMetrics,
7147
- snapshotsUsed: beforeSnapshots.length
7148
- });
7149
- }
7150
- }
7151
- let topSlowest;
7152
- let affectedEndpoints;
7153
- if (incidentType === "latency_spike") {
7154
- const histogram = currentMetrics.latency.histogram;
7155
- if (histogram && histogram.length > 0) {
7156
- const slowestRoutes = histogram.filter((h) => h.max > 100).sort((a, b) => b.max - a.max).slice(0, 10);
7157
- topSlowest = slowestRoutes.map((h) => {
7158
- const [method, ...pathParts] = h.route.split(" ");
7159
- const endpoint = pathParts.join(" ");
7160
- const statusCodes = Object.keys(h.byStatus);
7161
- const mostCommonStatus = statusCodes.length > 0 ? parseInt(statusCodes[0], 10) : void 0;
7162
- return {
7163
- endpoint,
7164
- method: method || "GET",
7165
- durationMs: Math.round(h.max),
7166
- statusCode: mostCommonStatus
7167
- };
7168
- });
7169
- affectedEndpoints = [...new Set(slowestRoutes.map((h) => h.route))];
7170
- this.log("debug", "Collected slow requests for latency incident", {
7171
- topSlowestCount: topSlowest.length,
7172
- affectedEndpointsCount: affectedEndpoints.length
7173
- });
7174
- }
7175
- }
7176
- relatedData.metrics = {
7177
- before: beforeMetrics,
7178
- during: duringMetrics,
7179
- topSlowest,
7180
- affectedEndpoints
7181
- };
7182
- relatedData.timeline.unshift({
7183
- timestamp: now,
7184
- event: `Incident detected: ${incidentType}`,
7185
- source: "detector"
7186
- });
7187
- relatedData.timeline.sort((a, b) => b.timestamp - a.timestamp);
7188
- return relatedData;
7189
- }
7190
- /**
7191
- * Create incident and track it
7192
- */
7193
- async createIncident(payload, key, timestamp) {
7194
- try {
7195
- const relatedData = await this.gatherRelatedData(payload.type);
7196
- const enrichedPayload = {
7197
- ...payload,
7198
- relatedData
7199
- };
7200
- const incident = await this.incidentStorage.createIncident(enrichedPayload);
7201
- this.recentIncidents.push({
7202
- type: payload.type,
7203
- key,
7204
- timestamp
7205
- });
7206
- this.log("info", "Auto-created incident with context", {
7207
- id: incident.id,
7208
- type: incident.type,
7209
- severity: incident.severity,
7210
- title: incident.title,
7211
- errorLogsCount: relatedData.logs?.errorCount ?? 0,
7212
- timelineEventsCount: relatedData.timeline?.length ?? 0
7213
- });
7214
- if (platform.analytics) {
7215
- platform.analytics.track("incident.created", {
7216
- incidentId: incident.id,
7217
- type: incident.type,
7218
- severity: incident.severity,
7219
- source: "auto-detector",
7220
- errorLogsCount: relatedData.logs?.errorCount ?? 0,
7221
- timelineEventsCount: relatedData.timeline?.length ?? 0,
7222
- hasBeforeMetrics: !!relatedData.metrics?.before,
7223
- affectedServicesCount: payload.affectedServices?.length ?? 0
7224
- }).catch(() => {
7225
- });
7226
- }
7227
- } catch (error) {
7228
- this.log("error", "Failed to create incident", {
7229
- key,
7230
- error: error instanceof Error ? error.message : String(error)
7231
- });
7232
- }
7233
- }
7234
- /**
7235
- * Get current detector status
7236
- */
7237
- getStatus() {
7238
- return {
7239
- running: this.isRunning,
7240
- config: this.config,
7241
- recentIncidentsCount: this.recentIncidents.length
7242
- };
7243
- }
7244
- /**
7245
- * Update thresholds at runtime
7246
- */
7247
- updateThresholds(thresholds) {
7248
- this.config.thresholds = { ...this.config.thresholds, ...thresholds };
7249
- this.log("info", "Thresholds updated", { thresholds: this.config.thresholds });
7250
- }
7251
- log(level, message, meta) {
7252
- if (level === "debug" && !this.config.debug) {
7253
- return;
7254
- }
7255
- const prefix = "[IncidentDetector]";
7256
- if (this.logger[level]) {
7257
- if (meta) {
7258
- this.logger[level]({ ...meta }, `${prefix} ${message}`);
7259
- } else {
7260
- this.logger[level](`${prefix} ${message}`);
7261
- }
7262
- } else {
7263
- console.log(`${prefix} [${level}] ${message}`, meta ?? "");
7264
- }
7265
- }
7266
- };
7267
5642
 
7268
5643
  // src/routes/index.ts
7269
5644
  function normalizeBasePath2(basePath) {
@@ -7459,71 +5834,7 @@ async function registerRoutes(server, config, repoRoot, registry) {
7459
5834
  historicalCollector.stop();
7460
5835
  platform.logger.info("Historical metrics collector stopped");
7461
5836
  });
7462
- let incidentStorage;
7463
- const db = platform.getAdapter("db");
7464
- if (db) {
7465
- try {
7466
- const { readFileSync: readFileSync2 } = await import('fs');
7467
- const { fileURLToPath } = await import('url');
7468
- const { dirname: dirname2, join: join3 } = await import('path');
7469
- const __filename = fileURLToPath(import.meta.url);
7470
- const __dirname = dirname2(__filename);
7471
- const schemaPath = join3(__dirname, "..", "services", "incident-schema.sql");
7472
- const schema = readFileSync2(schemaPath, "utf-8");
7473
- if ("exec" in db && typeof db.exec === "function") {
7474
- await db.exec(schema);
7475
- } else {
7476
- const statements = schema.split(";").map((s) => s.trim()).filter((s) => s.length > 0 && !s.startsWith("--"));
7477
- for (const statement of statements) {
7478
- if (statement) {
7479
- await db.query(statement);
7480
- }
7481
- }
7482
- }
7483
- platform.logger.info("Incidents schema initialized");
7484
- incidentStorage = new IncidentStorage(
7485
- db,
7486
- { debug: process.env.NODE_ENV !== "production" },
7487
- platformServices.logger
7488
- );
7489
- platform.logger.info("Incident storage initialized");
7490
- } catch (err) {
7491
- const error = err instanceof Error ? err : new Error(String(err));
7492
- platform.logger.warn("Failed to initialize incident storage, continuing without it", { error: error.message });
7493
- }
7494
- } else {
7495
- platform.logger.warn("Database adapter (db) not configured \u2014 incident storage disabled. Configure db adapter in kb.config.json to enable.");
7496
- }
7497
- if (incidentStorage) {
7498
- const incidentDetector = new IncidentDetector(
7499
- incidentStorage,
7500
- {
7501
- intervalMs: 3e4,
7502
- // Check every 30 seconds
7503
- cooldownMs: 5 * 60 * 1e3,
7504
- // 5 minute cooldown between same incidents
7505
- debug: process.env.NODE_ENV !== "production",
7506
- thresholds: {
7507
- errorRateWarning: 5,
7508
- errorRateCritical: 10,
7509
- latencyP99Warning: 2e3,
7510
- latencyP99Critical: 5e3,
7511
- latencyP95Warning: 1e3,
7512
- minRequestsForDetection: 10,
7513
- pluginErrorRateWarning: 10,
7514
- pluginErrorRateCritical: 25
7515
- }
7516
- },
7517
- platformServices.logger
7518
- );
7519
- incidentDetector.start();
7520
- platform.logger.info("Incident detector started");
7521
- server.addHook("onClose", async () => {
7522
- incidentDetector.stop();
7523
- platform.logger.info("Incident detector stopped");
7524
- });
7525
- }
7526
- await registerObservabilityRoutes(server, config, repoRoot, historicalCollector, incidentStorage, platformServices);
5837
+ await registerObservabilityRoutes(server, config, repoRoot, historicalCollector, platformServices);
7527
5838
  await registerAnalyticsRoutes(server, config);
7528
5839
  await registerAdaptersRoutes(server, config);
7529
5840
  await registerLogRoutes(server);
@@ -7614,6 +5925,9 @@ function registerEnvelopeMiddleware(server, config) {
7614
5925
  if (parsedPayload && typeof parsedPayload === "object" && "ok" in parsedPayload) {
7615
5926
  return payload;
7616
5927
  }
5928
+ if (reply.getHeader("x-openapi-spec")) {
5929
+ return payload;
5930
+ }
7617
5931
  let dataToWrap = parsedPayload;
7618
5932
  if (parsedPayload === null || parsedPayload === void 0) {
7619
5933
  dataToWrap = null;