@kb-labs/rest-api-app 2.14.0 → 2.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -21,8 +21,6 @@ import { WORKFLOW_REDIS_CHANNEL } from '@kb-labs/workflow-constants';
21
21
  import { validatorCompiler, serializerCompiler } from 'fastify-type-provider-zod';
22
22
  import { z } from 'zod';
23
23
  import { errorEnvelopeSchema, ErrorCode, ErrorResponseSchema, JobsListResponseSchema, ListJobsQuerySchema, JobStatsResponseSchema, JobResponseSchema, JobActionResponseSchema } from '@kb-labs/rest-api-contracts';
24
- import { exec } from 'child_process';
25
- import { promisify } from 'util';
26
24
  import { readKbConfig } from '@kb-labs/core-config';
27
25
  import { AdapterNameSchema } from '@kb-labs/gateway-contracts';
28
26
  import fastifyCors from '@fastify/cors';
@@ -2694,8 +2692,9 @@ async function registerJobsRoutes(server, config, cronManager) {
2694
2692
  throw createError(503, "CRON_UNAVAILABLE", "Cron manager not available");
2695
2693
  }
2696
2694
  let jobs = cronManager.list();
2697
- if (request.query.status) {
2698
- jobs = jobs.filter((job) => job.status === request.query.status);
2695
+ const query = request.query;
2696
+ if (query.status) {
2697
+ jobs = jobs.filter((job) => job.status === query.status);
2699
2698
  }
2700
2699
  return reply.send({ jobs });
2701
2700
  }
@@ -2900,261 +2899,15 @@ async function registerCacheRoutes(server, config, registry) {
2900
2899
  }
2901
2900
  });
2902
2901
  }
2903
- var DEFAULT_CONFIG = {
2904
- model: "gpt-4",
2905
- temperature: 0.3,
2906
- maxTokens: 1500,
2907
- debug: false
2908
- };
2909
- var IncidentAnalyzer = class {
2910
- config;
2911
- logger;
2912
- constructor(config = {}, logger = console) {
2913
- this.config = { ...DEFAULT_CONFIG, ...config };
2914
- this.logger = logger;
2915
- }
2916
- /**
2917
- * Analyze incident using LLM
2918
- *
2919
- * @param incident - Incident to analyze
2920
- * @returns AI analysis result
2921
- */
2922
- async analyze(incident) {
2923
- this.log("info", "Analyzing incident", { id: incident.id, type: incident.type });
2924
- const prompt = this.buildAnalysisPrompt(incident);
2925
- try {
2926
- const result = await platform.llm.complete(prompt, {
2927
- model: this.config.model,
2928
- temperature: this.config.temperature,
2929
- maxTokens: this.config.maxTokens,
2930
- systemPrompt: this.getSystemPrompt()
2931
- });
2932
- this.log("debug", "LLM response received", {
2933
- length: result.content.length,
2934
- tokensUsed: result.usage ? result.usage.promptTokens + result.usage.completionTokens : void 0
2935
- });
2936
- const analysis = this.parseAnalysisResponse(result.content);
2937
- this.log("info", "Incident analysis complete", {
2938
- id: incident.id,
2939
- rootCausesCount: analysis.rootCauses.length,
2940
- recommendationsCount: analysis.recommendations.length
2941
- });
2942
- return analysis;
2943
- } catch (error) {
2944
- this.log("error", "Incident analysis failed", {
2945
- id: incident.id,
2946
- error: error instanceof Error ? error.message : String(error)
2947
- });
2948
- return this.getFallbackAnalysis(incident);
2949
- }
2950
- }
2951
- /**
2952
- * Build analysis prompt from incident data
2953
- * @private
2954
- */
2955
- buildAnalysisPrompt(incident) {
2956
- let prompt = `# Incident Analysis Request
2957
-
2958
- `;
2959
- prompt += `## Incident Overview
2960
- `;
2961
- prompt += `- **ID**: ${incident.id}
2962
- `;
2963
- prompt += `- **Type**: ${incident.type}
2964
- `;
2965
- prompt += `- **Severity**: ${incident.severity}
2966
- `;
2967
- prompt += `- **Title**: ${incident.title}
2968
- `;
2969
- prompt += `- **Time**: ${new Date(incident.timestamp).toISOString()}
2970
-
2971
- `;
2972
- prompt += `## Description
2973
- ${incident.details}
2974
-
2975
- `;
2976
- if (incident.metadata && Object.keys(incident.metadata).length > 0) {
2977
- prompt += `## Metrics
2978
- `;
2979
- prompt += "```json\n";
2980
- prompt += JSON.stringify(incident.metadata, null, 2);
2981
- prompt += "\n```\n\n";
2982
- }
2983
- if (incident.relatedData?.logs && incident.relatedData.logs.sampleErrors.length > 0) {
2984
- prompt += `## Related Error Logs (${incident.relatedData.logs.errorCount} total)
2985
- `;
2986
- incident.relatedData.logs.sampleErrors.forEach((error, idx) => {
2987
- prompt += `${idx + 1}. ${error}
2988
- `;
2989
- });
2990
- prompt += "\n";
2991
- }
2992
- if (incident.relatedData?.timeline && incident.relatedData.timeline.length > 0) {
2993
- prompt += `## Event Timeline
2994
- `;
2995
- incident.relatedData.timeline.slice(0, 10).forEach((event) => {
2996
- const time = new Date(event.timestamp).toISOString();
2997
- prompt += `- [${time}] (${event.source}) ${event.event}
2998
- `;
2999
- });
3000
- prompt += "\n";
3001
- }
3002
- if (incident.relatedData?.metrics) {
3003
- prompt += `## System Metrics
3004
- `;
3005
- prompt += "```json\n";
3006
- prompt += JSON.stringify(incident.relatedData.metrics, null, 2);
3007
- prompt += "\n```\n\n";
3008
- }
3009
- prompt += `## Analysis Instructions
3010
- `;
3011
- prompt += `Please analyze this incident and provide:
3012
- `;
3013
- prompt += `1. A brief executive summary (2-3 sentences)
3014
- `;
3015
- prompt += `2. Root causes with confidence scores (0.0-1.0) and evidence
3016
- `;
3017
- prompt += `3. Patterns or trends you observe
3018
- `;
3019
- prompt += `4. Actionable recommendations for resolution and prevention
3020
-
3021
- `;
3022
- prompt += `Format your response as JSON:
3023
- `;
3024
- prompt += "```json\n";
3025
- prompt += `{
3026
- `;
3027
- prompt += ` "summary": "Executive summary here",
3028
- `;
3029
- prompt += ` "rootCauses": [
3030
- `;
3031
- prompt += ` {
3032
- `;
3033
- prompt += ` "factor": "Root cause description",
3034
- `;
3035
- prompt += ` "confidence": 0.85,
3036
- `;
3037
- prompt += ` "evidence": "Supporting evidence from logs/metrics"
3038
- `;
3039
- prompt += ` }
3040
- `;
3041
- prompt += ` ],
3042
- `;
3043
- prompt += ` "patterns": ["Pattern 1", "Pattern 2"],
3044
- `;
3045
- prompt += ` "recommendations": ["Recommendation 1", "Recommendation 2"]
3046
- `;
3047
- prompt += `}
3048
- `;
3049
- prompt += "```";
3050
- return prompt;
3051
- }
3052
- /**
3053
- * Get system prompt for incident analysis
3054
- * @private
3055
- */
3056
- getSystemPrompt() {
3057
- return `You are an expert SRE (Site Reliability Engineer) and incident response specialist.
3058
- Your role is to analyze system incidents, identify root causes, and provide actionable recommendations.
3059
-
3060
- Guidelines:
3061
- - Base your analysis on the actual data provided (logs, metrics, timeline)
3062
- - Provide confidence scores based on available evidence (0.0 = no evidence, 1.0 = certain)
3063
- - Focus on actionable recommendations that can prevent future incidents
3064
- - Be concise but thorough - this is for production systems
3065
- - Always respond with valid JSON matching the requested format`;
3066
- }
3067
- /**
3068
- * Parse LLM response into structured analysis
3069
- * @private
3070
- */
3071
- parseAnalysisResponse(response) {
3072
- try {
3073
- const jsonMatch = response.match(/```json\s*([\s\S]*?)\s*```/) ?? response.match(/```\s*([\s\S]*?)\s*```/) ?? [null, response];
3074
- const jsonStr = jsonMatch[1] || response;
3075
- const parsed = JSON.parse(jsonStr.trim());
3076
- return {
3077
- summary: parsed.summary || "No summary provided",
3078
- rootCauses: Array.isArray(parsed.rootCauses) ? parsed.rootCauses : [],
3079
- patterns: Array.isArray(parsed.patterns) ? parsed.patterns : [],
3080
- recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations : [],
3081
- analyzedAt: Date.now()
3082
- };
3083
- } catch (error) {
3084
- this.log("warn", "Failed to parse LLM response, using fallback", {
3085
- error: error instanceof Error ? error.message : String(error)
3086
- });
3087
- return {
3088
- summary: response.substring(0, 200),
3089
- rootCauses: [],
3090
- patterns: [],
3091
- recommendations: [],
3092
- analyzedAt: Date.now()
3093
- };
3094
- }
3095
- }
3096
- /**
3097
- * Get fallback analysis when LLM is unavailable
3098
- * @private
3099
- */
3100
- getFallbackAnalysis(incident) {
3101
- const errorCount = incident.relatedData?.logs?.errorCount ?? 0;
3102
- const sampleErrors = incident.relatedData?.logs?.sampleErrors ?? [];
3103
- return {
3104
- summary: `${incident.severity} ${incident.type} incident detected. ${errorCount} errors logged.`,
3105
- rootCauses: [
3106
- {
3107
- factor: incident.title,
3108
- confidence: 0.5,
3109
- evidence: incident.details
3110
- }
3111
- ],
3112
- patterns: sampleErrors.length > 0 ? [`Multiple error types detected: ${sampleErrors.length} unique errors`] : [],
3113
- recommendations: [
3114
- "Review related error logs for stack traces",
3115
- "Check system metrics during incident timeframe",
3116
- "Verify external dependencies are operational"
3117
- ],
3118
- analyzedAt: Date.now()
3119
- };
3120
- }
3121
- log(level, message, meta) {
3122
- if (level === "debug" && !this.config.debug) {
3123
- return;
3124
- }
3125
- const prefix = "[IncidentAnalyzer]";
3126
- if (this.logger[level]) {
3127
- if (meta) {
3128
- this.logger[level]({ ...meta }, `${prefix} ${message}`);
3129
- } else {
3130
- this.logger[level](`${prefix} ${message}`);
3131
- }
3132
- } else {
3133
- console.log(`${prefix} [${level}] ${message}`, meta ?? "");
3134
- }
3135
- }
3136
- };
3137
-
3138
- // src/routes/observability.ts
3139
- var execAsync = promisify(exec);
3140
- var DEVKIT_CACHE_KEY = "observability:devkit-health";
3141
- var DEVKIT_CACHE_TTL_MS = 10 * 60 * 1e3;
3142
- async function registerObservabilityRoutes(fastify, config, repoRoot, historicalMetrics, incidentStorage, platform19) {
2902
+ async function registerObservabilityRoutes(fastify, config, repoRoot, historicalMetrics, platform16) {
3143
2903
  const basePath = normalizeBasePath(config.basePath);
3144
2904
  const stateBrokerPaths = resolvePaths(basePath, "/observability/state-broker");
3145
- const devkitPaths = resolvePaths(basePath, "/observability/devkit");
3146
2905
  const systemMetricsPaths = resolvePaths(basePath, "/observability/system-metrics");
3147
2906
  const metricsHistoryPaths = resolvePaths(basePath, "/observability/metrics/history");
3148
2907
  const metricsHeatmapPaths = resolvePaths(basePath, "/observability/metrics/heatmap");
3149
- const incidentsListPaths = resolvePaths(basePath, "/observability/incidents");
3150
- const incidentsDetailPaths = resolvePaths(basePath, "/observability/incidents/:id");
3151
- const incidentsCreatePaths = resolvePaths(basePath, "/observability/incidents");
3152
- const incidentsHistoryPaths = resolvePaths(basePath, "/observability/incidents/history");
3153
- const incidentsResolvePaths = resolvePaths(basePath, "/observability/incidents/:id/resolve");
3154
- const incidentsAnalyzePaths = resolvePaths(basePath, "/observability/incidents/:id/analyze");
3155
2908
  const insightsChatPaths = resolvePaths(basePath, "/observability/insights/chat");
3156
2909
  for (const path3 of stateBrokerPaths) {
3157
- fastify.get(path3, async (_request, reply) => {
2910
+ fastify.get(path3, { schema: { tags: ["Observability"], summary: "State Broker statistics" } }, async (_request, reply) => {
3158
2911
  try {
3159
2912
  const stateBrokerUrl = process.env.KB_STATE_DAEMON_URL || "http://localhost:7777";
3160
2913
  fastify.log.debug({ url: stateBrokerUrl }, "Fetching State Broker stats");
@@ -3193,7 +2946,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3193
2946
  }
3194
2947
  };
3195
2948
  } catch (error) {
3196
- platform19?.logger.error("Failed to fetch State Broker stats", error instanceof Error ? error : new Error(String(error)));
2949
+ platform16?.logger.error("Failed to fetch State Broker stats", error instanceof Error ? error : new Error(String(error)));
3197
2950
  const isTimeout = error instanceof Error && error.name === "AbortError";
3198
2951
  return reply.code(503).send({
3199
2952
  ok: false,
@@ -3208,105 +2961,10 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3208
2961
  }
3209
2962
  });
3210
2963
  }
3211
- for (const path3 of devkitPaths) {
3212
- fastify.get(path3, async (_request, reply) => {
3213
- const now = Date.now();
3214
- if (platform19?.cache) {
3215
- try {
3216
- const cached = await platform19.cache.get(DEVKIT_CACHE_KEY);
3217
- if (cached && now < cached.expiresAt) {
3218
- const remainingTtl = Math.round((cached.expiresAt - now) / 1e3);
3219
- fastify.log.debug({ remainingTtl }, "Returning cached DevKit health from platform.cache");
3220
- return {
3221
- ok: true,
3222
- data: cached.data,
3223
- meta: {
3224
- source: "devkit-cli",
3225
- repoRoot,
3226
- cached: true,
3227
- cachedAt: cached.timestamp,
3228
- expiresAt: cached.expiresAt,
3229
- ttlSeconds: remainingTtl
3230
- }
3231
- };
3232
- }
3233
- } catch (cacheError) {
3234
- fastify.log.warn({ err: cacheError }, "Failed to read from platform.cache, proceeding without cache");
3235
- }
3236
- }
3237
- try {
3238
- fastify.log.debug({ cwd: repoRoot }, "Executing DevKit health check (cache miss)");
3239
- const { stdout, stderr } = await execAsync("npx kb-devkit-health --json", {
3240
- cwd: repoRoot,
3241
- timeout: 3e4,
3242
- // 30s timeout
3243
- env: {
3244
- ...process.env,
3245
- // Ensure DevKit runs in non-interactive mode
3246
- CI: "true"
3247
- }
3248
- });
3249
- if (stderr) {
3250
- fastify.log.warn({ stderr }, "DevKit health check produced warnings");
3251
- }
3252
- const health = JSON.parse(stdout);
3253
- const expiresAt = now + DEVKIT_CACHE_TTL_MS;
3254
- if (platform19?.cache) {
3255
- try {
3256
- await platform19.cache.set(DEVKIT_CACHE_KEY, {
3257
- data: health,
3258
- timestamp: now,
3259
- expiresAt
3260
- }, DEVKIT_CACHE_TTL_MS);
3261
- fastify.log.debug({
3262
- healthScore: health.healthScore,
3263
- grade: health.grade,
3264
- cachedUntil: new Date(expiresAt).toISOString()
3265
- }, "DevKit health check completed and cached in platform.cache");
3266
- } catch (cacheError) {
3267
- fastify.log.warn({ err: cacheError }, "Failed to write to platform.cache");
3268
- }
3269
- }
3270
- return {
3271
- ok: true,
3272
- data: health,
3273
- meta: {
3274
- source: "devkit-cli",
3275
- repoRoot,
3276
- command: "npx kb-devkit-health --json",
3277
- cached: false,
3278
- cachedAt: now,
3279
- expiresAt,
3280
- ttlSeconds: DEVKIT_CACHE_TTL_MS / 1e3
3281
- }
3282
- };
3283
- } catch (error) {
3284
- let partialData = null;
3285
- if (error && typeof error === "object" && "stdout" in error) {
3286
- try {
3287
- partialData = JSON.parse(error.stdout);
3288
- } catch {
3289
- partialData = null;
3290
- }
3291
- }
3292
- platform19?.logger.error("Failed to execute DevKit health check", error instanceof Error ? error : new Error(String(error)));
3293
- return reply.code(500).send({
3294
- ok: false,
3295
- error: {
3296
- code: "DEVKIT_ERROR",
3297
- message: error instanceof Error ? error.message : "Failed to execute DevKit health check",
3298
- details: {
3299
- partialData
3300
- }
3301
- }
3302
- });
3303
- }
3304
- });
3305
- }
3306
2964
  for (const path3 of systemMetricsPaths) {
3307
- fastify.get(path3, async (_request, reply) => {
2965
+ fastify.get(path3, { schema: { tags: ["Observability"], summary: "System resource metrics" } }, async (_request, reply) => {
3308
2966
  try {
3309
- if (!platform19?.cache) {
2967
+ if (!platform16?.cache) {
3310
2968
  return reply.code(503).send({
3311
2969
  ok: false,
3312
2970
  error: {
@@ -3318,17 +2976,17 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3318
2976
  fastify.log.debug("Fetching system metrics from all instances");
3319
2977
  const allMetrics = [];
3320
2978
  try {
3321
- if ("scan" in platform19.cache && typeof platform19.cache.scan === "function") {
3322
- const keys = await platform19.cache.scan("system-metrics:*");
2979
+ if ("scan" in platform16.cache && typeof platform16.cache.scan === "function") {
2980
+ const keys = await platform16.cache.scan("system-metrics:*");
3323
2981
  for (const key of keys) {
3324
- const metrics = await platform19.cache.get(key);
2982
+ const metrics = await platform16.cache.get(key);
3325
2983
  if (metrics) {
3326
2984
  allMetrics.push(metrics);
3327
2985
  }
3328
2986
  }
3329
2987
  } else {
3330
2988
  const currentInstanceId = hostname();
3331
- const metrics = await platform19.cache.get(`system-metrics:${currentInstanceId}`);
2989
+ const metrics = await platform16.cache.get(`system-metrics:${currentInstanceId}`);
3332
2990
  if (metrics) {
3333
2991
  allMetrics.push(metrics);
3334
2992
  }
@@ -3337,7 +2995,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3337
2995
  } catch (scanError) {
3338
2996
  fastify.log.warn({ err: scanError }, "Failed to scan platform.cache for system metrics");
3339
2997
  const currentInstanceId = hostname();
3340
- const metrics = await platform19.cache.get(`system-metrics:${currentInstanceId}`);
2998
+ const metrics = await platform16.cache.get(`system-metrics:${currentInstanceId}`);
3341
2999
  if (metrics) {
3342
3000
  allMetrics.push(metrics);
3343
3001
  }
@@ -3385,7 +3043,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3385
3043
  }
3386
3044
  };
3387
3045
  } catch (error) {
3388
- platform19?.logger.error("Failed to fetch system metrics", error instanceof Error ? error : new Error(String(error)));
3046
+ platform16?.logger.error("Failed to fetch system metrics", error instanceof Error ? error : new Error(String(error)));
3389
3047
  return reply.code(500).send({
3390
3048
  ok: false,
3391
3049
  error: {
@@ -3399,6 +3057,8 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3399
3057
  for (const path3 of metricsHistoryPaths) {
3400
3058
  fastify.get(path3, {
3401
3059
  schema: {
3060
+ tags: ["Observability"],
3061
+ summary: "Historical time-series metrics",
3402
3062
  querystring: {
3403
3063
  type: "object",
3404
3064
  properties: {
@@ -3481,7 +3141,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3481
3141
  }
3482
3142
  };
3483
3143
  } catch (error) {
3484
- platform19?.logger.error("Failed to query historical metrics", error instanceof Error ? error : new Error(String(error)), { query });
3144
+ platform16?.logger.error("Failed to query historical metrics", error instanceof Error ? error : new Error(String(error)), { query });
3485
3145
  return reply.code(500).send({
3486
3146
  ok: false,
3487
3147
  error: {
@@ -3495,6 +3155,8 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3495
3155
  for (const path3 of metricsHeatmapPaths) {
3496
3156
  fastify.get(path3, {
3497
3157
  schema: {
3158
+ tags: ["Observability"],
3159
+ summary: "Metrics heatmap data",
3498
3160
  querystring: {
3499
3161
  type: "object",
3500
3162
  properties: {
@@ -3570,7 +3232,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3570
3232
  }
3571
3233
  };
3572
3234
  } catch (error) {
3573
- platform19?.logger.error("Failed to query heatmap data", error instanceof Error ? error : new Error(String(error)), { query });
3235
+ platform16?.logger.error("Failed to query heatmap data", error instanceof Error ? error : new Error(String(error)), { query });
3574
3236
  return reply.code(500).send({
3575
3237
  ok: false,
3576
3238
  error: {
@@ -3581,310 +3243,11 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3581
3243
  }
3582
3244
  });
3583
3245
  }
3584
- for (const path3 of incidentsListPaths) {
3585
- fastify.get(path3, {
3586
- schema: {
3587
- querystring: {
3588
- type: "object",
3589
- properties: {
3590
- limit: { type: "integer", default: 50 },
3591
- severity: {
3592
- type: "string",
3593
- enum: ["critical", "warning", "info"]
3594
- },
3595
- type: {
3596
- type: "string",
3597
- enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
3598
- },
3599
- from: { type: "integer" },
3600
- to: { type: "integer" },
3601
- includeResolved: { type: "boolean", default: false }
3602
- }
3603
- }
3604
- }
3605
- }, async (request, reply) => {
3606
- if (!incidentStorage) {
3607
- return reply.code(503).send({
3608
- ok: false,
3609
- error: {
3610
- code: "INCIDENTS_NOT_CONFIGURED",
3611
- message: "Incident storage is not configured"
3612
- }
3613
- });
3614
- }
3615
- try {
3616
- const query = request.query;
3617
- const incidents = await incidentStorage.queryIncidents({
3618
- limit: query.limit ?? 50,
3619
- severity: query.severity,
3620
- type: query.type,
3621
- from: query.from,
3622
- to: query.to,
3623
- includeResolved: query.includeResolved ?? false
3624
- });
3625
- const stats = await incidentStorage.getStats();
3626
- return {
3627
- ok: true,
3628
- data: {
3629
- incidents,
3630
- summary: {
3631
- total: stats.total,
3632
- unresolved: stats.unresolved,
3633
- bySeverity: stats.bySeverity,
3634
- showing: incidents.length
3635
- }
3636
- }
3637
- };
3638
- } catch (error) {
3639
- fastify.log.error({ err: error }, "Failed to list incidents");
3640
- return reply.code(500).send({
3641
- ok: false,
3642
- error: {
3643
- code: "INCIDENTS_LIST_ERROR",
3644
- message: error instanceof Error ? error.message : "Failed to list incidents"
3645
- }
3646
- });
3647
- }
3648
- });
3649
- }
3650
- for (const path3 of incidentsDetailPaths) {
3651
- fastify.get(path3, {
3652
- schema: {
3653
- params: {
3654
- type: "object",
3655
- properties: {
3656
- id: { type: "string" }
3657
- },
3658
- required: ["id"]
3659
- }
3660
- }
3661
- }, async (request, reply) => {
3662
- if (!incidentStorage) {
3663
- return reply.code(503).send({
3664
- ok: false,
3665
- error: {
3666
- code: "INCIDENTS_NOT_CONFIGURED",
3667
- message: "Incident storage is not configured"
3668
- }
3669
- });
3670
- }
3671
- const { id } = request.params;
3672
- try {
3673
- const incident = await incidentStorage.getIncident(id);
3674
- if (!incident) {
3675
- return reply.code(404).send({
3676
- ok: false,
3677
- error: {
3678
- code: "INCIDENT_NOT_FOUND",
3679
- message: `Incident ${id} not found`
3680
- }
3681
- });
3682
- }
3683
- if (platform19?.analytics) {
3684
- platform19.analytics.track("incident.viewed", {
3685
- incidentId: id,
3686
- type: incident.type,
3687
- severity: incident.severity,
3688
- isResolved: !!incident.resolvedAt,
3689
- hasAIAnalysis: !!incident.aiAnalysis
3690
- }).catch(() => {
3691
- });
3692
- }
3693
- return {
3694
- ok: true,
3695
- data: incident
3696
- };
3697
- } catch (error) {
3698
- fastify.log.error({ err: error }, "Failed to get incident");
3699
- return reply.code(500).send({
3700
- ok: false,
3701
- error: {
3702
- code: "INCIDENT_GET_ERROR",
3703
- message: error instanceof Error ? error.message : "Failed to get incident"
3704
- }
3705
- });
3706
- }
3707
- });
3708
- }
3709
- for (const path3 of incidentsCreatePaths) {
3710
- fastify.post(path3, {
3711
- schema: {
3712
- body: {
3713
- type: "object",
3714
- properties: {
3715
- type: {
3716
- type: "string",
3717
- enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
3718
- },
3719
- severity: {
3720
- type: "string",
3721
- enum: ["critical", "warning", "info"]
3722
- },
3723
- title: { type: "string" },
3724
- details: { type: "string" },
3725
- rootCause: { type: "string" },
3726
- affectedServices: {
3727
- type: "array",
3728
- items: { type: "string" }
3729
- },
3730
- timestamp: { type: "number" },
3731
- metadata: { type: "object" }
3732
- },
3733
- required: ["type", "severity", "title", "details"]
3734
- }
3735
- }
3736
- }, async (request, reply) => {
3737
- if (!incidentStorage) {
3738
- return reply.code(503).send({
3739
- ok: false,
3740
- error: {
3741
- code: "INCIDENT_STORAGE_UNAVAILABLE",
3742
- message: "Incident storage is not initialized"
3743
- }
3744
- });
3745
- }
3746
- try {
3747
- const payload = request.body;
3748
- const incident = await incidentStorage.createIncident(payload);
3749
- return {
3750
- ok: true,
3751
- data: incident,
3752
- meta: {
3753
- source: "incident-storage"
3754
- }
3755
- };
3756
- } catch (error) {
3757
- platform19?.logger.error("Failed to create incident", error instanceof Error ? error : new Error(String(error)));
3758
- return reply.code(500).send({
3759
- ok: false,
3760
- error: {
3761
- code: "INCIDENT_CREATE_ERROR",
3762
- message: error instanceof Error ? error.message : "Failed to create incident"
3763
- }
3764
- });
3765
- }
3766
- });
3767
- }
3768
- for (const path3 of incidentsHistoryPaths) {
3769
- fastify.get(path3, {
3770
- schema: {
3771
- querystring: {
3772
- type: "object",
3773
- properties: {
3774
- limit: { type: "integer", minimum: 1, maximum: 500 },
3775
- severity: { type: "string", enum: ["critical", "warning", "info"] },
3776
- type: {
3777
- type: "string",
3778
- enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
3779
- },
3780
- from: { type: "integer" },
3781
- to: { type: "integer" },
3782
- includeResolved: { type: "boolean" }
3783
- }
3784
- }
3785
- }
3786
- }, async (request, reply) => {
3787
- if (!incidentStorage) {
3788
- return reply.code(503).send({
3789
- ok: false,
3790
- error: {
3791
- code: "INCIDENT_STORAGE_UNAVAILABLE",
3792
- message: "Incident storage is not initialized"
3793
- }
3794
- });
3795
- }
3796
- try {
3797
- const query = request.query;
3798
- const incidents = await incidentStorage.queryIncidents({
3799
- limit: query.limit ?? 50,
3800
- severity: query.severity,
3801
- type: query.type,
3802
- from: query.from,
3803
- to: query.to,
3804
- includeResolved: query.includeResolved ?? false
3805
- });
3806
- return {
3807
- ok: true,
3808
- data: incidents,
3809
- meta: {
3810
- source: "incident-storage",
3811
- count: incidents.length
3812
- }
3813
- };
3814
- } catch (error) {
3815
- platform19?.logger.error("Failed to query incidents", error instanceof Error ? error : new Error(String(error)));
3816
- return reply.code(500).send({
3817
- ok: false,
3818
- error: {
3819
- code: "INCIDENT_QUERY_ERROR",
3820
- message: error instanceof Error ? error.message : "Failed to query incidents"
3821
- }
3822
- });
3823
- }
3824
- });
3825
- }
3826
- for (const path3 of incidentsResolvePaths) {
3827
- fastify.post(path3, {
3828
- schema: {
3829
- params: {
3830
- type: "object",
3831
- properties: {
3832
- id: { type: "string" }
3833
- },
3834
- required: ["id"]
3835
- },
3836
- body: {
3837
- type: "object",
3838
- properties: {
3839
- resolutionNotes: { type: "string" }
3840
- }
3841
- }
3842
- }
3843
- }, async (request, reply) => {
3844
- if (!incidentStorage) {
3845
- return reply.code(503).send({
3846
- ok: false,
3847
- error: {
3848
- code: "INCIDENT_STORAGE_UNAVAILABLE",
3849
- message: "Incident storage is not initialized"
3850
- }
3851
- });
3852
- }
3853
- try {
3854
- const { id } = request.params;
3855
- const body = request.body;
3856
- const incident = await incidentStorage.resolveIncident(id, body.resolutionNotes);
3857
- if (!incident) {
3858
- return reply.code(404).send({
3859
- ok: false,
3860
- error: {
3861
- code: "INCIDENT_NOT_FOUND",
3862
- message: `Incident with id ${id} not found`
3863
- }
3864
- });
3865
- }
3866
- return {
3867
- ok: true,
3868
- data: incident,
3869
- meta: {
3870
- source: "incident-storage"
3871
- }
3872
- };
3873
- } catch (error) {
3874
- platform19?.logger.error("Failed to resolve incident", error instanceof Error ? error : new Error(String(error)));
3875
- return reply.code(500).send({
3876
- ok: false,
3877
- error: {
3878
- code: "INCIDENT_RESOLVE_ERROR",
3879
- message: error instanceof Error ? error.message : "Failed to resolve incident"
3880
- }
3881
- });
3882
- }
3883
- });
3884
- }
3885
3246
  for (const path3 of insightsChatPaths) {
3886
3247
  fastify.post(path3, {
3887
3248
  schema: {
3249
+ tags: ["Observability"],
3250
+ summary: "AI-powered observability insights chat",
3888
3251
  body: {
3889
3252
  type: "object",
3890
3253
  properties: {
@@ -3893,7 +3256,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3893
3256
  type: "object",
3894
3257
  properties: {
3895
3258
  includeMetrics: { type: "boolean" },
3896
- includeIncidents: { type: "boolean" },
3897
3259
  includeHistory: { type: "boolean" },
3898
3260
  timeRange: { type: "string", enum: ["1h", "6h", "24h", "7d"] },
3899
3261
  plugins: { type: "array", items: { type: "string" } }
@@ -3904,7 +3266,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3904
3266
  }
3905
3267
  }
3906
3268
  }, async (request, reply) => {
3907
- if (!platform19?.llm) {
3269
+ if (!platform16?.llm) {
3908
3270
  return reply.code(503).send({
3909
3271
  ok: false,
3910
3272
  error: {
@@ -3917,7 +3279,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3917
3279
  const body = request.body;
3918
3280
  const contextConfig = {
3919
3281
  includeMetrics: body.context?.includeMetrics ?? true,
3920
- includeIncidents: body.context?.includeIncidents ?? true,
3921
3282
  includeHistory: body.context?.includeHistory ?? true,
3922
3283
  timeRange: body.context?.timeRange ?? "24h",
3923
3284
  plugins: body.context?.plugins ?? []
@@ -3958,26 +3319,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
3958
3319
  fastify.log.warn({ err: metricsError }, "Failed to build metrics context for insights");
3959
3320
  }
3960
3321
  }
3961
- if (contextConfig.includeIncidents && incidentStorage) {
3962
- try {
3963
- const incidents = await incidentStorage.queryIncidents({ limit: 10 });
3964
- if (incidents.length > 0) {
3965
- contextText += `
3966
- ## Recent Incidents (${incidents.length})
3967
- `;
3968
- for (const incident of incidents) {
3969
- contextText += `- [${incident.severity.toUpperCase()}] ${incident.title}
3970
- `;
3971
- if (incident.details) {
3972
- contextText += ` Details: ${incident.details.slice(0, 100)}${incident.details.length > 100 ? "..." : ""}
3973
- `;
3974
- }
3975
- }
3976
- }
3977
- } catch (incidentError) {
3978
- fastify.log.warn({ err: incidentError }, "Failed to fetch incidents for insights context");
3979
- }
3980
- }
3981
3322
  const prompt = `You are an AI assistant analyzing a software platform's observability data.
3982
3323
 
3983
3324
  ${contextText}
@@ -3991,15 +3332,15 @@ Provide a clear, actionable response based on the data above. Include:
3991
3332
 
3992
3333
  Be concise but thorough. Use markdown formatting.`;
3993
3334
  fastify.log.debug({ question: body.question, contextLength: contextText.length }, "Calling LLM for insights");
3994
- const result = await platform19.llm.complete(prompt, {
3335
+ const result = await platform16.llm.complete(prompt, {
3995
3336
  systemPrompt: "You are a DevOps and SRE expert assistant. Analyze system metrics and provide actionable insights. Be concise, technical, and helpful.",
3996
3337
  temperature: 0.7,
3997
3338
  maxTokens: 1e3
3998
3339
  });
3999
3340
  const totalTokens = result.usage.promptTokens + result.usage.completionTokens;
4000
3341
  fastify.log.debug({ tokensUsed: totalTokens }, "LLM response received for insights");
4001
- if (platform19.analytics) {
4002
- platform19.analytics.track("ai_insights.chat", {
3342
+ if (platform16.analytics) {
3343
+ platform16.analytics.track("ai_insights.chat", {
4003
3344
  questionLength: body.question.length,
4004
3345
  contextIncluded: Object.keys(contextConfig).filter((k) => contextConfig[k]),
4005
3346
  timeRange: contextConfig.timeRange,
@@ -4026,246 +3367,27 @@ Be concise but thorough. Use markdown formatting.`;
4026
3367
  source: "llm-insights",
4027
3368
  model: result.model
4028
3369
  }
4029
- };
4030
- } catch (error) {
4031
- platform19?.logger.error("Failed to generate insights", error instanceof Error ? error : new Error(String(error)));
4032
- if (platform19?.analytics) {
4033
- platform19.analytics.track("ai_insights.error", {
4034
- error: error instanceof Error ? error.message : "Unknown error",
4035
- questionLength: request.body?.question?.length ?? 0
4036
- }).catch(() => {
4037
- });
4038
- }
4039
- return reply.code(500).send({
4040
- ok: false,
4041
- error: {
4042
- code: "INSIGHTS_ERROR",
4043
- message: error instanceof Error ? error.message : "Failed to generate insights"
4044
- }
4045
- });
4046
- }
4047
- });
4048
- }
4049
- for (const path3 of incidentsAnalyzePaths) {
4050
- fastify.post(path3, async (request, reply) => {
4051
- if (!incidentStorage) {
4052
- return reply.code(503).send({
4053
- ok: false,
4054
- error: {
4055
- code: "INCIDENTS_NOT_CONFIGURED",
4056
- message: "Incident storage is not configured"
4057
- }
4058
- });
4059
- }
4060
- const { id } = request.params;
4061
- try {
4062
- const incident = await incidentStorage.getIncident(id);
4063
- if (!incident) {
4064
- return reply.code(404).send({
4065
- ok: false,
4066
- error: {
4067
- code: "INCIDENT_NOT_FOUND",
4068
- message: `Incident ${id} not found`
4069
- }
4070
- });
4071
- }
4072
- if (incident.aiAnalysis && incident.aiAnalyzedAt) {
4073
- const ageMs = Date.now() - incident.aiAnalyzedAt;
4074
- const ageMinutes = Math.floor(ageMs / 6e4);
4075
- if (ageMs < 60 * 60 * 1e3) {
4076
- return {
4077
- ok: true,
4078
- data: {
4079
- ...incident.aiAnalysis,
4080
- cached: true,
4081
- analyzedAt: incident.aiAnalyzedAt,
4082
- ageMinutes
4083
- }
4084
- };
4085
- }
4086
- }
4087
- const analyzer = new IncidentAnalyzer(
4088
- {
4089
- debug: process.env.NODE_ENV !== "production"
4090
- },
4091
- fastify.log
4092
- );
4093
- const analysis = await analyzer.analyze(incident);
4094
- await incidentStorage.updateAIAnalysis(id, analysis);
4095
- if (platform19?.analytics) {
4096
- platform19.analytics.track("incident.analyzed", {
4097
- incidentId: id,
4098
- type: incident.type,
4099
- severity: incident.severity,
4100
- rootCausesCount: analysis.rootCauses.length,
4101
- recommendationsCount: analysis.recommendations.length
4102
- }).catch(() => {
4103
- });
4104
- }
4105
- return {
4106
- ok: true,
4107
- data: {
4108
- ...analysis,
4109
- cached: false
4110
- }
4111
- };
4112
- } catch (error) {
4113
- fastify.log.error({ err: error }, "Failed to analyze incident");
4114
- if (platform19?.analytics) {
4115
- platform19.analytics.track("incident.analysis_error", {
4116
- incidentId: id,
4117
- error: error instanceof Error ? error.message : "Unknown error"
4118
- }).catch(() => {
4119
- });
4120
- }
4121
- return reply.code(500).send({
4122
- ok: false,
4123
- error: {
4124
- code: "ANALYSIS_ERROR",
4125
- message: error instanceof Error ? error.message : "Failed to analyze incident"
4126
- }
4127
- });
4128
- }
4129
- });
4130
- }
4131
- const testCreateIncidentPaths = resolvePaths(basePath, "/test/create-incident");
4132
- for (const path3 of testCreateIncidentPaths) {
4133
- fastify.post(path3, async (request, reply) => {
4134
- if (!incidentStorage) {
4135
- return reply.code(500).send({
4136
- ok: false,
4137
- error: { code: "INCIDENT_STORAGE_NOT_INITIALIZED", message: "Incident storage not initialized" }
4138
- });
4139
- }
4140
- try {
4141
- const {
4142
- type = "custom",
4143
- severity = "warning",
4144
- title = "Test Incident",
4145
- details,
4146
- relatedData
4147
- } = request.body;
4148
- const incident = await incidentStorage.createIncident({
4149
- type,
4150
- severity,
4151
- title,
4152
- details: details || "This is a test incident created manually for testing purposes.",
4153
- timestamp: Date.now(),
4154
- metadata: {
4155
- testMode: true,
4156
- createdVia: "test-endpoint"
4157
- },
4158
- relatedData: relatedData || {
4159
- timeline: [
4160
- {
4161
- timestamp: Date.now(),
4162
- event: "Test incident created via /test/create-incident",
4163
- source: "manual"
4164
- }
4165
- ]
4166
- }
4167
- });
4168
- fastify.log.info({ id: incident.id }, "Test incident created");
4169
- return { ok: true, data: incident };
4170
- } catch (error) {
4171
- fastify.log.error({ err: error }, "Failed to create test incident");
4172
- return reply.code(500).send({
4173
- ok: false,
4174
- error: {
4175
- code: "TEST_INCIDENT_CREATION_FAILED",
4176
- message: error instanceof Error ? error.message : "Failed to create test incident"
4177
- }
4178
- });
4179
- }
4180
- });
4181
- }
4182
- const testTriggerErrorsPaths = resolvePaths(basePath, "/test/trigger-errors");
4183
- for (const path3 of testTriggerErrorsPaths) {
4184
- fastify.get(path3, async (request, reply) => {
4185
- const { count = 10 } = request.query;
4186
- const errorCount = Math.min(Math.max(1, parseInt(count, 10) || 10), 100);
4187
- fastify.log.info(`Triggering ${errorCount} test errors`);
4188
- for (let i = 0; i < errorCount; i++) {
4189
- platform19?.logger.error(`Test error ${i + 1}/${errorCount}`, void 0, {
4190
- testMode: true,
4191
- errorNumber: i + 1,
4192
- totalErrors: errorCount
4193
- });
4194
- if (i === errorCount - 1) {
4195
- return reply.code(500).send({
4196
- ok: false,
4197
- error: {
4198
- code: "TEST_ERROR_TRIGGERED",
4199
- message: `Generated ${errorCount} test errors. Check /observability/incidents in ~30 seconds for auto-created incident.`
4200
- }
3370
+ };
3371
+ } catch (error) {
3372
+ platform16?.logger.error("Failed to generate insights", error instanceof Error ? error : new Error(String(error)));
3373
+ if (platform16?.analytics) {
3374
+ platform16.analytics.track("ai_insights.error", {
3375
+ error: error instanceof Error ? error.message : "Unknown error",
3376
+ questionLength: request.body?.question?.length ?? 0
3377
+ }).catch(() => {
4201
3378
  });
4202
3379
  }
4203
- }
4204
- return {
4205
- ok: true,
4206
- message: `Triggered ${errorCount} test errors. Incident should be auto-created in next detection cycle (~30s).`
4207
- };
4208
- });
4209
- }
4210
- const testSimulateLatencyPaths = resolvePaths(basePath, "/test/simulate-latency");
4211
- for (const path3 of testSimulateLatencyPaths) {
4212
- fastify.get(path3, async (request, reply) => {
4213
- const { delay = 2e3 } = request.query;
4214
- const delayMs = Math.min(Math.max(100, parseInt(delay, 10) || 2e3), 1e4);
4215
- fastify.log.info(`Simulating ${delayMs}ms latency`);
4216
- await new Promise((resolve3) => {
4217
- setTimeout(resolve3, delayMs);
4218
- });
4219
- return {
4220
- ok: true,
4221
- message: `Simulated ${delayMs}ms latency. Call this endpoint multiple times to trigger latency_spike incident.`,
4222
- actualDelay: delayMs
4223
- };
4224
- });
4225
- }
4226
- const testBulkErrorsPaths = resolvePaths(basePath, "/test/bulk-errors");
4227
- for (const path3 of testBulkErrorsPaths) {
4228
- fastify.post(path3, async (request, reply) => {
4229
- const { successCount = 10, errorCount = 50 } = request.body;
4230
- fastify.log.info({ successCount, errorCount }, "Generating bulk requests for incident testing");
4231
- for (let i = 0; i < successCount; i++) {
4232
- platform19?.logger.info(`Bulk test - success ${i + 1}/${successCount}`);
4233
- }
4234
- const testEndpoints = [
4235
- "POST /api/v1/test/endpoint-a",
4236
- "GET /api/v1/test/endpoint-b",
4237
- "PUT /api/v1/test/endpoint-c",
4238
- "DELETE /api/v1/test/endpoint-d",
4239
- "POST /api/v1/test/endpoint-e"
4240
- ];
4241
- for (let i = 0; i < errorCount; i++) {
4242
- const endpoint = testEndpoints[i % testEndpoints.length];
4243
- const errorTypes = [
4244
- "Connection timeout",
4245
- "Validation failed",
4246
- "Database query error",
4247
- "Authentication failed",
4248
- "Rate limit exceeded"
4249
- ];
4250
- const errorType = errorTypes[i % errorTypes.length];
4251
- platform19?.logger.error(`${errorType}: ${endpoint}`, void 0, {
4252
- testMode: true,
4253
- bulkTest: true,
4254
- errorIndex: i,
4255
- endpoint
3380
+ return reply.code(500).send({
3381
+ ok: false,
3382
+ error: {
3383
+ code: "INSIGHTS_ERROR",
3384
+ message: error instanceof Error ? error.message : "Failed to generate insights"
3385
+ }
4256
3386
  });
4257
3387
  }
4258
- const totalRequests = successCount + errorCount;
4259
- const errorRate = errorCount / totalRequests * 100;
4260
- return {
4261
- ok: true,
4262
- message: `Generated ${totalRequests} requests (${successCount} success, ${errorCount} errors)`,
4263
- errorRate: `${errorRate.toFixed(1)}%`,
4264
- note: "Incident should be auto-created in next detection cycle (~30s)"
4265
- };
4266
3388
  });
4267
3389
  }
4268
- fastify.log.info("Observability routes registered (including test endpoints)");
3390
+ fastify.log.info("Observability routes registered");
4269
3391
  }
4270
3392
  async function registerAnalyticsRoutes(fastify, config) {
4271
3393
  const basePath = normalizeBasePath(config.basePath);
@@ -5598,7 +4720,7 @@ function generateFallbackSummary(stats, logs) {
5598
4720
  return summary;
5599
4721
  }
5600
4722
  async function registerLogRoutes(server, config, eventHub) {
5601
- server.get("/api/v1/logs", async (request, reply) => {
4723
+ server.get("/api/v1/logs", { schema: { tags: ["Logs"], summary: "Query logs with filters" } }, async (request, reply) => {
5602
4724
  try {
5603
4725
  const limit = request.query.limit ? parseInt(request.query.limit, 10) : 100;
5604
4726
  const offset = request.query.offset ? parseInt(request.query.offset, 10) : 0;
@@ -5656,6 +4778,7 @@ async function registerLogRoutes(server, config, eventHub) {
5656
4778
  });
5657
4779
  server.get(
5658
4780
  "/api/v1/logs/:id",
4781
+ { schema: { tags: ["Logs"], summary: "Get log entry by ID" } },
5659
4782
  async (request, reply) => {
5660
4783
  try {
5661
4784
  const log = await platform.logs.getById(request.params.id);
@@ -5689,6 +4812,7 @@ async function registerLogRoutes(server, config, eventHub) {
5689
4812
  );
5690
4813
  server.get(
5691
4814
  "/api/v1/logs/:id/related",
4815
+ { schema: { tags: ["Logs"], summary: "Get logs related to a specific entry" } },
5692
4816
  async (request, reply) => {
5693
4817
  try {
5694
4818
  const log = await platform.logs.getById(request.params.id);
@@ -5717,7 +4841,7 @@ async function registerLogRoutes(server, config, eventHub) {
5717
4841
  }
5718
4842
  }
5719
4843
  );
5720
- server.get("/api/v1/logs/stream", async (request, reply) => {
4844
+ server.get("/api/v1/logs/stream", { schema: { hide: true } }, async (request, reply) => {
5721
4845
  const caps = platform.logs.getCapabilities();
5722
4846
  if (!caps.hasStreaming) {
5723
4847
  return reply.code(503).send({
@@ -5775,7 +4899,7 @@ async function registerLogRoutes(server, config, eventHub) {
5775
4899
  await new Promise(() => {
5776
4900
  });
5777
4901
  });
5778
- server.get("/api/v1/logs/stats", async (request, reply) => {
4902
+ server.get("/api/v1/logs/stats", { schema: { tags: ["Logs"], summary: "Get log storage statistics" } }, async (request, reply) => {
5779
4903
  try {
5780
4904
  const stats = await platform.logs.getStats();
5781
4905
  const caps = platform.logs.getCapabilities();
@@ -5805,7 +4929,7 @@ async function registerLogRoutes(server, config, eventHub) {
5805
4929
  });
5806
4930
  }
5807
4931
  });
5808
- server.post("/api/v1/logs/summarize", async (request, reply) => {
4932
+ server.post("/api/v1/logs/summarize", { schema: { tags: ["Logs"], summary: "AI-powered log summarization" } }, async (request, reply) => {
5809
4933
  const { timeRange, filters, groupBy, question, includeContext } = request.body;
5810
4934
  try {
5811
4935
  const query = {
@@ -6197,7 +5321,7 @@ async function registerDebugRoutes(fastify, config) {
6197
5321
  }
6198
5322
 
6199
5323
  // src/services/historical-metrics.ts
6200
- var DEFAULT_CONFIG2 = {
5324
+ var DEFAULT_CONFIG = {
6201
5325
  intervalMs: 5e3,
6202
5326
  maxPoints: {
6203
5327
  "1m": 12,
@@ -6216,7 +5340,7 @@ var HistoricalMetricsCollector = class {
6216
5340
  startTimeMs = Date.now();
6217
5341
  constructor(cache, config = {}, logger = console) {
6218
5342
  this.cache = cache;
6219
- this.config = { ...DEFAULT_CONFIG2, ...config, maxPoints: { ...DEFAULT_CONFIG2.maxPoints, ...config.maxPoints } };
5343
+ this.config = { ...DEFAULT_CONFIG, ...config, maxPoints: { ...DEFAULT_CONFIG.maxPoints, ...config.maxPoints } };
6220
5344
  this.logger = logger;
6221
5345
  }
6222
5346
  /**
@@ -6516,754 +5640,6 @@ var HistoricalMetricsCollector = class {
6516
5640
  }
6517
5641
  }
6518
5642
  };
6519
- var DEFAULT_CONFIG3 = {
6520
- ttlMs: 30 * 24 * 60 * 60 * 1e3,
6521
- // 30 days (not used with SQLite, kept for compatibility)
6522
- maxIncidents: 1e3,
6523
- // Not enforced with SQLite, use retention policy instead
6524
- debug: false
6525
- };
6526
- var IncidentStorage = class {
6527
- db;
6528
- config;
6529
- logger;
6530
- constructor(db, config = {}, logger = console) {
6531
- this.db = db;
6532
- this.config = { ...DEFAULT_CONFIG3, ...config };
6533
- this.logger = logger;
6534
- }
6535
- /**
6536
- * Create a new incident record
6537
- */
6538
- async createIncident(payload) {
6539
- const id = `inc-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
6540
- const timestamp = payload.timestamp ?? Date.now();
6541
- if (!payload.type || !payload.severity || !payload.title) {
6542
- throw new Error("Incident must have type, severity, and title");
6543
- }
6544
- const relatedLogsCount = payload.relatedData?.logs?.errorCount ?? 0;
6545
- const relatedLogsSample = payload.relatedData?.logs ? JSON.stringify(payload.relatedData.logs) : null;
6546
- const relatedMetrics = payload.relatedData?.metrics ? JSON.stringify(payload.relatedData.metrics) : null;
6547
- const timeline = payload.relatedData?.timeline ? JSON.stringify(payload.relatedData.timeline) : null;
6548
- await this.db.query(
6549
- `INSERT INTO incidents (
6550
- id, type, severity, title, details, timestamp,
6551
- resolved_at, resolution_notes,
6552
- affected_services, metadata,
6553
- related_logs_count, related_logs_sample, related_metrics, timeline
6554
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
6555
- [
6556
- id,
6557
- payload.type,
6558
- payload.severity,
6559
- payload.title,
6560
- payload.details,
6561
- timestamp,
6562
- payload.resolvedAt ?? null,
6563
- payload.resolutionNotes ?? null,
6564
- payload.affectedServices ? JSON.stringify(payload.affectedServices) : null,
6565
- payload.metadata ? JSON.stringify(payload.metadata) : null,
6566
- relatedLogsCount,
6567
- relatedLogsSample,
6568
- relatedMetrics,
6569
- timeline
6570
- ]
6571
- );
6572
- this.log("info", "Incident created", {
6573
- id,
6574
- type: payload.type,
6575
- severity: payload.severity
6576
- });
6577
- const created = await this.getIncident(id);
6578
- if (!created) {
6579
- throw new Error("Failed to retrieve created incident");
6580
- }
6581
- return created;
6582
- }
6583
- /**
6584
- * Get incident by ID
6585
- */
6586
- async getIncident(id) {
6587
- const result = await this.db.query(
6588
- "SELECT * FROM incidents WHERE id = ?",
6589
- [id]
6590
- );
6591
- if (result.rows.length === 0) {
6592
- return null;
6593
- }
6594
- return this.rowToIncident(result.rows[0]);
6595
- }
6596
- /**
6597
- * Query incidents with filters
6598
- */
6599
- async queryIncidents(options = {}) {
6600
- const {
6601
- limit = 50,
6602
- severity,
6603
- type,
6604
- from,
6605
- to,
6606
- includeResolved = false
6607
- } = options;
6608
- const conditions = [];
6609
- const params = [];
6610
- if (severity) {
6611
- const severityList = Array.isArray(severity) ? severity : [severity];
6612
- conditions.push(`severity IN (${severityList.map(() => "?").join(", ")})`);
6613
- params.push(...severityList);
6614
- }
6615
- if (type) {
6616
- const typeList = Array.isArray(type) ? type : [type];
6617
- conditions.push(`type IN (${typeList.map(() => "?").join(", ")})`);
6618
- params.push(...typeList);
6619
- }
6620
- if (from) {
6621
- conditions.push("timestamp >= ?");
6622
- params.push(from);
6623
- }
6624
- if (to) {
6625
- conditions.push("timestamp <= ?");
6626
- params.push(to);
6627
- }
6628
- if (!includeResolved) {
6629
- conditions.push("resolved_at IS NULL");
6630
- }
6631
- const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
6632
- const sql = `
6633
- SELECT * FROM incidents
6634
- ${whereClause}
6635
- ORDER BY timestamp DESC
6636
- LIMIT ?
6637
- `;
6638
- const result = await this.db.query(sql, [...params, limit]);
6639
- return result.rows.map((row) => this.rowToIncident(row));
6640
- }
6641
- /**
6642
- * Resolve an incident
6643
- */
6644
- async resolveIncident(id, resolutionNotes) {
6645
- const resolvedAt = Date.now();
6646
- const result = await this.db.query(
6647
- `UPDATE incidents
6648
- SET resolved_at = ?, resolution_notes = ?
6649
- WHERE id = ?`,
6650
- [resolvedAt, resolutionNotes ?? null, id]
6651
- );
6652
- if (result.rowCount === 0) {
6653
- return null;
6654
- }
6655
- this.log("info", "Incident resolved", { id, resolvedAt });
6656
- const incident = await this.getIncident(id);
6657
- if (incident && platform.analytics) {
6658
- const durationMs = resolvedAt - incident.timestamp;
6659
- const durationMinutes = Math.floor(durationMs / 6e4);
6660
- platform.analytics.track("incident.resolved", {
6661
- incidentId: id,
6662
- type: incident.type,
6663
- severity: incident.severity,
6664
- durationMs,
6665
- durationMinutes,
6666
- hasResolutionNotes: !!resolutionNotes,
6667
- wasAnalyzed: !!incident.aiAnalysis
6668
- }).catch(() => {
6669
- });
6670
- }
6671
- return incident;
6672
- }
6673
- /**
6674
- * Delete an incident
6675
- */
6676
- async deleteIncident(id) {
6677
- const incident = await this.getIncident(id);
6678
- const result = await this.db.query("DELETE FROM incidents WHERE id = ?", [id]);
6679
- const deleted = result.rowCount > 0;
6680
- if (deleted) {
6681
- this.log("info", "Incident deleted", { id });
6682
- if (incident && platform.analytics) {
6683
- platform.analytics.track("incident.deleted", {
6684
- incidentId: id,
6685
- type: incident.type,
6686
- severity: incident.severity,
6687
- wasResolved: !!incident.resolvedAt,
6688
- wasAnalyzed: !!incident.aiAnalysis
6689
- }).catch(() => {
6690
- });
6691
- }
6692
- }
6693
- return deleted;
6694
- }
6695
- /**
6696
- * Get statistics about stored incidents
6697
- */
6698
- async getStats() {
6699
- const totalResult = await this.db.query(`
6700
- SELECT
6701
- COUNT(*) as total,
6702
- MIN(timestamp) as oldest,
6703
- MAX(timestamp) as newest
6704
- FROM incidents
6705
- `);
6706
- const { total, oldest, newest } = totalResult.rows[0];
6707
- const severityResult = await this.db.query("SELECT severity, COUNT(*) as count FROM incidents GROUP BY severity");
6708
- const bySeverity = {
6709
- critical: 0,
6710
- warning: 0,
6711
- info: 0
6712
- };
6713
- for (const row of severityResult.rows) {
6714
- bySeverity[row.severity] = row.count;
6715
- }
6716
- const typeResult = await this.db.query("SELECT type, COUNT(*) as count FROM incidents GROUP BY type");
6717
- const byType = {};
6718
- for (const row of typeResult.rows) {
6719
- byType[row.type] = row.count;
6720
- }
6721
- const resolvedResult = await this.db.query(`
6722
- SELECT
6723
- SUM(CASE WHEN resolved_at IS NOT NULL THEN 1 ELSE 0 END) as resolved,
6724
- SUM(CASE WHEN resolved_at IS NULL THEN 1 ELSE 0 END) as unresolved
6725
- FROM incidents
6726
- `);
6727
- const { resolved, unresolved } = resolvedResult.rows[0];
6728
- return {
6729
- total,
6730
- bySeverity,
6731
- byType,
6732
- resolved: resolved ?? 0,
6733
- unresolved: unresolved ?? 0,
6734
- oldestTimestamp: oldest,
6735
- newestTimestamp: newest
6736
- };
6737
- }
6738
- /**
6739
- * Clear all incidents (admin function)
6740
- */
6741
- async clearAll() {
6742
- await this.db.query("DELETE FROM incidents");
6743
- this.log("warn", "All incidents cleared");
6744
- }
6745
- /**
6746
- * Update AI analysis for an incident
6747
- */
6748
- async updateAIAnalysis(id, analysis) {
6749
- await this.db.query(
6750
- `UPDATE incidents
6751
- SET ai_analysis = ?, ai_analyzed_at = ?
6752
- WHERE id = ?`,
6753
- [JSON.stringify(analysis), Date.now(), id]
6754
- );
6755
- this.log("debug", "AI analysis updated", { id });
6756
- }
6757
- /**
6758
- * Convert database row to Incident object
6759
- * @private
6760
- */
6761
- rowToIncident(row) {
6762
- const affectedServices = row.affected_services ? JSON.parse(row.affected_services) : void 0;
6763
- const metadata = row.metadata ? JSON.parse(row.metadata) : void 0;
6764
- const aiAnalysis = row.ai_analysis ? JSON.parse(row.ai_analysis) : void 0;
6765
- const relatedData = row.related_logs_sample || row.related_metrics || row.timeline ? {
6766
- logs: row.related_logs_sample ? JSON.parse(row.related_logs_sample) : void 0,
6767
- metrics: row.related_metrics ? JSON.parse(row.related_metrics) : void 0,
6768
- timeline: row.timeline ? JSON.parse(row.timeline) : void 0
6769
- } : void 0;
6770
- return {
6771
- id: row.id,
6772
- type: row.type,
6773
- severity: row.severity,
6774
- title: row.title,
6775
- details: row.details,
6776
- timestamp: row.timestamp,
6777
- resolvedAt: row.resolved_at ?? void 0,
6778
- resolutionNotes: row.resolution_notes ?? void 0,
6779
- affectedServices,
6780
- metadata,
6781
- relatedData,
6782
- aiAnalysis,
6783
- aiAnalyzedAt: row.ai_analyzed_at ?? void 0
6784
- };
6785
- }
6786
- log(level, message, meta) {
6787
- if (level === "debug" && !this.config.debug) {
6788
- return;
6789
- }
6790
- if (this.logger[level]) {
6791
- this.logger[level](`[IncidentStorage] ${message}`, meta);
6792
- } else {
6793
- console.log(`[IncidentStorage] [${level}] ${message}`, meta);
6794
- }
6795
- }
6796
- };
6797
- var DEFAULT_THRESHOLDS = {
6798
- errorRateWarning: 5,
6799
- errorRateCritical: 10,
6800
- latencyP99Warning: 2e3,
6801
- latencyP99Critical: 5e3,
6802
- latencyP95Warning: 1e3,
6803
- minRequestsForDetection: 10,
6804
- pluginErrorRateWarning: 10,
6805
- pluginErrorRateCritical: 25
6806
- };
6807
- var DEFAULT_CONFIG4 = {
6808
- intervalMs: 3e4,
6809
- // 30 seconds
6810
- thresholds: DEFAULT_THRESHOLDS,
6811
- cooldownMs: 5 * 60 * 1e3,
6812
- // 5 minutes
6813
- debug: false
6814
- };
6815
- var IncidentDetector = class {
6816
- incidentStorage;
6817
- config;
6818
- logger;
6819
- intervalHandle = null;
6820
- recentIncidents = [];
6821
- isRunning = false;
6822
- // Metrics history for "before" comparison (last 10 snapshots)
6823
- metricsHistory = [];
6824
- constructor(incidentStorage, config = {}, logger = console) {
6825
- this.incidentStorage = incidentStorage;
6826
- this.config = {
6827
- ...DEFAULT_CONFIG4,
6828
- ...config,
6829
- thresholds: { ...DEFAULT_THRESHOLDS, ...config.thresholds }
6830
- };
6831
- this.logger = logger;
6832
- }
6833
- /**
6834
- * Start automatic detection
6835
- */
6836
- start() {
6837
- if (this.isRunning) {
6838
- this.log("warn", "Detector already running");
6839
- return;
6840
- }
6841
- this.isRunning = true;
6842
- this.log("info", "Starting incident detector", {
6843
- intervalMs: this.config.intervalMs,
6844
- thresholds: this.config.thresholds
6845
- });
6846
- this.runDetection().catch((err) => {
6847
- this.log("error", "Initial detection failed", { error: err.message });
6848
- });
6849
- this.intervalHandle = setInterval(() => {
6850
- this.runDetection().catch((err) => {
6851
- this.log("error", "Detection cycle failed", { error: err.message });
6852
- });
6853
- }, this.config.intervalMs);
6854
- }
6855
- /**
6856
- * Stop automatic detection
6857
- */
6858
- stop() {
6859
- if (this.intervalHandle) {
6860
- clearInterval(this.intervalHandle);
6861
- this.intervalHandle = null;
6862
- }
6863
- this.isRunning = false;
6864
- this.log("info", "Incident detector stopped");
6865
- }
6866
- /**
6867
- * Run a single detection cycle
6868
- */
6869
- async runDetection() {
6870
- const metrics = metricsCollector.getMetrics();
6871
- const now = Date.now();
6872
- const totalErrors = (metrics.requests.clientErrors ?? 0) + (metrics.requests.serverErrors ?? 0);
6873
- const errorRate = metrics.requests.total > 0 ? totalErrors / metrics.requests.total * 100 : 0;
6874
- this.metricsHistory.push({
6875
- timestamp: now,
6876
- errorRate,
6877
- avgLatency: metrics.latency.average ?? 0,
6878
- totalRequests: metrics.requests.total
6879
- });
6880
- if (this.metricsHistory.length > 10) {
6881
- this.metricsHistory.shift();
6882
- }
6883
- this.recentIncidents = this.recentIncidents.filter(
6884
- (ri) => now - ri.timestamp < this.config.cooldownMs
6885
- );
6886
- if (metrics.requests.total < this.config.thresholds.minRequestsForDetection) {
6887
- this.log("debug", "Not enough requests for detection", {
6888
- total: metrics.requests.total,
6889
- min: this.config.thresholds.minRequestsForDetection
6890
- });
6891
- return;
6892
- }
6893
- await this.detectErrorRate(metrics, now);
6894
- await this.detectLatencyIssues(metrics, now);
6895
- await this.detectPluginFailures(metrics, now);
6896
- this.log("debug", "Detection cycle complete", {
6897
- recentIncidentsCount: this.recentIncidents.length
6898
- });
6899
- }
6900
- /**
6901
- * Detect high error rate
6902
- */
6903
- async detectErrorRate(metrics, now) {
6904
- const totalErrors = (metrics.requests.clientErrors ?? 0) + (metrics.requests.serverErrors ?? 0);
6905
- const errorRate = metrics.requests.total > 0 ? totalErrors / metrics.requests.total * 100 : 0;
6906
- const key = "error_rate";
6907
- if (this.isInCooldown(key)) {
6908
- return;
6909
- }
6910
- let severity = null;
6911
- let title = "";
6912
- let details = "";
6913
- if (errorRate >= this.config.thresholds.errorRateCritical) {
6914
- severity = "critical";
6915
- title = `Critical Error Rate: ${errorRate.toFixed(1)}%`;
6916
- details = `Error rate has exceeded critical threshold of ${this.config.thresholds.errorRateCritical}%. Current: ${errorRate.toFixed(2)}% (${totalErrors}/${metrics.requests.total} requests). Client errors: ${metrics.requests.clientErrors ?? 0}, Server errors: ${metrics.requests.serverErrors ?? 0}.`;
6917
- } else if (errorRate >= this.config.thresholds.errorRateWarning) {
6918
- severity = "warning";
6919
- title = `High Error Rate: ${errorRate.toFixed(1)}%`;
6920
- details = `Error rate has exceeded warning threshold of ${this.config.thresholds.errorRateWarning}%. Current: ${errorRate.toFixed(2)}% (${totalErrors}/${metrics.requests.total} requests).`;
6921
- }
6922
- if (severity) {
6923
- await this.createIncident({
6924
- type: "error_rate",
6925
- severity,
6926
- title,
6927
- details,
6928
- metadata: {
6929
- errorRate,
6930
- totalRequests: metrics.requests.total,
6931
- clientErrors: metrics.requests.clientErrors,
6932
- serverErrors: metrics.requests.serverErrors,
6933
- threshold: severity === "critical" ? this.config.thresholds.errorRateCritical : this.config.thresholds.errorRateWarning
6934
- }
6935
- }, key, now);
6936
- }
6937
- }
6938
- /**
6939
- * Detect latency spikes
6940
- */
6941
- async detectLatencyIssues(metrics, now) {
6942
- const avgLatency = metrics.latency.average ?? 0;
6943
- let p99 = avgLatency * 3;
6944
- let p95 = avgLatency * 2;
6945
- if (metrics.latency.histogram && metrics.latency.histogram.length > 0) {
6946
- const sorted = [...metrics.latency.histogram].sort((a, b) => b.max - a.max);
6947
- if (sorted.length > 0) {
6948
- p99 = sorted[0]?.max ?? p99;
6949
- p95 = sorted[Math.floor(sorted.length * 0.05)]?.max ?? p95;
6950
- }
6951
- }
6952
- const keyP99 = "latency_p99";
6953
- const keyP95 = "latency_p95";
6954
- if (!this.isInCooldown(keyP99)) {
6955
- let severity = null;
6956
- let title = "";
6957
- let details = "";
6958
- if (p99 >= this.config.thresholds.latencyP99Critical) {
6959
- severity = "critical";
6960
- title = `Critical P99 Latency: ${p99.toFixed(0)}ms`;
6961
- details = `P99 latency has exceeded critical threshold of ${this.config.thresholds.latencyP99Critical}ms. Current P99: ${p99.toFixed(0)}ms, Average: ${avgLatency.toFixed(0)}ms. This indicates severe performance degradation affecting tail latencies.`;
6962
- } else if (p99 >= this.config.thresholds.latencyP99Warning) {
6963
- severity = "warning";
6964
- title = `High P99 Latency: ${p99.toFixed(0)}ms`;
6965
- details = `P99 latency has exceeded warning threshold of ${this.config.thresholds.latencyP99Warning}ms. Current P99: ${p99.toFixed(0)}ms, Average: ${avgLatency.toFixed(0)}ms.`;
6966
- }
6967
- if (severity) {
6968
- await this.createIncident({
6969
- type: "latency_spike",
6970
- severity,
6971
- title,
6972
- details,
6973
- metadata: {
6974
- p99,
6975
- p95,
6976
- average: avgLatency,
6977
- threshold: severity === "critical" ? this.config.thresholds.latencyP99Critical : this.config.thresholds.latencyP99Warning
6978
- }
6979
- }, keyP99, now);
6980
- }
6981
- }
6982
- if (!this.isInCooldown(keyP95) && p95 >= this.config.thresholds.latencyP95Warning) {
6983
- await this.createIncident({
6984
- type: "latency_spike",
6985
- severity: "warning",
6986
- title: `Elevated P95 Latency: ${p95.toFixed(0)}ms`,
6987
- details: `P95 latency has exceeded threshold of ${this.config.thresholds.latencyP95Warning}ms. Current P95: ${p95.toFixed(0)}ms. Consider investigating slow endpoints.`,
6988
- metadata: {
6989
- p95,
6990
- average: avgLatency,
6991
- threshold: this.config.thresholds.latencyP95Warning
6992
- }
6993
- }, keyP95, now);
6994
- }
6995
- }
6996
- /**
6997
- * Detect plugin failures
6998
- */
6999
- async detectPluginFailures(metrics, now) {
7000
- if (!metrics.perPlugin || !Array.isArray(metrics.perPlugin)) {
7001
- return;
7002
- }
7003
- for (const plugin of metrics.perPlugin) {
7004
- if (!plugin.pluginId || plugin.requests < 5) {
7005
- continue;
7006
- }
7007
- const errorRate = plugin.requests > 0 ? (plugin.errors ?? 0) / plugin.requests * 100 : 0;
7008
- const key = `plugin:${plugin.pluginId}`;
7009
- if (this.isInCooldown(key)) {
7010
- continue;
7011
- }
7012
- let severity = null;
7013
- let title = "";
7014
- let details = "";
7015
- if (errorRate >= this.config.thresholds.pluginErrorRateCritical) {
7016
- severity = "critical";
7017
- title = `Plugin Failure: ${plugin.pluginId}`;
7018
- details = `Plugin ${plugin.pluginId} has critical error rate of ${errorRate.toFixed(1)}% (${plugin.errors ?? 0}/${plugin.requests} requests). This exceeds the critical threshold of ${this.config.thresholds.pluginErrorRateCritical}%.`;
7019
- } else if (errorRate >= this.config.thresholds.pluginErrorRateWarning) {
7020
- severity = "warning";
7021
- title = `Plugin Issues: ${plugin.pluginId}`;
7022
- details = `Plugin ${plugin.pluginId} has elevated error rate of ${errorRate.toFixed(1)}% (${plugin.errors ?? 0}/${plugin.requests} requests).`;
7023
- }
7024
- if (severity) {
7025
- await this.createIncident({
7026
- type: "plugin_failure",
7027
- severity,
7028
- title,
7029
- details,
7030
- affectedServices: [plugin.pluginId],
7031
- metadata: {
7032
- pluginId: plugin.pluginId,
7033
- errorRate,
7034
- requests: plugin.requests,
7035
- errors: plugin.errors,
7036
- avgLatency: plugin.latency?.average
7037
- }
7038
- }, key, now);
7039
- }
7040
- }
7041
- }
7042
- /**
7043
- * Check if incident key is in cooldown
7044
- */
7045
- isInCooldown(key) {
7046
- return this.recentIncidents.some((ri) => ri.key === key);
7047
- }
7048
- /**
7049
- * Gather related data (logs, metrics, timeline) for incident
7050
- * @private
7051
- */
7052
- async gatherRelatedData(incidentType, timeWindow = 5 * 60 * 1e3) {
7053
- const now = Date.now();
7054
- const from = now - timeWindow;
7055
- const relatedData = {
7056
- timeline: []
7057
- };
7058
- try {
7059
- const [errorLogsResult, fatalLogsResult] = await Promise.all([
7060
- platform.logs.query({ level: "error", from, to: now }, { limit: 50 }),
7061
- platform.logs.query({ level: "fatal", from, to: now }, { limit: 50 })
7062
- ]);
7063
- const allErrorLogs = [...errorLogsResult.logs, ...fatalLogsResult.logs];
7064
- if (allErrorLogs.length > 0) {
7065
- allErrorLogs.sort((a, b) => b.timestamp - a.timestamp);
7066
- const uniqueErrors = /* @__PURE__ */ new Set();
7067
- const endpointErrorCount = /* @__PURE__ */ new Map();
7068
- for (const log of allErrorLogs) {
7069
- let errorMsg = typeof log.message === "string" ? log.message : JSON.stringify(log.message);
7070
- if (log.err) {
7071
- const err = log.err;
7072
- const stack = err.stack ? `
7073
- ${err.stack.split("\n").slice(0, 3).join("\n")}` : "";
7074
- errorMsg = `${err.message || errorMsg}${stack}`;
7075
- if (log.plugin) {
7076
- errorMsg = `[${log.plugin}] ${errorMsg}`;
7077
- }
7078
- if (log.command) {
7079
- errorMsg += `
7080
- Command: ${log.command}`;
7081
- }
7082
- }
7083
- if (errorMsg && uniqueErrors.size < 5) {
7084
- uniqueErrors.add(errorMsg.substring(0, 500));
7085
- }
7086
- const endpoint = log.endpoint || log.url || "unknown";
7087
- const existing = endpointErrorCount.get(endpoint);
7088
- if (existing) {
7089
- existing.count++;
7090
- } else {
7091
- endpointErrorCount.set(endpoint, {
7092
- count: 1,
7093
- sample: errorMsg.substring(0, 200)
7094
- });
7095
- }
7096
- }
7097
- const topEndpoints = Array.from(endpointErrorCount.entries()).sort((a, b) => b[1].count - a[1].count).slice(0, 5).map(([endpoint, data]) => ({
7098
- endpoint,
7099
- count: data.count,
7100
- sample: data.sample
7101
- }));
7102
- relatedData.logs = {
7103
- errorCount: allErrorLogs.filter((l) => l.level === "error").length,
7104
- warnCount: 0,
7105
- // Could query warnings separately if needed
7106
- timeRange: [from, now],
7107
- sampleErrors: Array.from(uniqueErrors),
7108
- topEndpoints: topEndpoints.length > 0 ? topEndpoints : void 0
7109
- };
7110
- for (const log of allErrorLogs.slice(0, 10)) {
7111
- const msg = typeof log.message === "string" ? log.message : "Error occurred";
7112
- const plugin = log.plugin ? `[${log.plugin}] ` : "";
7113
- relatedData.timeline.push({
7114
- timestamp: log.timestamp,
7115
- event: `${plugin}${msg.substring(0, 100)}`,
7116
- source: "logs"
7117
- });
7118
- }
7119
- }
7120
- } catch (error) {
7121
- this.log("warn", "Failed to gather related logs", {
7122
- error: error instanceof Error ? error.message : String(error)
7123
- });
7124
- }
7125
- const currentMetrics = metricsCollector.getMetrics();
7126
- const duringMetrics = {
7127
- errorRate: currentMetrics.requests.total > 0 ? (currentMetrics.requests.clientErrors + currentMetrics.requests.serverErrors) / currentMetrics.requests.total * 100 : 0,
7128
- avgLatency: currentMetrics.latency.average ?? 0,
7129
- totalRequests: currentMetrics.requests.total,
7130
- totalErrors: currentMetrics.requests.clientErrors + currentMetrics.requests.serverErrors
7131
- };
7132
- let beforeMetrics;
7133
- if (this.metricsHistory.length >= 2) {
7134
- const beforeSnapshots = this.metricsHistory.slice(0, -1);
7135
- if (beforeSnapshots.length > 0) {
7136
- const avgErrorRate = beforeSnapshots.reduce((sum, s) => sum + s.errorRate, 0) / beforeSnapshots.length;
7137
- const avgLatency = beforeSnapshots.reduce((sum, s) => sum + s.avgLatency, 0) / beforeSnapshots.length;
7138
- const avgRequests = beforeSnapshots.reduce((sum, s) => sum + s.totalRequests, 0) / beforeSnapshots.length;
7139
- beforeMetrics = {
7140
- errorRate: avgErrorRate,
7141
- avgLatency,
7142
- totalRequests: avgRequests
7143
- };
7144
- this.log("debug", "Calculated before metrics", {
7145
- before: beforeMetrics,
7146
- during: duringMetrics,
7147
- snapshotsUsed: beforeSnapshots.length
7148
- });
7149
- }
7150
- }
7151
- let topSlowest;
7152
- let affectedEndpoints;
7153
- if (incidentType === "latency_spike") {
7154
- const histogram = currentMetrics.latency.histogram;
7155
- if (histogram && histogram.length > 0) {
7156
- const slowestRoutes = histogram.filter((h) => h.max > 100).sort((a, b) => b.max - a.max).slice(0, 10);
7157
- topSlowest = slowestRoutes.map((h) => {
7158
- const [method, ...pathParts] = h.route.split(" ");
7159
- const endpoint = pathParts.join(" ");
7160
- const statusCodes = Object.keys(h.byStatus);
7161
- const mostCommonStatus = statusCodes.length > 0 ? parseInt(statusCodes[0], 10) : void 0;
7162
- return {
7163
- endpoint,
7164
- method: method || "GET",
7165
- durationMs: Math.round(h.max),
7166
- statusCode: mostCommonStatus
7167
- };
7168
- });
7169
- affectedEndpoints = [...new Set(slowestRoutes.map((h) => h.route))];
7170
- this.log("debug", "Collected slow requests for latency incident", {
7171
- topSlowestCount: topSlowest.length,
7172
- affectedEndpointsCount: affectedEndpoints.length
7173
- });
7174
- }
7175
- }
7176
- relatedData.metrics = {
7177
- before: beforeMetrics,
7178
- during: duringMetrics,
7179
- topSlowest,
7180
- affectedEndpoints
7181
- };
7182
- relatedData.timeline.unshift({
7183
- timestamp: now,
7184
- event: `Incident detected: ${incidentType}`,
7185
- source: "detector"
7186
- });
7187
- relatedData.timeline.sort((a, b) => b.timestamp - a.timestamp);
7188
- return relatedData;
7189
- }
7190
- /**
7191
- * Create incident and track it
7192
- */
7193
- async createIncident(payload, key, timestamp) {
7194
- try {
7195
- const relatedData = await this.gatherRelatedData(payload.type);
7196
- const enrichedPayload = {
7197
- ...payload,
7198
- relatedData
7199
- };
7200
- const incident = await this.incidentStorage.createIncident(enrichedPayload);
7201
- this.recentIncidents.push({
7202
- type: payload.type,
7203
- key,
7204
- timestamp
7205
- });
7206
- this.log("info", "Auto-created incident with context", {
7207
- id: incident.id,
7208
- type: incident.type,
7209
- severity: incident.severity,
7210
- title: incident.title,
7211
- errorLogsCount: relatedData.logs?.errorCount ?? 0,
7212
- timelineEventsCount: relatedData.timeline?.length ?? 0
7213
- });
7214
- if (platform.analytics) {
7215
- platform.analytics.track("incident.created", {
7216
- incidentId: incident.id,
7217
- type: incident.type,
7218
- severity: incident.severity,
7219
- source: "auto-detector",
7220
- errorLogsCount: relatedData.logs?.errorCount ?? 0,
7221
- timelineEventsCount: relatedData.timeline?.length ?? 0,
7222
- hasBeforeMetrics: !!relatedData.metrics?.before,
7223
- affectedServicesCount: payload.affectedServices?.length ?? 0
7224
- }).catch(() => {
7225
- });
7226
- }
7227
- } catch (error) {
7228
- this.log("error", "Failed to create incident", {
7229
- key,
7230
- error: error instanceof Error ? error.message : String(error)
7231
- });
7232
- }
7233
- }
7234
- /**
7235
- * Get current detector status
7236
- */
7237
- getStatus() {
7238
- return {
7239
- running: this.isRunning,
7240
- config: this.config,
7241
- recentIncidentsCount: this.recentIncidents.length
7242
- };
7243
- }
7244
- /**
7245
- * Update thresholds at runtime
7246
- */
7247
- updateThresholds(thresholds) {
7248
- this.config.thresholds = { ...this.config.thresholds, ...thresholds };
7249
- this.log("info", "Thresholds updated", { thresholds: this.config.thresholds });
7250
- }
7251
- log(level, message, meta) {
7252
- if (level === "debug" && !this.config.debug) {
7253
- return;
7254
- }
7255
- const prefix = "[IncidentDetector]";
7256
- if (this.logger[level]) {
7257
- if (meta) {
7258
- this.logger[level]({ ...meta }, `${prefix} ${message}`);
7259
- } else {
7260
- this.logger[level](`${prefix} ${message}`);
7261
- }
7262
- } else {
7263
- console.log(`${prefix} [${level}] ${message}`, meta ?? "");
7264
- }
7265
- }
7266
- };
7267
5643
 
7268
5644
  // src/routes/index.ts
7269
5645
  function normalizeBasePath2(basePath) {
@@ -7459,71 +5835,7 @@ async function registerRoutes(server, config, repoRoot, registry) {
7459
5835
  historicalCollector.stop();
7460
5836
  platform.logger.info("Historical metrics collector stopped");
7461
5837
  });
7462
- let incidentStorage;
7463
- const db = platform.getAdapter("db");
7464
- if (db) {
7465
- try {
7466
- const { readFileSync: readFileSync2 } = await import('fs');
7467
- const { fileURLToPath } = await import('url');
7468
- const { dirname: dirname2, join: join3 } = await import('path');
7469
- const __filename = fileURLToPath(import.meta.url);
7470
- const __dirname = dirname2(__filename);
7471
- const schemaPath = join3(__dirname, "..", "services", "incident-schema.sql");
7472
- const schema = readFileSync2(schemaPath, "utf-8");
7473
- if ("exec" in db && typeof db.exec === "function") {
7474
- await db.exec(schema);
7475
- } else {
7476
- const statements = schema.split(";").map((s) => s.trim()).filter((s) => s.length > 0 && !s.startsWith("--"));
7477
- for (const statement of statements) {
7478
- if (statement) {
7479
- await db.query(statement);
7480
- }
7481
- }
7482
- }
7483
- platform.logger.info("Incidents schema initialized");
7484
- incidentStorage = new IncidentStorage(
7485
- db,
7486
- { debug: process.env.NODE_ENV !== "production" },
7487
- platformServices.logger
7488
- );
7489
- platform.logger.info("Incident storage initialized");
7490
- } catch (err) {
7491
- const error = err instanceof Error ? err : new Error(String(err));
7492
- platform.logger.warn("Failed to initialize incident storage, continuing without it", { error: error.message });
7493
- }
7494
- } else {
7495
- platform.logger.warn("Database adapter (db) not configured \u2014 incident storage disabled. Configure db adapter in kb.config.json to enable.");
7496
- }
7497
- if (incidentStorage) {
7498
- const incidentDetector = new IncidentDetector(
7499
- incidentStorage,
7500
- {
7501
- intervalMs: 3e4,
7502
- // Check every 30 seconds
7503
- cooldownMs: 5 * 60 * 1e3,
7504
- // 5 minute cooldown between same incidents
7505
- debug: process.env.NODE_ENV !== "production",
7506
- thresholds: {
7507
- errorRateWarning: 5,
7508
- errorRateCritical: 10,
7509
- latencyP99Warning: 2e3,
7510
- latencyP99Critical: 5e3,
7511
- latencyP95Warning: 1e3,
7512
- minRequestsForDetection: 10,
7513
- pluginErrorRateWarning: 10,
7514
- pluginErrorRateCritical: 25
7515
- }
7516
- },
7517
- platformServices.logger
7518
- );
7519
- incidentDetector.start();
7520
- platform.logger.info("Incident detector started");
7521
- server.addHook("onClose", async () => {
7522
- incidentDetector.stop();
7523
- platform.logger.info("Incident detector stopped");
7524
- });
7525
- }
7526
- await registerObservabilityRoutes(server, config, repoRoot, historicalCollector, incidentStorage, platformServices);
5838
+ await registerObservabilityRoutes(server, config, repoRoot, historicalCollector, platformServices);
7527
5839
  await registerAnalyticsRoutes(server, config);
7528
5840
  await registerAdaptersRoutes(server, config);
7529
5841
  await registerLogRoutes(server);
@@ -7614,6 +5926,9 @@ function registerEnvelopeMiddleware(server, config) {
7614
5926
  if (parsedPayload && typeof parsedPayload === "object" && "ok" in parsedPayload) {
7615
5927
  return payload;
7616
5928
  }
5929
+ if (reply.getHeader("x-openapi-spec")) {
5930
+ return payload;
5931
+ }
7617
5932
  let dataToWrap = parsedPayload;
7618
5933
  if (parsedPayload === null || parsedPayload === void 0) {
7619
5934
  dataToWrap = null;