@kb-labs/rest-api-app 2.14.0 → 2.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +50 -1736
- package/dist/index.js.map +1 -1
- package/package.json +21 -21
package/dist/index.js
CHANGED
|
@@ -21,8 +21,6 @@ import { WORKFLOW_REDIS_CHANNEL } from '@kb-labs/workflow-constants';
|
|
|
21
21
|
import { validatorCompiler, serializerCompiler } from 'fastify-type-provider-zod';
|
|
22
22
|
import { z } from 'zod';
|
|
23
23
|
import { errorEnvelopeSchema, ErrorCode, ErrorResponseSchema, JobsListResponseSchema, ListJobsQuerySchema, JobStatsResponseSchema, JobResponseSchema, JobActionResponseSchema } from '@kb-labs/rest-api-contracts';
|
|
24
|
-
import { exec } from 'child_process';
|
|
25
|
-
import { promisify } from 'util';
|
|
26
24
|
import { readKbConfig } from '@kb-labs/core-config';
|
|
27
25
|
import { AdapterNameSchema } from '@kb-labs/gateway-contracts';
|
|
28
26
|
import fastifyCors from '@fastify/cors';
|
|
@@ -2900,261 +2898,15 @@ async function registerCacheRoutes(server, config, registry) {
|
|
|
2900
2898
|
}
|
|
2901
2899
|
});
|
|
2902
2900
|
}
|
|
2903
|
-
|
|
2904
|
-
model: "gpt-4",
|
|
2905
|
-
temperature: 0.3,
|
|
2906
|
-
maxTokens: 1500,
|
|
2907
|
-
debug: false
|
|
2908
|
-
};
|
|
2909
|
-
var IncidentAnalyzer = class {
|
|
2910
|
-
config;
|
|
2911
|
-
logger;
|
|
2912
|
-
constructor(config = {}, logger = console) {
|
|
2913
|
-
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
2914
|
-
this.logger = logger;
|
|
2915
|
-
}
|
|
2916
|
-
/**
|
|
2917
|
-
* Analyze incident using LLM
|
|
2918
|
-
*
|
|
2919
|
-
* @param incident - Incident to analyze
|
|
2920
|
-
* @returns AI analysis result
|
|
2921
|
-
*/
|
|
2922
|
-
async analyze(incident) {
|
|
2923
|
-
this.log("info", "Analyzing incident", { id: incident.id, type: incident.type });
|
|
2924
|
-
const prompt = this.buildAnalysisPrompt(incident);
|
|
2925
|
-
try {
|
|
2926
|
-
const result = await platform.llm.complete(prompt, {
|
|
2927
|
-
model: this.config.model,
|
|
2928
|
-
temperature: this.config.temperature,
|
|
2929
|
-
maxTokens: this.config.maxTokens,
|
|
2930
|
-
systemPrompt: this.getSystemPrompt()
|
|
2931
|
-
});
|
|
2932
|
-
this.log("debug", "LLM response received", {
|
|
2933
|
-
length: result.content.length,
|
|
2934
|
-
tokensUsed: result.usage ? result.usage.promptTokens + result.usage.completionTokens : void 0
|
|
2935
|
-
});
|
|
2936
|
-
const analysis = this.parseAnalysisResponse(result.content);
|
|
2937
|
-
this.log("info", "Incident analysis complete", {
|
|
2938
|
-
id: incident.id,
|
|
2939
|
-
rootCausesCount: analysis.rootCauses.length,
|
|
2940
|
-
recommendationsCount: analysis.recommendations.length
|
|
2941
|
-
});
|
|
2942
|
-
return analysis;
|
|
2943
|
-
} catch (error) {
|
|
2944
|
-
this.log("error", "Incident analysis failed", {
|
|
2945
|
-
id: incident.id,
|
|
2946
|
-
error: error instanceof Error ? error.message : String(error)
|
|
2947
|
-
});
|
|
2948
|
-
return this.getFallbackAnalysis(incident);
|
|
2949
|
-
}
|
|
2950
|
-
}
|
|
2951
|
-
/**
|
|
2952
|
-
* Build analysis prompt from incident data
|
|
2953
|
-
* @private
|
|
2954
|
-
*/
|
|
2955
|
-
buildAnalysisPrompt(incident) {
|
|
2956
|
-
let prompt = `# Incident Analysis Request
|
|
2957
|
-
|
|
2958
|
-
`;
|
|
2959
|
-
prompt += `## Incident Overview
|
|
2960
|
-
`;
|
|
2961
|
-
prompt += `- **ID**: ${incident.id}
|
|
2962
|
-
`;
|
|
2963
|
-
prompt += `- **Type**: ${incident.type}
|
|
2964
|
-
`;
|
|
2965
|
-
prompt += `- **Severity**: ${incident.severity}
|
|
2966
|
-
`;
|
|
2967
|
-
prompt += `- **Title**: ${incident.title}
|
|
2968
|
-
`;
|
|
2969
|
-
prompt += `- **Time**: ${new Date(incident.timestamp).toISOString()}
|
|
2970
|
-
|
|
2971
|
-
`;
|
|
2972
|
-
prompt += `## Description
|
|
2973
|
-
${incident.details}
|
|
2974
|
-
|
|
2975
|
-
`;
|
|
2976
|
-
if (incident.metadata && Object.keys(incident.metadata).length > 0) {
|
|
2977
|
-
prompt += `## Metrics
|
|
2978
|
-
`;
|
|
2979
|
-
prompt += "```json\n";
|
|
2980
|
-
prompt += JSON.stringify(incident.metadata, null, 2);
|
|
2981
|
-
prompt += "\n```\n\n";
|
|
2982
|
-
}
|
|
2983
|
-
if (incident.relatedData?.logs && incident.relatedData.logs.sampleErrors.length > 0) {
|
|
2984
|
-
prompt += `## Related Error Logs (${incident.relatedData.logs.errorCount} total)
|
|
2985
|
-
`;
|
|
2986
|
-
incident.relatedData.logs.sampleErrors.forEach((error, idx) => {
|
|
2987
|
-
prompt += `${idx + 1}. ${error}
|
|
2988
|
-
`;
|
|
2989
|
-
});
|
|
2990
|
-
prompt += "\n";
|
|
2991
|
-
}
|
|
2992
|
-
if (incident.relatedData?.timeline && incident.relatedData.timeline.length > 0) {
|
|
2993
|
-
prompt += `## Event Timeline
|
|
2994
|
-
`;
|
|
2995
|
-
incident.relatedData.timeline.slice(0, 10).forEach((event) => {
|
|
2996
|
-
const time = new Date(event.timestamp).toISOString();
|
|
2997
|
-
prompt += `- [${time}] (${event.source}) ${event.event}
|
|
2998
|
-
`;
|
|
2999
|
-
});
|
|
3000
|
-
prompt += "\n";
|
|
3001
|
-
}
|
|
3002
|
-
if (incident.relatedData?.metrics) {
|
|
3003
|
-
prompt += `## System Metrics
|
|
3004
|
-
`;
|
|
3005
|
-
prompt += "```json\n";
|
|
3006
|
-
prompt += JSON.stringify(incident.relatedData.metrics, null, 2);
|
|
3007
|
-
prompt += "\n```\n\n";
|
|
3008
|
-
}
|
|
3009
|
-
prompt += `## Analysis Instructions
|
|
3010
|
-
`;
|
|
3011
|
-
prompt += `Please analyze this incident and provide:
|
|
3012
|
-
`;
|
|
3013
|
-
prompt += `1. A brief executive summary (2-3 sentences)
|
|
3014
|
-
`;
|
|
3015
|
-
prompt += `2. Root causes with confidence scores (0.0-1.0) and evidence
|
|
3016
|
-
`;
|
|
3017
|
-
prompt += `3. Patterns or trends you observe
|
|
3018
|
-
`;
|
|
3019
|
-
prompt += `4. Actionable recommendations for resolution and prevention
|
|
3020
|
-
|
|
3021
|
-
`;
|
|
3022
|
-
prompt += `Format your response as JSON:
|
|
3023
|
-
`;
|
|
3024
|
-
prompt += "```json\n";
|
|
3025
|
-
prompt += `{
|
|
3026
|
-
`;
|
|
3027
|
-
prompt += ` "summary": "Executive summary here",
|
|
3028
|
-
`;
|
|
3029
|
-
prompt += ` "rootCauses": [
|
|
3030
|
-
`;
|
|
3031
|
-
prompt += ` {
|
|
3032
|
-
`;
|
|
3033
|
-
prompt += ` "factor": "Root cause description",
|
|
3034
|
-
`;
|
|
3035
|
-
prompt += ` "confidence": 0.85,
|
|
3036
|
-
`;
|
|
3037
|
-
prompt += ` "evidence": "Supporting evidence from logs/metrics"
|
|
3038
|
-
`;
|
|
3039
|
-
prompt += ` }
|
|
3040
|
-
`;
|
|
3041
|
-
prompt += ` ],
|
|
3042
|
-
`;
|
|
3043
|
-
prompt += ` "patterns": ["Pattern 1", "Pattern 2"],
|
|
3044
|
-
`;
|
|
3045
|
-
prompt += ` "recommendations": ["Recommendation 1", "Recommendation 2"]
|
|
3046
|
-
`;
|
|
3047
|
-
prompt += `}
|
|
3048
|
-
`;
|
|
3049
|
-
prompt += "```";
|
|
3050
|
-
return prompt;
|
|
3051
|
-
}
|
|
3052
|
-
/**
|
|
3053
|
-
* Get system prompt for incident analysis
|
|
3054
|
-
* @private
|
|
3055
|
-
*/
|
|
3056
|
-
getSystemPrompt() {
|
|
3057
|
-
return `You are an expert SRE (Site Reliability Engineer) and incident response specialist.
|
|
3058
|
-
Your role is to analyze system incidents, identify root causes, and provide actionable recommendations.
|
|
3059
|
-
|
|
3060
|
-
Guidelines:
|
|
3061
|
-
- Base your analysis on the actual data provided (logs, metrics, timeline)
|
|
3062
|
-
- Provide confidence scores based on available evidence (0.0 = no evidence, 1.0 = certain)
|
|
3063
|
-
- Focus on actionable recommendations that can prevent future incidents
|
|
3064
|
-
- Be concise but thorough - this is for production systems
|
|
3065
|
-
- Always respond with valid JSON matching the requested format`;
|
|
3066
|
-
}
|
|
3067
|
-
/**
|
|
3068
|
-
* Parse LLM response into structured analysis
|
|
3069
|
-
* @private
|
|
3070
|
-
*/
|
|
3071
|
-
parseAnalysisResponse(response) {
|
|
3072
|
-
try {
|
|
3073
|
-
const jsonMatch = response.match(/```json\s*([\s\S]*?)\s*```/) ?? response.match(/```\s*([\s\S]*?)\s*```/) ?? [null, response];
|
|
3074
|
-
const jsonStr = jsonMatch[1] || response;
|
|
3075
|
-
const parsed = JSON.parse(jsonStr.trim());
|
|
3076
|
-
return {
|
|
3077
|
-
summary: parsed.summary || "No summary provided",
|
|
3078
|
-
rootCauses: Array.isArray(parsed.rootCauses) ? parsed.rootCauses : [],
|
|
3079
|
-
patterns: Array.isArray(parsed.patterns) ? parsed.patterns : [],
|
|
3080
|
-
recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations : [],
|
|
3081
|
-
analyzedAt: Date.now()
|
|
3082
|
-
};
|
|
3083
|
-
} catch (error) {
|
|
3084
|
-
this.log("warn", "Failed to parse LLM response, using fallback", {
|
|
3085
|
-
error: error instanceof Error ? error.message : String(error)
|
|
3086
|
-
});
|
|
3087
|
-
return {
|
|
3088
|
-
summary: response.substring(0, 200),
|
|
3089
|
-
rootCauses: [],
|
|
3090
|
-
patterns: [],
|
|
3091
|
-
recommendations: [],
|
|
3092
|
-
analyzedAt: Date.now()
|
|
3093
|
-
};
|
|
3094
|
-
}
|
|
3095
|
-
}
|
|
3096
|
-
/**
|
|
3097
|
-
* Get fallback analysis when LLM is unavailable
|
|
3098
|
-
* @private
|
|
3099
|
-
*/
|
|
3100
|
-
getFallbackAnalysis(incident) {
|
|
3101
|
-
const errorCount = incident.relatedData?.logs?.errorCount ?? 0;
|
|
3102
|
-
const sampleErrors = incident.relatedData?.logs?.sampleErrors ?? [];
|
|
3103
|
-
return {
|
|
3104
|
-
summary: `${incident.severity} ${incident.type} incident detected. ${errorCount} errors logged.`,
|
|
3105
|
-
rootCauses: [
|
|
3106
|
-
{
|
|
3107
|
-
factor: incident.title,
|
|
3108
|
-
confidence: 0.5,
|
|
3109
|
-
evidence: incident.details
|
|
3110
|
-
}
|
|
3111
|
-
],
|
|
3112
|
-
patterns: sampleErrors.length > 0 ? [`Multiple error types detected: ${sampleErrors.length} unique errors`] : [],
|
|
3113
|
-
recommendations: [
|
|
3114
|
-
"Review related error logs for stack traces",
|
|
3115
|
-
"Check system metrics during incident timeframe",
|
|
3116
|
-
"Verify external dependencies are operational"
|
|
3117
|
-
],
|
|
3118
|
-
analyzedAt: Date.now()
|
|
3119
|
-
};
|
|
3120
|
-
}
|
|
3121
|
-
log(level, message, meta) {
|
|
3122
|
-
if (level === "debug" && !this.config.debug) {
|
|
3123
|
-
return;
|
|
3124
|
-
}
|
|
3125
|
-
const prefix = "[IncidentAnalyzer]";
|
|
3126
|
-
if (this.logger[level]) {
|
|
3127
|
-
if (meta) {
|
|
3128
|
-
this.logger[level]({ ...meta }, `${prefix} ${message}`);
|
|
3129
|
-
} else {
|
|
3130
|
-
this.logger[level](`${prefix} ${message}`);
|
|
3131
|
-
}
|
|
3132
|
-
} else {
|
|
3133
|
-
console.log(`${prefix} [${level}] ${message}`, meta ?? "");
|
|
3134
|
-
}
|
|
3135
|
-
}
|
|
3136
|
-
};
|
|
3137
|
-
|
|
3138
|
-
// src/routes/observability.ts
|
|
3139
|
-
var execAsync = promisify(exec);
|
|
3140
|
-
var DEVKIT_CACHE_KEY = "observability:devkit-health";
|
|
3141
|
-
var DEVKIT_CACHE_TTL_MS = 10 * 60 * 1e3;
|
|
3142
|
-
async function registerObservabilityRoutes(fastify, config, repoRoot, historicalMetrics, incidentStorage, platform19) {
|
|
2901
|
+
async function registerObservabilityRoutes(fastify, config, repoRoot, historicalMetrics, platform16) {
|
|
3143
2902
|
const basePath = normalizeBasePath(config.basePath);
|
|
3144
2903
|
const stateBrokerPaths = resolvePaths(basePath, "/observability/state-broker");
|
|
3145
|
-
const devkitPaths = resolvePaths(basePath, "/observability/devkit");
|
|
3146
2904
|
const systemMetricsPaths = resolvePaths(basePath, "/observability/system-metrics");
|
|
3147
2905
|
const metricsHistoryPaths = resolvePaths(basePath, "/observability/metrics/history");
|
|
3148
2906
|
const metricsHeatmapPaths = resolvePaths(basePath, "/observability/metrics/heatmap");
|
|
3149
|
-
const incidentsListPaths = resolvePaths(basePath, "/observability/incidents");
|
|
3150
|
-
const incidentsDetailPaths = resolvePaths(basePath, "/observability/incidents/:id");
|
|
3151
|
-
const incidentsCreatePaths = resolvePaths(basePath, "/observability/incidents");
|
|
3152
|
-
const incidentsHistoryPaths = resolvePaths(basePath, "/observability/incidents/history");
|
|
3153
|
-
const incidentsResolvePaths = resolvePaths(basePath, "/observability/incidents/:id/resolve");
|
|
3154
|
-
const incidentsAnalyzePaths = resolvePaths(basePath, "/observability/incidents/:id/analyze");
|
|
3155
2907
|
const insightsChatPaths = resolvePaths(basePath, "/observability/insights/chat");
|
|
3156
2908
|
for (const path3 of stateBrokerPaths) {
|
|
3157
|
-
fastify.get(path3, async (_request, reply) => {
|
|
2909
|
+
fastify.get(path3, { schema: { tags: ["Observability"], summary: "State Broker statistics" } }, async (_request, reply) => {
|
|
3158
2910
|
try {
|
|
3159
2911
|
const stateBrokerUrl = process.env.KB_STATE_DAEMON_URL || "http://localhost:7777";
|
|
3160
2912
|
fastify.log.debug({ url: stateBrokerUrl }, "Fetching State Broker stats");
|
|
@@ -3193,7 +2945,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3193
2945
|
}
|
|
3194
2946
|
};
|
|
3195
2947
|
} catch (error) {
|
|
3196
|
-
|
|
2948
|
+
platform16?.logger.error("Failed to fetch State Broker stats", error instanceof Error ? error : new Error(String(error)));
|
|
3197
2949
|
const isTimeout = error instanceof Error && error.name === "AbortError";
|
|
3198
2950
|
return reply.code(503).send({
|
|
3199
2951
|
ok: false,
|
|
@@ -3208,105 +2960,10 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3208
2960
|
}
|
|
3209
2961
|
});
|
|
3210
2962
|
}
|
|
3211
|
-
for (const path3 of devkitPaths) {
|
|
3212
|
-
fastify.get(path3, async (_request, reply) => {
|
|
3213
|
-
const now = Date.now();
|
|
3214
|
-
if (platform19?.cache) {
|
|
3215
|
-
try {
|
|
3216
|
-
const cached = await platform19.cache.get(DEVKIT_CACHE_KEY);
|
|
3217
|
-
if (cached && now < cached.expiresAt) {
|
|
3218
|
-
const remainingTtl = Math.round((cached.expiresAt - now) / 1e3);
|
|
3219
|
-
fastify.log.debug({ remainingTtl }, "Returning cached DevKit health from platform.cache");
|
|
3220
|
-
return {
|
|
3221
|
-
ok: true,
|
|
3222
|
-
data: cached.data,
|
|
3223
|
-
meta: {
|
|
3224
|
-
source: "devkit-cli",
|
|
3225
|
-
repoRoot,
|
|
3226
|
-
cached: true,
|
|
3227
|
-
cachedAt: cached.timestamp,
|
|
3228
|
-
expiresAt: cached.expiresAt,
|
|
3229
|
-
ttlSeconds: remainingTtl
|
|
3230
|
-
}
|
|
3231
|
-
};
|
|
3232
|
-
}
|
|
3233
|
-
} catch (cacheError) {
|
|
3234
|
-
fastify.log.warn({ err: cacheError }, "Failed to read from platform.cache, proceeding without cache");
|
|
3235
|
-
}
|
|
3236
|
-
}
|
|
3237
|
-
try {
|
|
3238
|
-
fastify.log.debug({ cwd: repoRoot }, "Executing DevKit health check (cache miss)");
|
|
3239
|
-
const { stdout, stderr } = await execAsync("npx kb-devkit-health --json", {
|
|
3240
|
-
cwd: repoRoot,
|
|
3241
|
-
timeout: 3e4,
|
|
3242
|
-
// 30s timeout
|
|
3243
|
-
env: {
|
|
3244
|
-
...process.env,
|
|
3245
|
-
// Ensure DevKit runs in non-interactive mode
|
|
3246
|
-
CI: "true"
|
|
3247
|
-
}
|
|
3248
|
-
});
|
|
3249
|
-
if (stderr) {
|
|
3250
|
-
fastify.log.warn({ stderr }, "DevKit health check produced warnings");
|
|
3251
|
-
}
|
|
3252
|
-
const health = JSON.parse(stdout);
|
|
3253
|
-
const expiresAt = now + DEVKIT_CACHE_TTL_MS;
|
|
3254
|
-
if (platform19?.cache) {
|
|
3255
|
-
try {
|
|
3256
|
-
await platform19.cache.set(DEVKIT_CACHE_KEY, {
|
|
3257
|
-
data: health,
|
|
3258
|
-
timestamp: now,
|
|
3259
|
-
expiresAt
|
|
3260
|
-
}, DEVKIT_CACHE_TTL_MS);
|
|
3261
|
-
fastify.log.debug({
|
|
3262
|
-
healthScore: health.healthScore,
|
|
3263
|
-
grade: health.grade,
|
|
3264
|
-
cachedUntil: new Date(expiresAt).toISOString()
|
|
3265
|
-
}, "DevKit health check completed and cached in platform.cache");
|
|
3266
|
-
} catch (cacheError) {
|
|
3267
|
-
fastify.log.warn({ err: cacheError }, "Failed to write to platform.cache");
|
|
3268
|
-
}
|
|
3269
|
-
}
|
|
3270
|
-
return {
|
|
3271
|
-
ok: true,
|
|
3272
|
-
data: health,
|
|
3273
|
-
meta: {
|
|
3274
|
-
source: "devkit-cli",
|
|
3275
|
-
repoRoot,
|
|
3276
|
-
command: "npx kb-devkit-health --json",
|
|
3277
|
-
cached: false,
|
|
3278
|
-
cachedAt: now,
|
|
3279
|
-
expiresAt,
|
|
3280
|
-
ttlSeconds: DEVKIT_CACHE_TTL_MS / 1e3
|
|
3281
|
-
}
|
|
3282
|
-
};
|
|
3283
|
-
} catch (error) {
|
|
3284
|
-
let partialData = null;
|
|
3285
|
-
if (error && typeof error === "object" && "stdout" in error) {
|
|
3286
|
-
try {
|
|
3287
|
-
partialData = JSON.parse(error.stdout);
|
|
3288
|
-
} catch {
|
|
3289
|
-
partialData = null;
|
|
3290
|
-
}
|
|
3291
|
-
}
|
|
3292
|
-
platform19?.logger.error("Failed to execute DevKit health check", error instanceof Error ? error : new Error(String(error)));
|
|
3293
|
-
return reply.code(500).send({
|
|
3294
|
-
ok: false,
|
|
3295
|
-
error: {
|
|
3296
|
-
code: "DEVKIT_ERROR",
|
|
3297
|
-
message: error instanceof Error ? error.message : "Failed to execute DevKit health check",
|
|
3298
|
-
details: {
|
|
3299
|
-
partialData
|
|
3300
|
-
}
|
|
3301
|
-
}
|
|
3302
|
-
});
|
|
3303
|
-
}
|
|
3304
|
-
});
|
|
3305
|
-
}
|
|
3306
2963
|
for (const path3 of systemMetricsPaths) {
|
|
3307
|
-
fastify.get(path3, async (_request, reply) => {
|
|
2964
|
+
fastify.get(path3, { schema: { tags: ["Observability"], summary: "System resource metrics" } }, async (_request, reply) => {
|
|
3308
2965
|
try {
|
|
3309
|
-
if (!
|
|
2966
|
+
if (!platform16?.cache) {
|
|
3310
2967
|
return reply.code(503).send({
|
|
3311
2968
|
ok: false,
|
|
3312
2969
|
error: {
|
|
@@ -3318,17 +2975,17 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3318
2975
|
fastify.log.debug("Fetching system metrics from all instances");
|
|
3319
2976
|
const allMetrics = [];
|
|
3320
2977
|
try {
|
|
3321
|
-
if ("scan" in
|
|
3322
|
-
const keys = await
|
|
2978
|
+
if ("scan" in platform16.cache && typeof platform16.cache.scan === "function") {
|
|
2979
|
+
const keys = await platform16.cache.scan("system-metrics:*");
|
|
3323
2980
|
for (const key of keys) {
|
|
3324
|
-
const metrics = await
|
|
2981
|
+
const metrics = await platform16.cache.get(key);
|
|
3325
2982
|
if (metrics) {
|
|
3326
2983
|
allMetrics.push(metrics);
|
|
3327
2984
|
}
|
|
3328
2985
|
}
|
|
3329
2986
|
} else {
|
|
3330
2987
|
const currentInstanceId = hostname();
|
|
3331
|
-
const metrics = await
|
|
2988
|
+
const metrics = await platform16.cache.get(`system-metrics:${currentInstanceId}`);
|
|
3332
2989
|
if (metrics) {
|
|
3333
2990
|
allMetrics.push(metrics);
|
|
3334
2991
|
}
|
|
@@ -3337,7 +2994,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3337
2994
|
} catch (scanError) {
|
|
3338
2995
|
fastify.log.warn({ err: scanError }, "Failed to scan platform.cache for system metrics");
|
|
3339
2996
|
const currentInstanceId = hostname();
|
|
3340
|
-
const metrics = await
|
|
2997
|
+
const metrics = await platform16.cache.get(`system-metrics:${currentInstanceId}`);
|
|
3341
2998
|
if (metrics) {
|
|
3342
2999
|
allMetrics.push(metrics);
|
|
3343
3000
|
}
|
|
@@ -3385,7 +3042,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3385
3042
|
}
|
|
3386
3043
|
};
|
|
3387
3044
|
} catch (error) {
|
|
3388
|
-
|
|
3045
|
+
platform16?.logger.error("Failed to fetch system metrics", error instanceof Error ? error : new Error(String(error)));
|
|
3389
3046
|
return reply.code(500).send({
|
|
3390
3047
|
ok: false,
|
|
3391
3048
|
error: {
|
|
@@ -3399,6 +3056,8 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3399
3056
|
for (const path3 of metricsHistoryPaths) {
|
|
3400
3057
|
fastify.get(path3, {
|
|
3401
3058
|
schema: {
|
|
3059
|
+
tags: ["Observability"],
|
|
3060
|
+
summary: "Historical time-series metrics",
|
|
3402
3061
|
querystring: {
|
|
3403
3062
|
type: "object",
|
|
3404
3063
|
properties: {
|
|
@@ -3481,7 +3140,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3481
3140
|
}
|
|
3482
3141
|
};
|
|
3483
3142
|
} catch (error) {
|
|
3484
|
-
|
|
3143
|
+
platform16?.logger.error("Failed to query historical metrics", error instanceof Error ? error : new Error(String(error)), { query });
|
|
3485
3144
|
return reply.code(500).send({
|
|
3486
3145
|
ok: false,
|
|
3487
3146
|
error: {
|
|
@@ -3495,6 +3154,8 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3495
3154
|
for (const path3 of metricsHeatmapPaths) {
|
|
3496
3155
|
fastify.get(path3, {
|
|
3497
3156
|
schema: {
|
|
3157
|
+
tags: ["Observability"],
|
|
3158
|
+
summary: "Metrics heatmap data",
|
|
3498
3159
|
querystring: {
|
|
3499
3160
|
type: "object",
|
|
3500
3161
|
properties: {
|
|
@@ -3570,7 +3231,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3570
3231
|
}
|
|
3571
3232
|
};
|
|
3572
3233
|
} catch (error) {
|
|
3573
|
-
|
|
3234
|
+
platform16?.logger.error("Failed to query heatmap data", error instanceof Error ? error : new Error(String(error)), { query });
|
|
3574
3235
|
return reply.code(500).send({
|
|
3575
3236
|
ok: false,
|
|
3576
3237
|
error: {
|
|
@@ -3581,310 +3242,11 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3581
3242
|
}
|
|
3582
3243
|
});
|
|
3583
3244
|
}
|
|
3584
|
-
for (const path3 of incidentsListPaths) {
|
|
3585
|
-
fastify.get(path3, {
|
|
3586
|
-
schema: {
|
|
3587
|
-
querystring: {
|
|
3588
|
-
type: "object",
|
|
3589
|
-
properties: {
|
|
3590
|
-
limit: { type: "integer", default: 50 },
|
|
3591
|
-
severity: {
|
|
3592
|
-
type: "string",
|
|
3593
|
-
enum: ["critical", "warning", "info"]
|
|
3594
|
-
},
|
|
3595
|
-
type: {
|
|
3596
|
-
type: "string",
|
|
3597
|
-
enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
|
|
3598
|
-
},
|
|
3599
|
-
from: { type: "integer" },
|
|
3600
|
-
to: { type: "integer" },
|
|
3601
|
-
includeResolved: { type: "boolean", default: false }
|
|
3602
|
-
}
|
|
3603
|
-
}
|
|
3604
|
-
}
|
|
3605
|
-
}, async (request, reply) => {
|
|
3606
|
-
if (!incidentStorage) {
|
|
3607
|
-
return reply.code(503).send({
|
|
3608
|
-
ok: false,
|
|
3609
|
-
error: {
|
|
3610
|
-
code: "INCIDENTS_NOT_CONFIGURED",
|
|
3611
|
-
message: "Incident storage is not configured"
|
|
3612
|
-
}
|
|
3613
|
-
});
|
|
3614
|
-
}
|
|
3615
|
-
try {
|
|
3616
|
-
const query = request.query;
|
|
3617
|
-
const incidents = await incidentStorage.queryIncidents({
|
|
3618
|
-
limit: query.limit ?? 50,
|
|
3619
|
-
severity: query.severity,
|
|
3620
|
-
type: query.type,
|
|
3621
|
-
from: query.from,
|
|
3622
|
-
to: query.to,
|
|
3623
|
-
includeResolved: query.includeResolved ?? false
|
|
3624
|
-
});
|
|
3625
|
-
const stats = await incidentStorage.getStats();
|
|
3626
|
-
return {
|
|
3627
|
-
ok: true,
|
|
3628
|
-
data: {
|
|
3629
|
-
incidents,
|
|
3630
|
-
summary: {
|
|
3631
|
-
total: stats.total,
|
|
3632
|
-
unresolved: stats.unresolved,
|
|
3633
|
-
bySeverity: stats.bySeverity,
|
|
3634
|
-
showing: incidents.length
|
|
3635
|
-
}
|
|
3636
|
-
}
|
|
3637
|
-
};
|
|
3638
|
-
} catch (error) {
|
|
3639
|
-
fastify.log.error({ err: error }, "Failed to list incidents");
|
|
3640
|
-
return reply.code(500).send({
|
|
3641
|
-
ok: false,
|
|
3642
|
-
error: {
|
|
3643
|
-
code: "INCIDENTS_LIST_ERROR",
|
|
3644
|
-
message: error instanceof Error ? error.message : "Failed to list incidents"
|
|
3645
|
-
}
|
|
3646
|
-
});
|
|
3647
|
-
}
|
|
3648
|
-
});
|
|
3649
|
-
}
|
|
3650
|
-
for (const path3 of incidentsDetailPaths) {
|
|
3651
|
-
fastify.get(path3, {
|
|
3652
|
-
schema: {
|
|
3653
|
-
params: {
|
|
3654
|
-
type: "object",
|
|
3655
|
-
properties: {
|
|
3656
|
-
id: { type: "string" }
|
|
3657
|
-
},
|
|
3658
|
-
required: ["id"]
|
|
3659
|
-
}
|
|
3660
|
-
}
|
|
3661
|
-
}, async (request, reply) => {
|
|
3662
|
-
if (!incidentStorage) {
|
|
3663
|
-
return reply.code(503).send({
|
|
3664
|
-
ok: false,
|
|
3665
|
-
error: {
|
|
3666
|
-
code: "INCIDENTS_NOT_CONFIGURED",
|
|
3667
|
-
message: "Incident storage is not configured"
|
|
3668
|
-
}
|
|
3669
|
-
});
|
|
3670
|
-
}
|
|
3671
|
-
const { id } = request.params;
|
|
3672
|
-
try {
|
|
3673
|
-
const incident = await incidentStorage.getIncident(id);
|
|
3674
|
-
if (!incident) {
|
|
3675
|
-
return reply.code(404).send({
|
|
3676
|
-
ok: false,
|
|
3677
|
-
error: {
|
|
3678
|
-
code: "INCIDENT_NOT_FOUND",
|
|
3679
|
-
message: `Incident ${id} not found`
|
|
3680
|
-
}
|
|
3681
|
-
});
|
|
3682
|
-
}
|
|
3683
|
-
if (platform19?.analytics) {
|
|
3684
|
-
platform19.analytics.track("incident.viewed", {
|
|
3685
|
-
incidentId: id,
|
|
3686
|
-
type: incident.type,
|
|
3687
|
-
severity: incident.severity,
|
|
3688
|
-
isResolved: !!incident.resolvedAt,
|
|
3689
|
-
hasAIAnalysis: !!incident.aiAnalysis
|
|
3690
|
-
}).catch(() => {
|
|
3691
|
-
});
|
|
3692
|
-
}
|
|
3693
|
-
return {
|
|
3694
|
-
ok: true,
|
|
3695
|
-
data: incident
|
|
3696
|
-
};
|
|
3697
|
-
} catch (error) {
|
|
3698
|
-
fastify.log.error({ err: error }, "Failed to get incident");
|
|
3699
|
-
return reply.code(500).send({
|
|
3700
|
-
ok: false,
|
|
3701
|
-
error: {
|
|
3702
|
-
code: "INCIDENT_GET_ERROR",
|
|
3703
|
-
message: error instanceof Error ? error.message : "Failed to get incident"
|
|
3704
|
-
}
|
|
3705
|
-
});
|
|
3706
|
-
}
|
|
3707
|
-
});
|
|
3708
|
-
}
|
|
3709
|
-
for (const path3 of incidentsCreatePaths) {
|
|
3710
|
-
fastify.post(path3, {
|
|
3711
|
-
schema: {
|
|
3712
|
-
body: {
|
|
3713
|
-
type: "object",
|
|
3714
|
-
properties: {
|
|
3715
|
-
type: {
|
|
3716
|
-
type: "string",
|
|
3717
|
-
enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
|
|
3718
|
-
},
|
|
3719
|
-
severity: {
|
|
3720
|
-
type: "string",
|
|
3721
|
-
enum: ["critical", "warning", "info"]
|
|
3722
|
-
},
|
|
3723
|
-
title: { type: "string" },
|
|
3724
|
-
details: { type: "string" },
|
|
3725
|
-
rootCause: { type: "string" },
|
|
3726
|
-
affectedServices: {
|
|
3727
|
-
type: "array",
|
|
3728
|
-
items: { type: "string" }
|
|
3729
|
-
},
|
|
3730
|
-
timestamp: { type: "number" },
|
|
3731
|
-
metadata: { type: "object" }
|
|
3732
|
-
},
|
|
3733
|
-
required: ["type", "severity", "title", "details"]
|
|
3734
|
-
}
|
|
3735
|
-
}
|
|
3736
|
-
}, async (request, reply) => {
|
|
3737
|
-
if (!incidentStorage) {
|
|
3738
|
-
return reply.code(503).send({
|
|
3739
|
-
ok: false,
|
|
3740
|
-
error: {
|
|
3741
|
-
code: "INCIDENT_STORAGE_UNAVAILABLE",
|
|
3742
|
-
message: "Incident storage is not initialized"
|
|
3743
|
-
}
|
|
3744
|
-
});
|
|
3745
|
-
}
|
|
3746
|
-
try {
|
|
3747
|
-
const payload = request.body;
|
|
3748
|
-
const incident = await incidentStorage.createIncident(payload);
|
|
3749
|
-
return {
|
|
3750
|
-
ok: true,
|
|
3751
|
-
data: incident,
|
|
3752
|
-
meta: {
|
|
3753
|
-
source: "incident-storage"
|
|
3754
|
-
}
|
|
3755
|
-
};
|
|
3756
|
-
} catch (error) {
|
|
3757
|
-
platform19?.logger.error("Failed to create incident", error instanceof Error ? error : new Error(String(error)));
|
|
3758
|
-
return reply.code(500).send({
|
|
3759
|
-
ok: false,
|
|
3760
|
-
error: {
|
|
3761
|
-
code: "INCIDENT_CREATE_ERROR",
|
|
3762
|
-
message: error instanceof Error ? error.message : "Failed to create incident"
|
|
3763
|
-
}
|
|
3764
|
-
});
|
|
3765
|
-
}
|
|
3766
|
-
});
|
|
3767
|
-
}
|
|
3768
|
-
for (const path3 of incidentsHistoryPaths) {
|
|
3769
|
-
fastify.get(path3, {
|
|
3770
|
-
schema: {
|
|
3771
|
-
querystring: {
|
|
3772
|
-
type: "object",
|
|
3773
|
-
properties: {
|
|
3774
|
-
limit: { type: "integer", minimum: 1, maximum: 500 },
|
|
3775
|
-
severity: { type: "string", enum: ["critical", "warning", "info"] },
|
|
3776
|
-
type: {
|
|
3777
|
-
type: "string",
|
|
3778
|
-
enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
|
|
3779
|
-
},
|
|
3780
|
-
from: { type: "integer" },
|
|
3781
|
-
to: { type: "integer" },
|
|
3782
|
-
includeResolved: { type: "boolean" }
|
|
3783
|
-
}
|
|
3784
|
-
}
|
|
3785
|
-
}
|
|
3786
|
-
}, async (request, reply) => {
|
|
3787
|
-
if (!incidentStorage) {
|
|
3788
|
-
return reply.code(503).send({
|
|
3789
|
-
ok: false,
|
|
3790
|
-
error: {
|
|
3791
|
-
code: "INCIDENT_STORAGE_UNAVAILABLE",
|
|
3792
|
-
message: "Incident storage is not initialized"
|
|
3793
|
-
}
|
|
3794
|
-
});
|
|
3795
|
-
}
|
|
3796
|
-
try {
|
|
3797
|
-
const query = request.query;
|
|
3798
|
-
const incidents = await incidentStorage.queryIncidents({
|
|
3799
|
-
limit: query.limit ?? 50,
|
|
3800
|
-
severity: query.severity,
|
|
3801
|
-
type: query.type,
|
|
3802
|
-
from: query.from,
|
|
3803
|
-
to: query.to,
|
|
3804
|
-
includeResolved: query.includeResolved ?? false
|
|
3805
|
-
});
|
|
3806
|
-
return {
|
|
3807
|
-
ok: true,
|
|
3808
|
-
data: incidents,
|
|
3809
|
-
meta: {
|
|
3810
|
-
source: "incident-storage",
|
|
3811
|
-
count: incidents.length
|
|
3812
|
-
}
|
|
3813
|
-
};
|
|
3814
|
-
} catch (error) {
|
|
3815
|
-
platform19?.logger.error("Failed to query incidents", error instanceof Error ? error : new Error(String(error)));
|
|
3816
|
-
return reply.code(500).send({
|
|
3817
|
-
ok: false,
|
|
3818
|
-
error: {
|
|
3819
|
-
code: "INCIDENT_QUERY_ERROR",
|
|
3820
|
-
message: error instanceof Error ? error.message : "Failed to query incidents"
|
|
3821
|
-
}
|
|
3822
|
-
});
|
|
3823
|
-
}
|
|
3824
|
-
});
|
|
3825
|
-
}
|
|
3826
|
-
for (const path3 of incidentsResolvePaths) {
|
|
3827
|
-
fastify.post(path3, {
|
|
3828
|
-
schema: {
|
|
3829
|
-
params: {
|
|
3830
|
-
type: "object",
|
|
3831
|
-
properties: {
|
|
3832
|
-
id: { type: "string" }
|
|
3833
|
-
},
|
|
3834
|
-
required: ["id"]
|
|
3835
|
-
},
|
|
3836
|
-
body: {
|
|
3837
|
-
type: "object",
|
|
3838
|
-
properties: {
|
|
3839
|
-
resolutionNotes: { type: "string" }
|
|
3840
|
-
}
|
|
3841
|
-
}
|
|
3842
|
-
}
|
|
3843
|
-
}, async (request, reply) => {
|
|
3844
|
-
if (!incidentStorage) {
|
|
3845
|
-
return reply.code(503).send({
|
|
3846
|
-
ok: false,
|
|
3847
|
-
error: {
|
|
3848
|
-
code: "INCIDENT_STORAGE_UNAVAILABLE",
|
|
3849
|
-
message: "Incident storage is not initialized"
|
|
3850
|
-
}
|
|
3851
|
-
});
|
|
3852
|
-
}
|
|
3853
|
-
try {
|
|
3854
|
-
const { id } = request.params;
|
|
3855
|
-
const body = request.body;
|
|
3856
|
-
const incident = await incidentStorage.resolveIncident(id, body.resolutionNotes);
|
|
3857
|
-
if (!incident) {
|
|
3858
|
-
return reply.code(404).send({
|
|
3859
|
-
ok: false,
|
|
3860
|
-
error: {
|
|
3861
|
-
code: "INCIDENT_NOT_FOUND",
|
|
3862
|
-
message: `Incident with id ${id} not found`
|
|
3863
|
-
}
|
|
3864
|
-
});
|
|
3865
|
-
}
|
|
3866
|
-
return {
|
|
3867
|
-
ok: true,
|
|
3868
|
-
data: incident,
|
|
3869
|
-
meta: {
|
|
3870
|
-
source: "incident-storage"
|
|
3871
|
-
}
|
|
3872
|
-
};
|
|
3873
|
-
} catch (error) {
|
|
3874
|
-
platform19?.logger.error("Failed to resolve incident", error instanceof Error ? error : new Error(String(error)));
|
|
3875
|
-
return reply.code(500).send({
|
|
3876
|
-
ok: false,
|
|
3877
|
-
error: {
|
|
3878
|
-
code: "INCIDENT_RESOLVE_ERROR",
|
|
3879
|
-
message: error instanceof Error ? error.message : "Failed to resolve incident"
|
|
3880
|
-
}
|
|
3881
|
-
});
|
|
3882
|
-
}
|
|
3883
|
-
});
|
|
3884
|
-
}
|
|
3885
3245
|
for (const path3 of insightsChatPaths) {
|
|
3886
3246
|
fastify.post(path3, {
|
|
3887
3247
|
schema: {
|
|
3248
|
+
tags: ["Observability"],
|
|
3249
|
+
summary: "AI-powered observability insights chat",
|
|
3888
3250
|
body: {
|
|
3889
3251
|
type: "object",
|
|
3890
3252
|
properties: {
|
|
@@ -3893,7 +3255,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3893
3255
|
type: "object",
|
|
3894
3256
|
properties: {
|
|
3895
3257
|
includeMetrics: { type: "boolean" },
|
|
3896
|
-
includeIncidents: { type: "boolean" },
|
|
3897
3258
|
includeHistory: { type: "boolean" },
|
|
3898
3259
|
timeRange: { type: "string", enum: ["1h", "6h", "24h", "7d"] },
|
|
3899
3260
|
plugins: { type: "array", items: { type: "string" } }
|
|
@@ -3904,7 +3265,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3904
3265
|
}
|
|
3905
3266
|
}
|
|
3906
3267
|
}, async (request, reply) => {
|
|
3907
|
-
if (!
|
|
3268
|
+
if (!platform16?.llm) {
|
|
3908
3269
|
return reply.code(503).send({
|
|
3909
3270
|
ok: false,
|
|
3910
3271
|
error: {
|
|
@@ -3917,7 +3278,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3917
3278
|
const body = request.body;
|
|
3918
3279
|
const contextConfig = {
|
|
3919
3280
|
includeMetrics: body.context?.includeMetrics ?? true,
|
|
3920
|
-
includeIncidents: body.context?.includeIncidents ?? true,
|
|
3921
3281
|
includeHistory: body.context?.includeHistory ?? true,
|
|
3922
3282
|
timeRange: body.context?.timeRange ?? "24h",
|
|
3923
3283
|
plugins: body.context?.plugins ?? []
|
|
@@ -3958,26 +3318,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3958
3318
|
fastify.log.warn({ err: metricsError }, "Failed to build metrics context for insights");
|
|
3959
3319
|
}
|
|
3960
3320
|
}
|
|
3961
|
-
if (contextConfig.includeIncidents && incidentStorage) {
|
|
3962
|
-
try {
|
|
3963
|
-
const incidents = await incidentStorage.queryIncidents({ limit: 10 });
|
|
3964
|
-
if (incidents.length > 0) {
|
|
3965
|
-
contextText += `
|
|
3966
|
-
## Recent Incidents (${incidents.length})
|
|
3967
|
-
`;
|
|
3968
|
-
for (const incident of incidents) {
|
|
3969
|
-
contextText += `- [${incident.severity.toUpperCase()}] ${incident.title}
|
|
3970
|
-
`;
|
|
3971
|
-
if (incident.details) {
|
|
3972
|
-
contextText += ` Details: ${incident.details.slice(0, 100)}${incident.details.length > 100 ? "..." : ""}
|
|
3973
|
-
`;
|
|
3974
|
-
}
|
|
3975
|
-
}
|
|
3976
|
-
}
|
|
3977
|
-
} catch (incidentError) {
|
|
3978
|
-
fastify.log.warn({ err: incidentError }, "Failed to fetch incidents for insights context");
|
|
3979
|
-
}
|
|
3980
|
-
}
|
|
3981
3321
|
const prompt = `You are an AI assistant analyzing a software platform's observability data.
|
|
3982
3322
|
|
|
3983
3323
|
${contextText}
|
|
@@ -3991,15 +3331,15 @@ Provide a clear, actionable response based on the data above. Include:
|
|
|
3991
3331
|
|
|
3992
3332
|
Be concise but thorough. Use markdown formatting.`;
|
|
3993
3333
|
fastify.log.debug({ question: body.question, contextLength: contextText.length }, "Calling LLM for insights");
|
|
3994
|
-
const result = await
|
|
3334
|
+
const result = await platform16.llm.complete(prompt, {
|
|
3995
3335
|
systemPrompt: "You are a DevOps and SRE expert assistant. Analyze system metrics and provide actionable insights. Be concise, technical, and helpful.",
|
|
3996
3336
|
temperature: 0.7,
|
|
3997
3337
|
maxTokens: 1e3
|
|
3998
3338
|
});
|
|
3999
3339
|
const totalTokens = result.usage.promptTokens + result.usage.completionTokens;
|
|
4000
3340
|
fastify.log.debug({ tokensUsed: totalTokens }, "LLM response received for insights");
|
|
4001
|
-
if (
|
|
4002
|
-
|
|
3341
|
+
if (platform16.analytics) {
|
|
3342
|
+
platform16.analytics.track("ai_insights.chat", {
|
|
4003
3343
|
questionLength: body.question.length,
|
|
4004
3344
|
contextIncluded: Object.keys(contextConfig).filter((k) => contextConfig[k]),
|
|
4005
3345
|
timeRange: contextConfig.timeRange,
|
|
@@ -4026,246 +3366,27 @@ Be concise but thorough. Use markdown formatting.`;
|
|
|
4026
3366
|
source: "llm-insights",
|
|
4027
3367
|
model: result.model
|
|
4028
3368
|
}
|
|
4029
|
-
};
|
|
4030
|
-
} catch (error) {
|
|
4031
|
-
|
|
4032
|
-
if (
|
|
4033
|
-
|
|
4034
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
4035
|
-
questionLength: request.body?.question?.length ?? 0
|
|
4036
|
-
}).catch(() => {
|
|
4037
|
-
});
|
|
4038
|
-
}
|
|
4039
|
-
return reply.code(500).send({
|
|
4040
|
-
ok: false,
|
|
4041
|
-
error: {
|
|
4042
|
-
code: "INSIGHTS_ERROR",
|
|
4043
|
-
message: error instanceof Error ? error.message : "Failed to generate insights"
|
|
4044
|
-
}
|
|
4045
|
-
});
|
|
4046
|
-
}
|
|
4047
|
-
});
|
|
4048
|
-
}
|
|
4049
|
-
for (const path3 of incidentsAnalyzePaths) {
|
|
4050
|
-
fastify.post(path3, async (request, reply) => {
|
|
4051
|
-
if (!incidentStorage) {
|
|
4052
|
-
return reply.code(503).send({
|
|
4053
|
-
ok: false,
|
|
4054
|
-
error: {
|
|
4055
|
-
code: "INCIDENTS_NOT_CONFIGURED",
|
|
4056
|
-
message: "Incident storage is not configured"
|
|
4057
|
-
}
|
|
4058
|
-
});
|
|
4059
|
-
}
|
|
4060
|
-
const { id } = request.params;
|
|
4061
|
-
try {
|
|
4062
|
-
const incident = await incidentStorage.getIncident(id);
|
|
4063
|
-
if (!incident) {
|
|
4064
|
-
return reply.code(404).send({
|
|
4065
|
-
ok: false,
|
|
4066
|
-
error: {
|
|
4067
|
-
code: "INCIDENT_NOT_FOUND",
|
|
4068
|
-
message: `Incident ${id} not found`
|
|
4069
|
-
}
|
|
4070
|
-
});
|
|
4071
|
-
}
|
|
4072
|
-
if (incident.aiAnalysis && incident.aiAnalyzedAt) {
|
|
4073
|
-
const ageMs = Date.now() - incident.aiAnalyzedAt;
|
|
4074
|
-
const ageMinutes = Math.floor(ageMs / 6e4);
|
|
4075
|
-
if (ageMs < 60 * 60 * 1e3) {
|
|
4076
|
-
return {
|
|
4077
|
-
ok: true,
|
|
4078
|
-
data: {
|
|
4079
|
-
...incident.aiAnalysis,
|
|
4080
|
-
cached: true,
|
|
4081
|
-
analyzedAt: incident.aiAnalyzedAt,
|
|
4082
|
-
ageMinutes
|
|
4083
|
-
}
|
|
4084
|
-
};
|
|
4085
|
-
}
|
|
4086
|
-
}
|
|
4087
|
-
const analyzer = new IncidentAnalyzer(
|
|
4088
|
-
{
|
|
4089
|
-
debug: process.env.NODE_ENV !== "production"
|
|
4090
|
-
},
|
|
4091
|
-
fastify.log
|
|
4092
|
-
);
|
|
4093
|
-
const analysis = await analyzer.analyze(incident);
|
|
4094
|
-
await incidentStorage.updateAIAnalysis(id, analysis);
|
|
4095
|
-
if (platform19?.analytics) {
|
|
4096
|
-
platform19.analytics.track("incident.analyzed", {
|
|
4097
|
-
incidentId: id,
|
|
4098
|
-
type: incident.type,
|
|
4099
|
-
severity: incident.severity,
|
|
4100
|
-
rootCausesCount: analysis.rootCauses.length,
|
|
4101
|
-
recommendationsCount: analysis.recommendations.length
|
|
4102
|
-
}).catch(() => {
|
|
4103
|
-
});
|
|
4104
|
-
}
|
|
4105
|
-
return {
|
|
4106
|
-
ok: true,
|
|
4107
|
-
data: {
|
|
4108
|
-
...analysis,
|
|
4109
|
-
cached: false
|
|
4110
|
-
}
|
|
4111
|
-
};
|
|
4112
|
-
} catch (error) {
|
|
4113
|
-
fastify.log.error({ err: error }, "Failed to analyze incident");
|
|
4114
|
-
if (platform19?.analytics) {
|
|
4115
|
-
platform19.analytics.track("incident.analysis_error", {
|
|
4116
|
-
incidentId: id,
|
|
4117
|
-
error: error instanceof Error ? error.message : "Unknown error"
|
|
4118
|
-
}).catch(() => {
|
|
4119
|
-
});
|
|
4120
|
-
}
|
|
4121
|
-
return reply.code(500).send({
|
|
4122
|
-
ok: false,
|
|
4123
|
-
error: {
|
|
4124
|
-
code: "ANALYSIS_ERROR",
|
|
4125
|
-
message: error instanceof Error ? error.message : "Failed to analyze incident"
|
|
4126
|
-
}
|
|
4127
|
-
});
|
|
4128
|
-
}
|
|
4129
|
-
});
|
|
4130
|
-
}
|
|
4131
|
-
const testCreateIncidentPaths = resolvePaths(basePath, "/test/create-incident");
|
|
4132
|
-
for (const path3 of testCreateIncidentPaths) {
|
|
4133
|
-
fastify.post(path3, async (request, reply) => {
|
|
4134
|
-
if (!incidentStorage) {
|
|
4135
|
-
return reply.code(500).send({
|
|
4136
|
-
ok: false,
|
|
4137
|
-
error: { code: "INCIDENT_STORAGE_NOT_INITIALIZED", message: "Incident storage not initialized" }
|
|
4138
|
-
});
|
|
4139
|
-
}
|
|
4140
|
-
try {
|
|
4141
|
-
const {
|
|
4142
|
-
type = "custom",
|
|
4143
|
-
severity = "warning",
|
|
4144
|
-
title = "Test Incident",
|
|
4145
|
-
details,
|
|
4146
|
-
relatedData
|
|
4147
|
-
} = request.body;
|
|
4148
|
-
const incident = await incidentStorage.createIncident({
|
|
4149
|
-
type,
|
|
4150
|
-
severity,
|
|
4151
|
-
title,
|
|
4152
|
-
details: details || "This is a test incident created manually for testing purposes.",
|
|
4153
|
-
timestamp: Date.now(),
|
|
4154
|
-
metadata: {
|
|
4155
|
-
testMode: true,
|
|
4156
|
-
createdVia: "test-endpoint"
|
|
4157
|
-
},
|
|
4158
|
-
relatedData: relatedData || {
|
|
4159
|
-
timeline: [
|
|
4160
|
-
{
|
|
4161
|
-
timestamp: Date.now(),
|
|
4162
|
-
event: "Test incident created via /test/create-incident",
|
|
4163
|
-
source: "manual"
|
|
4164
|
-
}
|
|
4165
|
-
]
|
|
4166
|
-
}
|
|
4167
|
-
});
|
|
4168
|
-
fastify.log.info({ id: incident.id }, "Test incident created");
|
|
4169
|
-
return { ok: true, data: incident };
|
|
4170
|
-
} catch (error) {
|
|
4171
|
-
fastify.log.error({ err: error }, "Failed to create test incident");
|
|
4172
|
-
return reply.code(500).send({
|
|
4173
|
-
ok: false,
|
|
4174
|
-
error: {
|
|
4175
|
-
code: "TEST_INCIDENT_CREATION_FAILED",
|
|
4176
|
-
message: error instanceof Error ? error.message : "Failed to create test incident"
|
|
4177
|
-
}
|
|
4178
|
-
});
|
|
4179
|
-
}
|
|
4180
|
-
});
|
|
4181
|
-
}
|
|
4182
|
-
const testTriggerErrorsPaths = resolvePaths(basePath, "/test/trigger-errors");
|
|
4183
|
-
for (const path3 of testTriggerErrorsPaths) {
|
|
4184
|
-
fastify.get(path3, async (request, reply) => {
|
|
4185
|
-
const { count = 10 } = request.query;
|
|
4186
|
-
const errorCount = Math.min(Math.max(1, parseInt(count, 10) || 10), 100);
|
|
4187
|
-
fastify.log.info(`Triggering ${errorCount} test errors`);
|
|
4188
|
-
for (let i = 0; i < errorCount; i++) {
|
|
4189
|
-
platform19?.logger.error(`Test error ${i + 1}/${errorCount}`, void 0, {
|
|
4190
|
-
testMode: true,
|
|
4191
|
-
errorNumber: i + 1,
|
|
4192
|
-
totalErrors: errorCount
|
|
4193
|
-
});
|
|
4194
|
-
if (i === errorCount - 1) {
|
|
4195
|
-
return reply.code(500).send({
|
|
4196
|
-
ok: false,
|
|
4197
|
-
error: {
|
|
4198
|
-
code: "TEST_ERROR_TRIGGERED",
|
|
4199
|
-
message: `Generated ${errorCount} test errors. Check /observability/incidents in ~30 seconds for auto-created incident.`
|
|
4200
|
-
}
|
|
3369
|
+
};
|
|
3370
|
+
} catch (error) {
|
|
3371
|
+
platform16?.logger.error("Failed to generate insights", error instanceof Error ? error : new Error(String(error)));
|
|
3372
|
+
if (platform16?.analytics) {
|
|
3373
|
+
platform16.analytics.track("ai_insights.error", {
|
|
3374
|
+
error: error instanceof Error ? error.message : "Unknown error",
|
|
3375
|
+
questionLength: request.body?.question?.length ?? 0
|
|
3376
|
+
}).catch(() => {
|
|
4201
3377
|
});
|
|
4202
3378
|
}
|
|
4203
|
-
|
|
4204
|
-
|
|
4205
|
-
|
|
4206
|
-
|
|
4207
|
-
|
|
4208
|
-
|
|
4209
|
-
}
|
|
4210
|
-
const testSimulateLatencyPaths = resolvePaths(basePath, "/test/simulate-latency");
|
|
4211
|
-
for (const path3 of testSimulateLatencyPaths) {
|
|
4212
|
-
fastify.get(path3, async (request, reply) => {
|
|
4213
|
-
const { delay = 2e3 } = request.query;
|
|
4214
|
-
const delayMs = Math.min(Math.max(100, parseInt(delay, 10) || 2e3), 1e4);
|
|
4215
|
-
fastify.log.info(`Simulating ${delayMs}ms latency`);
|
|
4216
|
-
await new Promise((resolve3) => {
|
|
4217
|
-
setTimeout(resolve3, delayMs);
|
|
4218
|
-
});
|
|
4219
|
-
return {
|
|
4220
|
-
ok: true,
|
|
4221
|
-
message: `Simulated ${delayMs}ms latency. Call this endpoint multiple times to trigger latency_spike incident.`,
|
|
4222
|
-
actualDelay: delayMs
|
|
4223
|
-
};
|
|
4224
|
-
});
|
|
4225
|
-
}
|
|
4226
|
-
const testBulkErrorsPaths = resolvePaths(basePath, "/test/bulk-errors");
|
|
4227
|
-
for (const path3 of testBulkErrorsPaths) {
|
|
4228
|
-
fastify.post(path3, async (request, reply) => {
|
|
4229
|
-
const { successCount = 10, errorCount = 50 } = request.body;
|
|
4230
|
-
fastify.log.info({ successCount, errorCount }, "Generating bulk requests for incident testing");
|
|
4231
|
-
for (let i = 0; i < successCount; i++) {
|
|
4232
|
-
platform19?.logger.info(`Bulk test - success ${i + 1}/${successCount}`);
|
|
4233
|
-
}
|
|
4234
|
-
const testEndpoints = [
|
|
4235
|
-
"POST /api/v1/test/endpoint-a",
|
|
4236
|
-
"GET /api/v1/test/endpoint-b",
|
|
4237
|
-
"PUT /api/v1/test/endpoint-c",
|
|
4238
|
-
"DELETE /api/v1/test/endpoint-d",
|
|
4239
|
-
"POST /api/v1/test/endpoint-e"
|
|
4240
|
-
];
|
|
4241
|
-
for (let i = 0; i < errorCount; i++) {
|
|
4242
|
-
const endpoint = testEndpoints[i % testEndpoints.length];
|
|
4243
|
-
const errorTypes = [
|
|
4244
|
-
"Connection timeout",
|
|
4245
|
-
"Validation failed",
|
|
4246
|
-
"Database query error",
|
|
4247
|
-
"Authentication failed",
|
|
4248
|
-
"Rate limit exceeded"
|
|
4249
|
-
];
|
|
4250
|
-
const errorType = errorTypes[i % errorTypes.length];
|
|
4251
|
-
platform19?.logger.error(`${errorType}: ${endpoint}`, void 0, {
|
|
4252
|
-
testMode: true,
|
|
4253
|
-
bulkTest: true,
|
|
4254
|
-
errorIndex: i,
|
|
4255
|
-
endpoint
|
|
3379
|
+
return reply.code(500).send({
|
|
3380
|
+
ok: false,
|
|
3381
|
+
error: {
|
|
3382
|
+
code: "INSIGHTS_ERROR",
|
|
3383
|
+
message: error instanceof Error ? error.message : "Failed to generate insights"
|
|
3384
|
+
}
|
|
4256
3385
|
});
|
|
4257
3386
|
}
|
|
4258
|
-
const totalRequests = successCount + errorCount;
|
|
4259
|
-
const errorRate = errorCount / totalRequests * 100;
|
|
4260
|
-
return {
|
|
4261
|
-
ok: true,
|
|
4262
|
-
message: `Generated ${totalRequests} requests (${successCount} success, ${errorCount} errors)`,
|
|
4263
|
-
errorRate: `${errorRate.toFixed(1)}%`,
|
|
4264
|
-
note: "Incident should be auto-created in next detection cycle (~30s)"
|
|
4265
|
-
};
|
|
4266
3387
|
});
|
|
4267
3388
|
}
|
|
4268
|
-
fastify.log.info("Observability routes registered
|
|
3389
|
+
fastify.log.info("Observability routes registered");
|
|
4269
3390
|
}
|
|
4270
3391
|
async function registerAnalyticsRoutes(fastify, config) {
|
|
4271
3392
|
const basePath = normalizeBasePath(config.basePath);
|
|
@@ -5598,7 +4719,7 @@ function generateFallbackSummary(stats, logs) {
|
|
|
5598
4719
|
return summary;
|
|
5599
4720
|
}
|
|
5600
4721
|
async function registerLogRoutes(server, config, eventHub) {
|
|
5601
|
-
server.get("/api/v1/logs", async (request, reply) => {
|
|
4722
|
+
server.get("/api/v1/logs", { schema: { tags: ["Logs"], summary: "Query logs with filters" } }, async (request, reply) => {
|
|
5602
4723
|
try {
|
|
5603
4724
|
const limit = request.query.limit ? parseInt(request.query.limit, 10) : 100;
|
|
5604
4725
|
const offset = request.query.offset ? parseInt(request.query.offset, 10) : 0;
|
|
@@ -5656,6 +4777,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5656
4777
|
});
|
|
5657
4778
|
server.get(
|
|
5658
4779
|
"/api/v1/logs/:id",
|
|
4780
|
+
{ schema: { tags: ["Logs"], summary: "Get log entry by ID" } },
|
|
5659
4781
|
async (request, reply) => {
|
|
5660
4782
|
try {
|
|
5661
4783
|
const log = await platform.logs.getById(request.params.id);
|
|
@@ -5689,6 +4811,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5689
4811
|
);
|
|
5690
4812
|
server.get(
|
|
5691
4813
|
"/api/v1/logs/:id/related",
|
|
4814
|
+
{ schema: { tags: ["Logs"], summary: "Get logs related to a specific entry" } },
|
|
5692
4815
|
async (request, reply) => {
|
|
5693
4816
|
try {
|
|
5694
4817
|
const log = await platform.logs.getById(request.params.id);
|
|
@@ -5717,7 +4840,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5717
4840
|
}
|
|
5718
4841
|
}
|
|
5719
4842
|
);
|
|
5720
|
-
server.get("/api/v1/logs/stream", async (request, reply) => {
|
|
4843
|
+
server.get("/api/v1/logs/stream", { schema: { hide: true } }, async (request, reply) => {
|
|
5721
4844
|
const caps = platform.logs.getCapabilities();
|
|
5722
4845
|
if (!caps.hasStreaming) {
|
|
5723
4846
|
return reply.code(503).send({
|
|
@@ -5775,7 +4898,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5775
4898
|
await new Promise(() => {
|
|
5776
4899
|
});
|
|
5777
4900
|
});
|
|
5778
|
-
server.get("/api/v1/logs/stats", async (request, reply) => {
|
|
4901
|
+
server.get("/api/v1/logs/stats", { schema: { tags: ["Logs"], summary: "Get log storage statistics" } }, async (request, reply) => {
|
|
5779
4902
|
try {
|
|
5780
4903
|
const stats = await platform.logs.getStats();
|
|
5781
4904
|
const caps = platform.logs.getCapabilities();
|
|
@@ -5805,7 +4928,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5805
4928
|
});
|
|
5806
4929
|
}
|
|
5807
4930
|
});
|
|
5808
|
-
server.post("/api/v1/logs/summarize", async (request, reply) => {
|
|
4931
|
+
server.post("/api/v1/logs/summarize", { schema: { tags: ["Logs"], summary: "AI-powered log summarization" } }, async (request, reply) => {
|
|
5809
4932
|
const { timeRange, filters, groupBy, question, includeContext } = request.body;
|
|
5810
4933
|
try {
|
|
5811
4934
|
const query = {
|
|
@@ -6197,7 +5320,7 @@ async function registerDebugRoutes(fastify, config) {
|
|
|
6197
5320
|
}
|
|
6198
5321
|
|
|
6199
5322
|
// src/services/historical-metrics.ts
|
|
6200
|
-
var
|
|
5323
|
+
var DEFAULT_CONFIG = {
|
|
6201
5324
|
intervalMs: 5e3,
|
|
6202
5325
|
maxPoints: {
|
|
6203
5326
|
"1m": 12,
|
|
@@ -6216,7 +5339,7 @@ var HistoricalMetricsCollector = class {
|
|
|
6216
5339
|
startTimeMs = Date.now();
|
|
6217
5340
|
constructor(cache, config = {}, logger = console) {
|
|
6218
5341
|
this.cache = cache;
|
|
6219
|
-
this.config = { ...
|
|
5342
|
+
this.config = { ...DEFAULT_CONFIG, ...config, maxPoints: { ...DEFAULT_CONFIG.maxPoints, ...config.maxPoints } };
|
|
6220
5343
|
this.logger = logger;
|
|
6221
5344
|
}
|
|
6222
5345
|
/**
|
|
@@ -6516,754 +5639,6 @@ var HistoricalMetricsCollector = class {
|
|
|
6516
5639
|
}
|
|
6517
5640
|
}
|
|
6518
5641
|
};
|
|
6519
|
-
var DEFAULT_CONFIG3 = {
|
|
6520
|
-
ttlMs: 30 * 24 * 60 * 60 * 1e3,
|
|
6521
|
-
// 30 days (not used with SQLite, kept for compatibility)
|
|
6522
|
-
maxIncidents: 1e3,
|
|
6523
|
-
// Not enforced with SQLite, use retention policy instead
|
|
6524
|
-
debug: false
|
|
6525
|
-
};
|
|
6526
|
-
var IncidentStorage = class {
|
|
6527
|
-
db;
|
|
6528
|
-
config;
|
|
6529
|
-
logger;
|
|
6530
|
-
constructor(db, config = {}, logger = console) {
|
|
6531
|
-
this.db = db;
|
|
6532
|
-
this.config = { ...DEFAULT_CONFIG3, ...config };
|
|
6533
|
-
this.logger = logger;
|
|
6534
|
-
}
|
|
6535
|
-
/**
|
|
6536
|
-
* Create a new incident record
|
|
6537
|
-
*/
|
|
6538
|
-
async createIncident(payload) {
|
|
6539
|
-
const id = `inc-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
|
|
6540
|
-
const timestamp = payload.timestamp ?? Date.now();
|
|
6541
|
-
if (!payload.type || !payload.severity || !payload.title) {
|
|
6542
|
-
throw new Error("Incident must have type, severity, and title");
|
|
6543
|
-
}
|
|
6544
|
-
const relatedLogsCount = payload.relatedData?.logs?.errorCount ?? 0;
|
|
6545
|
-
const relatedLogsSample = payload.relatedData?.logs ? JSON.stringify(payload.relatedData.logs) : null;
|
|
6546
|
-
const relatedMetrics = payload.relatedData?.metrics ? JSON.stringify(payload.relatedData.metrics) : null;
|
|
6547
|
-
const timeline = payload.relatedData?.timeline ? JSON.stringify(payload.relatedData.timeline) : null;
|
|
6548
|
-
await this.db.query(
|
|
6549
|
-
`INSERT INTO incidents (
|
|
6550
|
-
id, type, severity, title, details, timestamp,
|
|
6551
|
-
resolved_at, resolution_notes,
|
|
6552
|
-
affected_services, metadata,
|
|
6553
|
-
related_logs_count, related_logs_sample, related_metrics, timeline
|
|
6554
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
6555
|
-
[
|
|
6556
|
-
id,
|
|
6557
|
-
payload.type,
|
|
6558
|
-
payload.severity,
|
|
6559
|
-
payload.title,
|
|
6560
|
-
payload.details,
|
|
6561
|
-
timestamp,
|
|
6562
|
-
payload.resolvedAt ?? null,
|
|
6563
|
-
payload.resolutionNotes ?? null,
|
|
6564
|
-
payload.affectedServices ? JSON.stringify(payload.affectedServices) : null,
|
|
6565
|
-
payload.metadata ? JSON.stringify(payload.metadata) : null,
|
|
6566
|
-
relatedLogsCount,
|
|
6567
|
-
relatedLogsSample,
|
|
6568
|
-
relatedMetrics,
|
|
6569
|
-
timeline
|
|
6570
|
-
]
|
|
6571
|
-
);
|
|
6572
|
-
this.log("info", "Incident created", {
|
|
6573
|
-
id,
|
|
6574
|
-
type: payload.type,
|
|
6575
|
-
severity: payload.severity
|
|
6576
|
-
});
|
|
6577
|
-
const created = await this.getIncident(id);
|
|
6578
|
-
if (!created) {
|
|
6579
|
-
throw new Error("Failed to retrieve created incident");
|
|
6580
|
-
}
|
|
6581
|
-
return created;
|
|
6582
|
-
}
|
|
6583
|
-
/**
|
|
6584
|
-
* Get incident by ID
|
|
6585
|
-
*/
|
|
6586
|
-
async getIncident(id) {
|
|
6587
|
-
const result = await this.db.query(
|
|
6588
|
-
"SELECT * FROM incidents WHERE id = ?",
|
|
6589
|
-
[id]
|
|
6590
|
-
);
|
|
6591
|
-
if (result.rows.length === 0) {
|
|
6592
|
-
return null;
|
|
6593
|
-
}
|
|
6594
|
-
return this.rowToIncident(result.rows[0]);
|
|
6595
|
-
}
|
|
6596
|
-
/**
|
|
6597
|
-
* Query incidents with filters
|
|
6598
|
-
*/
|
|
6599
|
-
async queryIncidents(options = {}) {
|
|
6600
|
-
const {
|
|
6601
|
-
limit = 50,
|
|
6602
|
-
severity,
|
|
6603
|
-
type,
|
|
6604
|
-
from,
|
|
6605
|
-
to,
|
|
6606
|
-
includeResolved = false
|
|
6607
|
-
} = options;
|
|
6608
|
-
const conditions = [];
|
|
6609
|
-
const params = [];
|
|
6610
|
-
if (severity) {
|
|
6611
|
-
const severityList = Array.isArray(severity) ? severity : [severity];
|
|
6612
|
-
conditions.push(`severity IN (${severityList.map(() => "?").join(", ")})`);
|
|
6613
|
-
params.push(...severityList);
|
|
6614
|
-
}
|
|
6615
|
-
if (type) {
|
|
6616
|
-
const typeList = Array.isArray(type) ? type : [type];
|
|
6617
|
-
conditions.push(`type IN (${typeList.map(() => "?").join(", ")})`);
|
|
6618
|
-
params.push(...typeList);
|
|
6619
|
-
}
|
|
6620
|
-
if (from) {
|
|
6621
|
-
conditions.push("timestamp >= ?");
|
|
6622
|
-
params.push(from);
|
|
6623
|
-
}
|
|
6624
|
-
if (to) {
|
|
6625
|
-
conditions.push("timestamp <= ?");
|
|
6626
|
-
params.push(to);
|
|
6627
|
-
}
|
|
6628
|
-
if (!includeResolved) {
|
|
6629
|
-
conditions.push("resolved_at IS NULL");
|
|
6630
|
-
}
|
|
6631
|
-
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
6632
|
-
const sql = `
|
|
6633
|
-
SELECT * FROM incidents
|
|
6634
|
-
${whereClause}
|
|
6635
|
-
ORDER BY timestamp DESC
|
|
6636
|
-
LIMIT ?
|
|
6637
|
-
`;
|
|
6638
|
-
const result = await this.db.query(sql, [...params, limit]);
|
|
6639
|
-
return result.rows.map((row) => this.rowToIncident(row));
|
|
6640
|
-
}
|
|
6641
|
-
/**
|
|
6642
|
-
* Resolve an incident
|
|
6643
|
-
*/
|
|
6644
|
-
async resolveIncident(id, resolutionNotes) {
|
|
6645
|
-
const resolvedAt = Date.now();
|
|
6646
|
-
const result = await this.db.query(
|
|
6647
|
-
`UPDATE incidents
|
|
6648
|
-
SET resolved_at = ?, resolution_notes = ?
|
|
6649
|
-
WHERE id = ?`,
|
|
6650
|
-
[resolvedAt, resolutionNotes ?? null, id]
|
|
6651
|
-
);
|
|
6652
|
-
if (result.rowCount === 0) {
|
|
6653
|
-
return null;
|
|
6654
|
-
}
|
|
6655
|
-
this.log("info", "Incident resolved", { id, resolvedAt });
|
|
6656
|
-
const incident = await this.getIncident(id);
|
|
6657
|
-
if (incident && platform.analytics) {
|
|
6658
|
-
const durationMs = resolvedAt - incident.timestamp;
|
|
6659
|
-
const durationMinutes = Math.floor(durationMs / 6e4);
|
|
6660
|
-
platform.analytics.track("incident.resolved", {
|
|
6661
|
-
incidentId: id,
|
|
6662
|
-
type: incident.type,
|
|
6663
|
-
severity: incident.severity,
|
|
6664
|
-
durationMs,
|
|
6665
|
-
durationMinutes,
|
|
6666
|
-
hasResolutionNotes: !!resolutionNotes,
|
|
6667
|
-
wasAnalyzed: !!incident.aiAnalysis
|
|
6668
|
-
}).catch(() => {
|
|
6669
|
-
});
|
|
6670
|
-
}
|
|
6671
|
-
return incident;
|
|
6672
|
-
}
|
|
6673
|
-
/**
|
|
6674
|
-
* Delete an incident
|
|
6675
|
-
*/
|
|
6676
|
-
async deleteIncident(id) {
|
|
6677
|
-
const incident = await this.getIncident(id);
|
|
6678
|
-
const result = await this.db.query("DELETE FROM incidents WHERE id = ?", [id]);
|
|
6679
|
-
const deleted = result.rowCount > 0;
|
|
6680
|
-
if (deleted) {
|
|
6681
|
-
this.log("info", "Incident deleted", { id });
|
|
6682
|
-
if (incident && platform.analytics) {
|
|
6683
|
-
platform.analytics.track("incident.deleted", {
|
|
6684
|
-
incidentId: id,
|
|
6685
|
-
type: incident.type,
|
|
6686
|
-
severity: incident.severity,
|
|
6687
|
-
wasResolved: !!incident.resolvedAt,
|
|
6688
|
-
wasAnalyzed: !!incident.aiAnalysis
|
|
6689
|
-
}).catch(() => {
|
|
6690
|
-
});
|
|
6691
|
-
}
|
|
6692
|
-
}
|
|
6693
|
-
return deleted;
|
|
6694
|
-
}
|
|
6695
|
-
/**
|
|
6696
|
-
* Get statistics about stored incidents
|
|
6697
|
-
*/
|
|
6698
|
-
async getStats() {
|
|
6699
|
-
const totalResult = await this.db.query(`
|
|
6700
|
-
SELECT
|
|
6701
|
-
COUNT(*) as total,
|
|
6702
|
-
MIN(timestamp) as oldest,
|
|
6703
|
-
MAX(timestamp) as newest
|
|
6704
|
-
FROM incidents
|
|
6705
|
-
`);
|
|
6706
|
-
const { total, oldest, newest } = totalResult.rows[0];
|
|
6707
|
-
const severityResult = await this.db.query("SELECT severity, COUNT(*) as count FROM incidents GROUP BY severity");
|
|
6708
|
-
const bySeverity = {
|
|
6709
|
-
critical: 0,
|
|
6710
|
-
warning: 0,
|
|
6711
|
-
info: 0
|
|
6712
|
-
};
|
|
6713
|
-
for (const row of severityResult.rows) {
|
|
6714
|
-
bySeverity[row.severity] = row.count;
|
|
6715
|
-
}
|
|
6716
|
-
const typeResult = await this.db.query("SELECT type, COUNT(*) as count FROM incidents GROUP BY type");
|
|
6717
|
-
const byType = {};
|
|
6718
|
-
for (const row of typeResult.rows) {
|
|
6719
|
-
byType[row.type] = row.count;
|
|
6720
|
-
}
|
|
6721
|
-
const resolvedResult = await this.db.query(`
|
|
6722
|
-
SELECT
|
|
6723
|
-
SUM(CASE WHEN resolved_at IS NOT NULL THEN 1 ELSE 0 END) as resolved,
|
|
6724
|
-
SUM(CASE WHEN resolved_at IS NULL THEN 1 ELSE 0 END) as unresolved
|
|
6725
|
-
FROM incidents
|
|
6726
|
-
`);
|
|
6727
|
-
const { resolved, unresolved } = resolvedResult.rows[0];
|
|
6728
|
-
return {
|
|
6729
|
-
total,
|
|
6730
|
-
bySeverity,
|
|
6731
|
-
byType,
|
|
6732
|
-
resolved: resolved ?? 0,
|
|
6733
|
-
unresolved: unresolved ?? 0,
|
|
6734
|
-
oldestTimestamp: oldest,
|
|
6735
|
-
newestTimestamp: newest
|
|
6736
|
-
};
|
|
6737
|
-
}
|
|
6738
|
-
/**
|
|
6739
|
-
* Clear all incidents (admin function)
|
|
6740
|
-
*/
|
|
6741
|
-
async clearAll() {
|
|
6742
|
-
await this.db.query("DELETE FROM incidents");
|
|
6743
|
-
this.log("warn", "All incidents cleared");
|
|
6744
|
-
}
|
|
6745
|
-
/**
|
|
6746
|
-
* Update AI analysis for an incident
|
|
6747
|
-
*/
|
|
6748
|
-
async updateAIAnalysis(id, analysis) {
|
|
6749
|
-
await this.db.query(
|
|
6750
|
-
`UPDATE incidents
|
|
6751
|
-
SET ai_analysis = ?, ai_analyzed_at = ?
|
|
6752
|
-
WHERE id = ?`,
|
|
6753
|
-
[JSON.stringify(analysis), Date.now(), id]
|
|
6754
|
-
);
|
|
6755
|
-
this.log("debug", "AI analysis updated", { id });
|
|
6756
|
-
}
|
|
6757
|
-
/**
|
|
6758
|
-
* Convert database row to Incident object
|
|
6759
|
-
* @private
|
|
6760
|
-
*/
|
|
6761
|
-
rowToIncident(row) {
|
|
6762
|
-
const affectedServices = row.affected_services ? JSON.parse(row.affected_services) : void 0;
|
|
6763
|
-
const metadata = row.metadata ? JSON.parse(row.metadata) : void 0;
|
|
6764
|
-
const aiAnalysis = row.ai_analysis ? JSON.parse(row.ai_analysis) : void 0;
|
|
6765
|
-
const relatedData = row.related_logs_sample || row.related_metrics || row.timeline ? {
|
|
6766
|
-
logs: row.related_logs_sample ? JSON.parse(row.related_logs_sample) : void 0,
|
|
6767
|
-
metrics: row.related_metrics ? JSON.parse(row.related_metrics) : void 0,
|
|
6768
|
-
timeline: row.timeline ? JSON.parse(row.timeline) : void 0
|
|
6769
|
-
} : void 0;
|
|
6770
|
-
return {
|
|
6771
|
-
id: row.id,
|
|
6772
|
-
type: row.type,
|
|
6773
|
-
severity: row.severity,
|
|
6774
|
-
title: row.title,
|
|
6775
|
-
details: row.details,
|
|
6776
|
-
timestamp: row.timestamp,
|
|
6777
|
-
resolvedAt: row.resolved_at ?? void 0,
|
|
6778
|
-
resolutionNotes: row.resolution_notes ?? void 0,
|
|
6779
|
-
affectedServices,
|
|
6780
|
-
metadata,
|
|
6781
|
-
relatedData,
|
|
6782
|
-
aiAnalysis,
|
|
6783
|
-
aiAnalyzedAt: row.ai_analyzed_at ?? void 0
|
|
6784
|
-
};
|
|
6785
|
-
}
|
|
6786
|
-
log(level, message, meta) {
|
|
6787
|
-
if (level === "debug" && !this.config.debug) {
|
|
6788
|
-
return;
|
|
6789
|
-
}
|
|
6790
|
-
if (this.logger[level]) {
|
|
6791
|
-
this.logger[level](`[IncidentStorage] ${message}`, meta);
|
|
6792
|
-
} else {
|
|
6793
|
-
console.log(`[IncidentStorage] [${level}] ${message}`, meta);
|
|
6794
|
-
}
|
|
6795
|
-
}
|
|
6796
|
-
};
|
|
6797
|
-
var DEFAULT_THRESHOLDS = {
|
|
6798
|
-
errorRateWarning: 5,
|
|
6799
|
-
errorRateCritical: 10,
|
|
6800
|
-
latencyP99Warning: 2e3,
|
|
6801
|
-
latencyP99Critical: 5e3,
|
|
6802
|
-
latencyP95Warning: 1e3,
|
|
6803
|
-
minRequestsForDetection: 10,
|
|
6804
|
-
pluginErrorRateWarning: 10,
|
|
6805
|
-
pluginErrorRateCritical: 25
|
|
6806
|
-
};
|
|
6807
|
-
var DEFAULT_CONFIG4 = {
|
|
6808
|
-
intervalMs: 3e4,
|
|
6809
|
-
// 30 seconds
|
|
6810
|
-
thresholds: DEFAULT_THRESHOLDS,
|
|
6811
|
-
cooldownMs: 5 * 60 * 1e3,
|
|
6812
|
-
// 5 minutes
|
|
6813
|
-
debug: false
|
|
6814
|
-
};
|
|
6815
|
-
var IncidentDetector = class {
|
|
6816
|
-
incidentStorage;
|
|
6817
|
-
config;
|
|
6818
|
-
logger;
|
|
6819
|
-
intervalHandle = null;
|
|
6820
|
-
recentIncidents = [];
|
|
6821
|
-
isRunning = false;
|
|
6822
|
-
// Metrics history for "before" comparison (last 10 snapshots)
|
|
6823
|
-
metricsHistory = [];
|
|
6824
|
-
constructor(incidentStorage, config = {}, logger = console) {
|
|
6825
|
-
this.incidentStorage = incidentStorage;
|
|
6826
|
-
this.config = {
|
|
6827
|
-
...DEFAULT_CONFIG4,
|
|
6828
|
-
...config,
|
|
6829
|
-
thresholds: { ...DEFAULT_THRESHOLDS, ...config.thresholds }
|
|
6830
|
-
};
|
|
6831
|
-
this.logger = logger;
|
|
6832
|
-
}
|
|
6833
|
-
/**
|
|
6834
|
-
* Start automatic detection
|
|
6835
|
-
*/
|
|
6836
|
-
start() {
|
|
6837
|
-
if (this.isRunning) {
|
|
6838
|
-
this.log("warn", "Detector already running");
|
|
6839
|
-
return;
|
|
6840
|
-
}
|
|
6841
|
-
this.isRunning = true;
|
|
6842
|
-
this.log("info", "Starting incident detector", {
|
|
6843
|
-
intervalMs: this.config.intervalMs,
|
|
6844
|
-
thresholds: this.config.thresholds
|
|
6845
|
-
});
|
|
6846
|
-
this.runDetection().catch((err) => {
|
|
6847
|
-
this.log("error", "Initial detection failed", { error: err.message });
|
|
6848
|
-
});
|
|
6849
|
-
this.intervalHandle = setInterval(() => {
|
|
6850
|
-
this.runDetection().catch((err) => {
|
|
6851
|
-
this.log("error", "Detection cycle failed", { error: err.message });
|
|
6852
|
-
});
|
|
6853
|
-
}, this.config.intervalMs);
|
|
6854
|
-
}
|
|
6855
|
-
/**
|
|
6856
|
-
* Stop automatic detection
|
|
6857
|
-
*/
|
|
6858
|
-
stop() {
|
|
6859
|
-
if (this.intervalHandle) {
|
|
6860
|
-
clearInterval(this.intervalHandle);
|
|
6861
|
-
this.intervalHandle = null;
|
|
6862
|
-
}
|
|
6863
|
-
this.isRunning = false;
|
|
6864
|
-
this.log("info", "Incident detector stopped");
|
|
6865
|
-
}
|
|
6866
|
-
/**
|
|
6867
|
-
* Run a single detection cycle
|
|
6868
|
-
*/
|
|
6869
|
-
async runDetection() {
|
|
6870
|
-
const metrics = metricsCollector.getMetrics();
|
|
6871
|
-
const now = Date.now();
|
|
6872
|
-
const totalErrors = (metrics.requests.clientErrors ?? 0) + (metrics.requests.serverErrors ?? 0);
|
|
6873
|
-
const errorRate = metrics.requests.total > 0 ? totalErrors / metrics.requests.total * 100 : 0;
|
|
6874
|
-
this.metricsHistory.push({
|
|
6875
|
-
timestamp: now,
|
|
6876
|
-
errorRate,
|
|
6877
|
-
avgLatency: metrics.latency.average ?? 0,
|
|
6878
|
-
totalRequests: metrics.requests.total
|
|
6879
|
-
});
|
|
6880
|
-
if (this.metricsHistory.length > 10) {
|
|
6881
|
-
this.metricsHistory.shift();
|
|
6882
|
-
}
|
|
6883
|
-
this.recentIncidents = this.recentIncidents.filter(
|
|
6884
|
-
(ri) => now - ri.timestamp < this.config.cooldownMs
|
|
6885
|
-
);
|
|
6886
|
-
if (metrics.requests.total < this.config.thresholds.minRequestsForDetection) {
|
|
6887
|
-
this.log("debug", "Not enough requests for detection", {
|
|
6888
|
-
total: metrics.requests.total,
|
|
6889
|
-
min: this.config.thresholds.minRequestsForDetection
|
|
6890
|
-
});
|
|
6891
|
-
return;
|
|
6892
|
-
}
|
|
6893
|
-
await this.detectErrorRate(metrics, now);
|
|
6894
|
-
await this.detectLatencyIssues(metrics, now);
|
|
6895
|
-
await this.detectPluginFailures(metrics, now);
|
|
6896
|
-
this.log("debug", "Detection cycle complete", {
|
|
6897
|
-
recentIncidentsCount: this.recentIncidents.length
|
|
6898
|
-
});
|
|
6899
|
-
}
|
|
6900
|
-
/**
|
|
6901
|
-
* Detect high error rate
|
|
6902
|
-
*/
|
|
6903
|
-
async detectErrorRate(metrics, now) {
|
|
6904
|
-
const totalErrors = (metrics.requests.clientErrors ?? 0) + (metrics.requests.serverErrors ?? 0);
|
|
6905
|
-
const errorRate = metrics.requests.total > 0 ? totalErrors / metrics.requests.total * 100 : 0;
|
|
6906
|
-
const key = "error_rate";
|
|
6907
|
-
if (this.isInCooldown(key)) {
|
|
6908
|
-
return;
|
|
6909
|
-
}
|
|
6910
|
-
let severity = null;
|
|
6911
|
-
let title = "";
|
|
6912
|
-
let details = "";
|
|
6913
|
-
if (errorRate >= this.config.thresholds.errorRateCritical) {
|
|
6914
|
-
severity = "critical";
|
|
6915
|
-
title = `Critical Error Rate: ${errorRate.toFixed(1)}%`;
|
|
6916
|
-
details = `Error rate has exceeded critical threshold of ${this.config.thresholds.errorRateCritical}%. Current: ${errorRate.toFixed(2)}% (${totalErrors}/${metrics.requests.total} requests). Client errors: ${metrics.requests.clientErrors ?? 0}, Server errors: ${metrics.requests.serverErrors ?? 0}.`;
|
|
6917
|
-
} else if (errorRate >= this.config.thresholds.errorRateWarning) {
|
|
6918
|
-
severity = "warning";
|
|
6919
|
-
title = `High Error Rate: ${errorRate.toFixed(1)}%`;
|
|
6920
|
-
details = `Error rate has exceeded warning threshold of ${this.config.thresholds.errorRateWarning}%. Current: ${errorRate.toFixed(2)}% (${totalErrors}/${metrics.requests.total} requests).`;
|
|
6921
|
-
}
|
|
6922
|
-
if (severity) {
|
|
6923
|
-
await this.createIncident({
|
|
6924
|
-
type: "error_rate",
|
|
6925
|
-
severity,
|
|
6926
|
-
title,
|
|
6927
|
-
details,
|
|
6928
|
-
metadata: {
|
|
6929
|
-
errorRate,
|
|
6930
|
-
totalRequests: metrics.requests.total,
|
|
6931
|
-
clientErrors: metrics.requests.clientErrors,
|
|
6932
|
-
serverErrors: metrics.requests.serverErrors,
|
|
6933
|
-
threshold: severity === "critical" ? this.config.thresholds.errorRateCritical : this.config.thresholds.errorRateWarning
|
|
6934
|
-
}
|
|
6935
|
-
}, key, now);
|
|
6936
|
-
}
|
|
6937
|
-
}
|
|
6938
|
-
/**
|
|
6939
|
-
* Detect latency spikes
|
|
6940
|
-
*/
|
|
6941
|
-
async detectLatencyIssues(metrics, now) {
|
|
6942
|
-
const avgLatency = metrics.latency.average ?? 0;
|
|
6943
|
-
let p99 = avgLatency * 3;
|
|
6944
|
-
let p95 = avgLatency * 2;
|
|
6945
|
-
if (metrics.latency.histogram && metrics.latency.histogram.length > 0) {
|
|
6946
|
-
const sorted = [...metrics.latency.histogram].sort((a, b) => b.max - a.max);
|
|
6947
|
-
if (sorted.length > 0) {
|
|
6948
|
-
p99 = sorted[0]?.max ?? p99;
|
|
6949
|
-
p95 = sorted[Math.floor(sorted.length * 0.05)]?.max ?? p95;
|
|
6950
|
-
}
|
|
6951
|
-
}
|
|
6952
|
-
const keyP99 = "latency_p99";
|
|
6953
|
-
const keyP95 = "latency_p95";
|
|
6954
|
-
if (!this.isInCooldown(keyP99)) {
|
|
6955
|
-
let severity = null;
|
|
6956
|
-
let title = "";
|
|
6957
|
-
let details = "";
|
|
6958
|
-
if (p99 >= this.config.thresholds.latencyP99Critical) {
|
|
6959
|
-
severity = "critical";
|
|
6960
|
-
title = `Critical P99 Latency: ${p99.toFixed(0)}ms`;
|
|
6961
|
-
details = `P99 latency has exceeded critical threshold of ${this.config.thresholds.latencyP99Critical}ms. Current P99: ${p99.toFixed(0)}ms, Average: ${avgLatency.toFixed(0)}ms. This indicates severe performance degradation affecting tail latencies.`;
|
|
6962
|
-
} else if (p99 >= this.config.thresholds.latencyP99Warning) {
|
|
6963
|
-
severity = "warning";
|
|
6964
|
-
title = `High P99 Latency: ${p99.toFixed(0)}ms`;
|
|
6965
|
-
details = `P99 latency has exceeded warning threshold of ${this.config.thresholds.latencyP99Warning}ms. Current P99: ${p99.toFixed(0)}ms, Average: ${avgLatency.toFixed(0)}ms.`;
|
|
6966
|
-
}
|
|
6967
|
-
if (severity) {
|
|
6968
|
-
await this.createIncident({
|
|
6969
|
-
type: "latency_spike",
|
|
6970
|
-
severity,
|
|
6971
|
-
title,
|
|
6972
|
-
details,
|
|
6973
|
-
metadata: {
|
|
6974
|
-
p99,
|
|
6975
|
-
p95,
|
|
6976
|
-
average: avgLatency,
|
|
6977
|
-
threshold: severity === "critical" ? this.config.thresholds.latencyP99Critical : this.config.thresholds.latencyP99Warning
|
|
6978
|
-
}
|
|
6979
|
-
}, keyP99, now);
|
|
6980
|
-
}
|
|
6981
|
-
}
|
|
6982
|
-
if (!this.isInCooldown(keyP95) && p95 >= this.config.thresholds.latencyP95Warning) {
|
|
6983
|
-
await this.createIncident({
|
|
6984
|
-
type: "latency_spike",
|
|
6985
|
-
severity: "warning",
|
|
6986
|
-
title: `Elevated P95 Latency: ${p95.toFixed(0)}ms`,
|
|
6987
|
-
details: `P95 latency has exceeded threshold of ${this.config.thresholds.latencyP95Warning}ms. Current P95: ${p95.toFixed(0)}ms. Consider investigating slow endpoints.`,
|
|
6988
|
-
metadata: {
|
|
6989
|
-
p95,
|
|
6990
|
-
average: avgLatency,
|
|
6991
|
-
threshold: this.config.thresholds.latencyP95Warning
|
|
6992
|
-
}
|
|
6993
|
-
}, keyP95, now);
|
|
6994
|
-
}
|
|
6995
|
-
}
|
|
6996
|
-
/**
|
|
6997
|
-
* Detect plugin failures
|
|
6998
|
-
*/
|
|
6999
|
-
async detectPluginFailures(metrics, now) {
|
|
7000
|
-
if (!metrics.perPlugin || !Array.isArray(metrics.perPlugin)) {
|
|
7001
|
-
return;
|
|
7002
|
-
}
|
|
7003
|
-
for (const plugin of metrics.perPlugin) {
|
|
7004
|
-
if (!plugin.pluginId || plugin.requests < 5) {
|
|
7005
|
-
continue;
|
|
7006
|
-
}
|
|
7007
|
-
const errorRate = plugin.requests > 0 ? (plugin.errors ?? 0) / plugin.requests * 100 : 0;
|
|
7008
|
-
const key = `plugin:${plugin.pluginId}`;
|
|
7009
|
-
if (this.isInCooldown(key)) {
|
|
7010
|
-
continue;
|
|
7011
|
-
}
|
|
7012
|
-
let severity = null;
|
|
7013
|
-
let title = "";
|
|
7014
|
-
let details = "";
|
|
7015
|
-
if (errorRate >= this.config.thresholds.pluginErrorRateCritical) {
|
|
7016
|
-
severity = "critical";
|
|
7017
|
-
title = `Plugin Failure: ${plugin.pluginId}`;
|
|
7018
|
-
details = `Plugin ${plugin.pluginId} has critical error rate of ${errorRate.toFixed(1)}% (${plugin.errors ?? 0}/${plugin.requests} requests). This exceeds the critical threshold of ${this.config.thresholds.pluginErrorRateCritical}%.`;
|
|
7019
|
-
} else if (errorRate >= this.config.thresholds.pluginErrorRateWarning) {
|
|
7020
|
-
severity = "warning";
|
|
7021
|
-
title = `Plugin Issues: ${plugin.pluginId}`;
|
|
7022
|
-
details = `Plugin ${plugin.pluginId} has elevated error rate of ${errorRate.toFixed(1)}% (${plugin.errors ?? 0}/${plugin.requests} requests).`;
|
|
7023
|
-
}
|
|
7024
|
-
if (severity) {
|
|
7025
|
-
await this.createIncident({
|
|
7026
|
-
type: "plugin_failure",
|
|
7027
|
-
severity,
|
|
7028
|
-
title,
|
|
7029
|
-
details,
|
|
7030
|
-
affectedServices: [plugin.pluginId],
|
|
7031
|
-
metadata: {
|
|
7032
|
-
pluginId: plugin.pluginId,
|
|
7033
|
-
errorRate,
|
|
7034
|
-
requests: plugin.requests,
|
|
7035
|
-
errors: plugin.errors,
|
|
7036
|
-
avgLatency: plugin.latency?.average
|
|
7037
|
-
}
|
|
7038
|
-
}, key, now);
|
|
7039
|
-
}
|
|
7040
|
-
}
|
|
7041
|
-
}
|
|
7042
|
-
/**
|
|
7043
|
-
* Check if incident key is in cooldown
|
|
7044
|
-
*/
|
|
7045
|
-
isInCooldown(key) {
|
|
7046
|
-
return this.recentIncidents.some((ri) => ri.key === key);
|
|
7047
|
-
}
|
|
7048
|
-
/**
|
|
7049
|
-
* Gather related data (logs, metrics, timeline) for incident
|
|
7050
|
-
* @private
|
|
7051
|
-
*/
|
|
7052
|
-
async gatherRelatedData(incidentType, timeWindow = 5 * 60 * 1e3) {
|
|
7053
|
-
const now = Date.now();
|
|
7054
|
-
const from = now - timeWindow;
|
|
7055
|
-
const relatedData = {
|
|
7056
|
-
timeline: []
|
|
7057
|
-
};
|
|
7058
|
-
try {
|
|
7059
|
-
const [errorLogsResult, fatalLogsResult] = await Promise.all([
|
|
7060
|
-
platform.logs.query({ level: "error", from, to: now }, { limit: 50 }),
|
|
7061
|
-
platform.logs.query({ level: "fatal", from, to: now }, { limit: 50 })
|
|
7062
|
-
]);
|
|
7063
|
-
const allErrorLogs = [...errorLogsResult.logs, ...fatalLogsResult.logs];
|
|
7064
|
-
if (allErrorLogs.length > 0) {
|
|
7065
|
-
allErrorLogs.sort((a, b) => b.timestamp - a.timestamp);
|
|
7066
|
-
const uniqueErrors = /* @__PURE__ */ new Set();
|
|
7067
|
-
const endpointErrorCount = /* @__PURE__ */ new Map();
|
|
7068
|
-
for (const log of allErrorLogs) {
|
|
7069
|
-
let errorMsg = typeof log.message === "string" ? log.message : JSON.stringify(log.message);
|
|
7070
|
-
if (log.err) {
|
|
7071
|
-
const err = log.err;
|
|
7072
|
-
const stack = err.stack ? `
|
|
7073
|
-
${err.stack.split("\n").slice(0, 3).join("\n")}` : "";
|
|
7074
|
-
errorMsg = `${err.message || errorMsg}${stack}`;
|
|
7075
|
-
if (log.plugin) {
|
|
7076
|
-
errorMsg = `[${log.plugin}] ${errorMsg}`;
|
|
7077
|
-
}
|
|
7078
|
-
if (log.command) {
|
|
7079
|
-
errorMsg += `
|
|
7080
|
-
Command: ${log.command}`;
|
|
7081
|
-
}
|
|
7082
|
-
}
|
|
7083
|
-
if (errorMsg && uniqueErrors.size < 5) {
|
|
7084
|
-
uniqueErrors.add(errorMsg.substring(0, 500));
|
|
7085
|
-
}
|
|
7086
|
-
const endpoint = log.endpoint || log.url || "unknown";
|
|
7087
|
-
const existing = endpointErrorCount.get(endpoint);
|
|
7088
|
-
if (existing) {
|
|
7089
|
-
existing.count++;
|
|
7090
|
-
} else {
|
|
7091
|
-
endpointErrorCount.set(endpoint, {
|
|
7092
|
-
count: 1,
|
|
7093
|
-
sample: errorMsg.substring(0, 200)
|
|
7094
|
-
});
|
|
7095
|
-
}
|
|
7096
|
-
}
|
|
7097
|
-
const topEndpoints = Array.from(endpointErrorCount.entries()).sort((a, b) => b[1].count - a[1].count).slice(0, 5).map(([endpoint, data]) => ({
|
|
7098
|
-
endpoint,
|
|
7099
|
-
count: data.count,
|
|
7100
|
-
sample: data.sample
|
|
7101
|
-
}));
|
|
7102
|
-
relatedData.logs = {
|
|
7103
|
-
errorCount: allErrorLogs.filter((l) => l.level === "error").length,
|
|
7104
|
-
warnCount: 0,
|
|
7105
|
-
// Could query warnings separately if needed
|
|
7106
|
-
timeRange: [from, now],
|
|
7107
|
-
sampleErrors: Array.from(uniqueErrors),
|
|
7108
|
-
topEndpoints: topEndpoints.length > 0 ? topEndpoints : void 0
|
|
7109
|
-
};
|
|
7110
|
-
for (const log of allErrorLogs.slice(0, 10)) {
|
|
7111
|
-
const msg = typeof log.message === "string" ? log.message : "Error occurred";
|
|
7112
|
-
const plugin = log.plugin ? `[${log.plugin}] ` : "";
|
|
7113
|
-
relatedData.timeline.push({
|
|
7114
|
-
timestamp: log.timestamp,
|
|
7115
|
-
event: `${plugin}${msg.substring(0, 100)}`,
|
|
7116
|
-
source: "logs"
|
|
7117
|
-
});
|
|
7118
|
-
}
|
|
7119
|
-
}
|
|
7120
|
-
} catch (error) {
|
|
7121
|
-
this.log("warn", "Failed to gather related logs", {
|
|
7122
|
-
error: error instanceof Error ? error.message : String(error)
|
|
7123
|
-
});
|
|
7124
|
-
}
|
|
7125
|
-
const currentMetrics = metricsCollector.getMetrics();
|
|
7126
|
-
const duringMetrics = {
|
|
7127
|
-
errorRate: currentMetrics.requests.total > 0 ? (currentMetrics.requests.clientErrors + currentMetrics.requests.serverErrors) / currentMetrics.requests.total * 100 : 0,
|
|
7128
|
-
avgLatency: currentMetrics.latency.average ?? 0,
|
|
7129
|
-
totalRequests: currentMetrics.requests.total,
|
|
7130
|
-
totalErrors: currentMetrics.requests.clientErrors + currentMetrics.requests.serverErrors
|
|
7131
|
-
};
|
|
7132
|
-
let beforeMetrics;
|
|
7133
|
-
if (this.metricsHistory.length >= 2) {
|
|
7134
|
-
const beforeSnapshots = this.metricsHistory.slice(0, -1);
|
|
7135
|
-
if (beforeSnapshots.length > 0) {
|
|
7136
|
-
const avgErrorRate = beforeSnapshots.reduce((sum, s) => sum + s.errorRate, 0) / beforeSnapshots.length;
|
|
7137
|
-
const avgLatency = beforeSnapshots.reduce((sum, s) => sum + s.avgLatency, 0) / beforeSnapshots.length;
|
|
7138
|
-
const avgRequests = beforeSnapshots.reduce((sum, s) => sum + s.totalRequests, 0) / beforeSnapshots.length;
|
|
7139
|
-
beforeMetrics = {
|
|
7140
|
-
errorRate: avgErrorRate,
|
|
7141
|
-
avgLatency,
|
|
7142
|
-
totalRequests: avgRequests
|
|
7143
|
-
};
|
|
7144
|
-
this.log("debug", "Calculated before metrics", {
|
|
7145
|
-
before: beforeMetrics,
|
|
7146
|
-
during: duringMetrics,
|
|
7147
|
-
snapshotsUsed: beforeSnapshots.length
|
|
7148
|
-
});
|
|
7149
|
-
}
|
|
7150
|
-
}
|
|
7151
|
-
let topSlowest;
|
|
7152
|
-
let affectedEndpoints;
|
|
7153
|
-
if (incidentType === "latency_spike") {
|
|
7154
|
-
const histogram = currentMetrics.latency.histogram;
|
|
7155
|
-
if (histogram && histogram.length > 0) {
|
|
7156
|
-
const slowestRoutes = histogram.filter((h) => h.max > 100).sort((a, b) => b.max - a.max).slice(0, 10);
|
|
7157
|
-
topSlowest = slowestRoutes.map((h) => {
|
|
7158
|
-
const [method, ...pathParts] = h.route.split(" ");
|
|
7159
|
-
const endpoint = pathParts.join(" ");
|
|
7160
|
-
const statusCodes = Object.keys(h.byStatus);
|
|
7161
|
-
const mostCommonStatus = statusCodes.length > 0 ? parseInt(statusCodes[0], 10) : void 0;
|
|
7162
|
-
return {
|
|
7163
|
-
endpoint,
|
|
7164
|
-
method: method || "GET",
|
|
7165
|
-
durationMs: Math.round(h.max),
|
|
7166
|
-
statusCode: mostCommonStatus
|
|
7167
|
-
};
|
|
7168
|
-
});
|
|
7169
|
-
affectedEndpoints = [...new Set(slowestRoutes.map((h) => h.route))];
|
|
7170
|
-
this.log("debug", "Collected slow requests for latency incident", {
|
|
7171
|
-
topSlowestCount: topSlowest.length,
|
|
7172
|
-
affectedEndpointsCount: affectedEndpoints.length
|
|
7173
|
-
});
|
|
7174
|
-
}
|
|
7175
|
-
}
|
|
7176
|
-
relatedData.metrics = {
|
|
7177
|
-
before: beforeMetrics,
|
|
7178
|
-
during: duringMetrics,
|
|
7179
|
-
topSlowest,
|
|
7180
|
-
affectedEndpoints
|
|
7181
|
-
};
|
|
7182
|
-
relatedData.timeline.unshift({
|
|
7183
|
-
timestamp: now,
|
|
7184
|
-
event: `Incident detected: ${incidentType}`,
|
|
7185
|
-
source: "detector"
|
|
7186
|
-
});
|
|
7187
|
-
relatedData.timeline.sort((a, b) => b.timestamp - a.timestamp);
|
|
7188
|
-
return relatedData;
|
|
7189
|
-
}
|
|
7190
|
-
/**
|
|
7191
|
-
* Create incident and track it
|
|
7192
|
-
*/
|
|
7193
|
-
async createIncident(payload, key, timestamp) {
|
|
7194
|
-
try {
|
|
7195
|
-
const relatedData = await this.gatherRelatedData(payload.type);
|
|
7196
|
-
const enrichedPayload = {
|
|
7197
|
-
...payload,
|
|
7198
|
-
relatedData
|
|
7199
|
-
};
|
|
7200
|
-
const incident = await this.incidentStorage.createIncident(enrichedPayload);
|
|
7201
|
-
this.recentIncidents.push({
|
|
7202
|
-
type: payload.type,
|
|
7203
|
-
key,
|
|
7204
|
-
timestamp
|
|
7205
|
-
});
|
|
7206
|
-
this.log("info", "Auto-created incident with context", {
|
|
7207
|
-
id: incident.id,
|
|
7208
|
-
type: incident.type,
|
|
7209
|
-
severity: incident.severity,
|
|
7210
|
-
title: incident.title,
|
|
7211
|
-
errorLogsCount: relatedData.logs?.errorCount ?? 0,
|
|
7212
|
-
timelineEventsCount: relatedData.timeline?.length ?? 0
|
|
7213
|
-
});
|
|
7214
|
-
if (platform.analytics) {
|
|
7215
|
-
platform.analytics.track("incident.created", {
|
|
7216
|
-
incidentId: incident.id,
|
|
7217
|
-
type: incident.type,
|
|
7218
|
-
severity: incident.severity,
|
|
7219
|
-
source: "auto-detector",
|
|
7220
|
-
errorLogsCount: relatedData.logs?.errorCount ?? 0,
|
|
7221
|
-
timelineEventsCount: relatedData.timeline?.length ?? 0,
|
|
7222
|
-
hasBeforeMetrics: !!relatedData.metrics?.before,
|
|
7223
|
-
affectedServicesCount: payload.affectedServices?.length ?? 0
|
|
7224
|
-
}).catch(() => {
|
|
7225
|
-
});
|
|
7226
|
-
}
|
|
7227
|
-
} catch (error) {
|
|
7228
|
-
this.log("error", "Failed to create incident", {
|
|
7229
|
-
key,
|
|
7230
|
-
error: error instanceof Error ? error.message : String(error)
|
|
7231
|
-
});
|
|
7232
|
-
}
|
|
7233
|
-
}
|
|
7234
|
-
/**
|
|
7235
|
-
* Get current detector status
|
|
7236
|
-
*/
|
|
7237
|
-
getStatus() {
|
|
7238
|
-
return {
|
|
7239
|
-
running: this.isRunning,
|
|
7240
|
-
config: this.config,
|
|
7241
|
-
recentIncidentsCount: this.recentIncidents.length
|
|
7242
|
-
};
|
|
7243
|
-
}
|
|
7244
|
-
/**
|
|
7245
|
-
* Update thresholds at runtime
|
|
7246
|
-
*/
|
|
7247
|
-
updateThresholds(thresholds) {
|
|
7248
|
-
this.config.thresholds = { ...this.config.thresholds, ...thresholds };
|
|
7249
|
-
this.log("info", "Thresholds updated", { thresholds: this.config.thresholds });
|
|
7250
|
-
}
|
|
7251
|
-
log(level, message, meta) {
|
|
7252
|
-
if (level === "debug" && !this.config.debug) {
|
|
7253
|
-
return;
|
|
7254
|
-
}
|
|
7255
|
-
const prefix = "[IncidentDetector]";
|
|
7256
|
-
if (this.logger[level]) {
|
|
7257
|
-
if (meta) {
|
|
7258
|
-
this.logger[level]({ ...meta }, `${prefix} ${message}`);
|
|
7259
|
-
} else {
|
|
7260
|
-
this.logger[level](`${prefix} ${message}`);
|
|
7261
|
-
}
|
|
7262
|
-
} else {
|
|
7263
|
-
console.log(`${prefix} [${level}] ${message}`, meta ?? "");
|
|
7264
|
-
}
|
|
7265
|
-
}
|
|
7266
|
-
};
|
|
7267
5642
|
|
|
7268
5643
|
// src/routes/index.ts
|
|
7269
5644
|
function normalizeBasePath2(basePath) {
|
|
@@ -7459,71 +5834,7 @@ async function registerRoutes(server, config, repoRoot, registry) {
|
|
|
7459
5834
|
historicalCollector.stop();
|
|
7460
5835
|
platform.logger.info("Historical metrics collector stopped");
|
|
7461
5836
|
});
|
|
7462
|
-
|
|
7463
|
-
const db = platform.getAdapter("db");
|
|
7464
|
-
if (db) {
|
|
7465
|
-
try {
|
|
7466
|
-
const { readFileSync: readFileSync2 } = await import('fs');
|
|
7467
|
-
const { fileURLToPath } = await import('url');
|
|
7468
|
-
const { dirname: dirname2, join: join3 } = await import('path');
|
|
7469
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
7470
|
-
const __dirname = dirname2(__filename);
|
|
7471
|
-
const schemaPath = join3(__dirname, "..", "services", "incident-schema.sql");
|
|
7472
|
-
const schema = readFileSync2(schemaPath, "utf-8");
|
|
7473
|
-
if ("exec" in db && typeof db.exec === "function") {
|
|
7474
|
-
await db.exec(schema);
|
|
7475
|
-
} else {
|
|
7476
|
-
const statements = schema.split(";").map((s) => s.trim()).filter((s) => s.length > 0 && !s.startsWith("--"));
|
|
7477
|
-
for (const statement of statements) {
|
|
7478
|
-
if (statement) {
|
|
7479
|
-
await db.query(statement);
|
|
7480
|
-
}
|
|
7481
|
-
}
|
|
7482
|
-
}
|
|
7483
|
-
platform.logger.info("Incidents schema initialized");
|
|
7484
|
-
incidentStorage = new IncidentStorage(
|
|
7485
|
-
db,
|
|
7486
|
-
{ debug: process.env.NODE_ENV !== "production" },
|
|
7487
|
-
platformServices.logger
|
|
7488
|
-
);
|
|
7489
|
-
platform.logger.info("Incident storage initialized");
|
|
7490
|
-
} catch (err) {
|
|
7491
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
7492
|
-
platform.logger.warn("Failed to initialize incident storage, continuing without it", { error: error.message });
|
|
7493
|
-
}
|
|
7494
|
-
} else {
|
|
7495
|
-
platform.logger.warn("Database adapter (db) not configured \u2014 incident storage disabled. Configure db adapter in kb.config.json to enable.");
|
|
7496
|
-
}
|
|
7497
|
-
if (incidentStorage) {
|
|
7498
|
-
const incidentDetector = new IncidentDetector(
|
|
7499
|
-
incidentStorage,
|
|
7500
|
-
{
|
|
7501
|
-
intervalMs: 3e4,
|
|
7502
|
-
// Check every 30 seconds
|
|
7503
|
-
cooldownMs: 5 * 60 * 1e3,
|
|
7504
|
-
// 5 minute cooldown between same incidents
|
|
7505
|
-
debug: process.env.NODE_ENV !== "production",
|
|
7506
|
-
thresholds: {
|
|
7507
|
-
errorRateWarning: 5,
|
|
7508
|
-
errorRateCritical: 10,
|
|
7509
|
-
latencyP99Warning: 2e3,
|
|
7510
|
-
latencyP99Critical: 5e3,
|
|
7511
|
-
latencyP95Warning: 1e3,
|
|
7512
|
-
minRequestsForDetection: 10,
|
|
7513
|
-
pluginErrorRateWarning: 10,
|
|
7514
|
-
pluginErrorRateCritical: 25
|
|
7515
|
-
}
|
|
7516
|
-
},
|
|
7517
|
-
platformServices.logger
|
|
7518
|
-
);
|
|
7519
|
-
incidentDetector.start();
|
|
7520
|
-
platform.logger.info("Incident detector started");
|
|
7521
|
-
server.addHook("onClose", async () => {
|
|
7522
|
-
incidentDetector.stop();
|
|
7523
|
-
platform.logger.info("Incident detector stopped");
|
|
7524
|
-
});
|
|
7525
|
-
}
|
|
7526
|
-
await registerObservabilityRoutes(server, config, repoRoot, historicalCollector, incidentStorage, platformServices);
|
|
5837
|
+
await registerObservabilityRoutes(server, config, repoRoot, historicalCollector, platformServices);
|
|
7527
5838
|
await registerAnalyticsRoutes(server, config);
|
|
7528
5839
|
await registerAdaptersRoutes(server, config);
|
|
7529
5840
|
await registerLogRoutes(server);
|
|
@@ -7614,6 +5925,9 @@ function registerEnvelopeMiddleware(server, config) {
|
|
|
7614
5925
|
if (parsedPayload && typeof parsedPayload === "object" && "ok" in parsedPayload) {
|
|
7615
5926
|
return payload;
|
|
7616
5927
|
}
|
|
5928
|
+
if (reply.getHeader("x-openapi-spec")) {
|
|
5929
|
+
return payload;
|
|
5930
|
+
}
|
|
7617
5931
|
let dataToWrap = parsedPayload;
|
|
7618
5932
|
if (parsedPayload === null || parsedPayload === void 0) {
|
|
7619
5933
|
dataToWrap = null;
|