@kb-labs/rest-api-app 2.14.0 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +53 -1738
- package/dist/index.js.map +1 -1
- package/package.json +21 -21
package/dist/index.js
CHANGED
|
@@ -21,8 +21,6 @@ import { WORKFLOW_REDIS_CHANNEL } from '@kb-labs/workflow-constants';
|
|
|
21
21
|
import { validatorCompiler, serializerCompiler } from 'fastify-type-provider-zod';
|
|
22
22
|
import { z } from 'zod';
|
|
23
23
|
import { errorEnvelopeSchema, ErrorCode, ErrorResponseSchema, JobsListResponseSchema, ListJobsQuerySchema, JobStatsResponseSchema, JobResponseSchema, JobActionResponseSchema } from '@kb-labs/rest-api-contracts';
|
|
24
|
-
import { exec } from 'child_process';
|
|
25
|
-
import { promisify } from 'util';
|
|
26
24
|
import { readKbConfig } from '@kb-labs/core-config';
|
|
27
25
|
import { AdapterNameSchema } from '@kb-labs/gateway-contracts';
|
|
28
26
|
import fastifyCors from '@fastify/cors';
|
|
@@ -2694,8 +2692,9 @@ async function registerJobsRoutes(server, config, cronManager) {
|
|
|
2694
2692
|
throw createError(503, "CRON_UNAVAILABLE", "Cron manager not available");
|
|
2695
2693
|
}
|
|
2696
2694
|
let jobs = cronManager.list();
|
|
2697
|
-
|
|
2698
|
-
|
|
2695
|
+
const query = request.query;
|
|
2696
|
+
if (query.status) {
|
|
2697
|
+
jobs = jobs.filter((job) => job.status === query.status);
|
|
2699
2698
|
}
|
|
2700
2699
|
return reply.send({ jobs });
|
|
2701
2700
|
}
|
|
@@ -2900,261 +2899,15 @@ async function registerCacheRoutes(server, config, registry) {
|
|
|
2900
2899
|
}
|
|
2901
2900
|
});
|
|
2902
2901
|
}
|
|
2903
|
-
|
|
2904
|
-
model: "gpt-4",
|
|
2905
|
-
temperature: 0.3,
|
|
2906
|
-
maxTokens: 1500,
|
|
2907
|
-
debug: false
|
|
2908
|
-
};
|
|
2909
|
-
var IncidentAnalyzer = class {
|
|
2910
|
-
config;
|
|
2911
|
-
logger;
|
|
2912
|
-
constructor(config = {}, logger = console) {
|
|
2913
|
-
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
2914
|
-
this.logger = logger;
|
|
2915
|
-
}
|
|
2916
|
-
/**
|
|
2917
|
-
* Analyze incident using LLM
|
|
2918
|
-
*
|
|
2919
|
-
* @param incident - Incident to analyze
|
|
2920
|
-
* @returns AI analysis result
|
|
2921
|
-
*/
|
|
2922
|
-
async analyze(incident) {
|
|
2923
|
-
this.log("info", "Analyzing incident", { id: incident.id, type: incident.type });
|
|
2924
|
-
const prompt = this.buildAnalysisPrompt(incident);
|
|
2925
|
-
try {
|
|
2926
|
-
const result = await platform.llm.complete(prompt, {
|
|
2927
|
-
model: this.config.model,
|
|
2928
|
-
temperature: this.config.temperature,
|
|
2929
|
-
maxTokens: this.config.maxTokens,
|
|
2930
|
-
systemPrompt: this.getSystemPrompt()
|
|
2931
|
-
});
|
|
2932
|
-
this.log("debug", "LLM response received", {
|
|
2933
|
-
length: result.content.length,
|
|
2934
|
-
tokensUsed: result.usage ? result.usage.promptTokens + result.usage.completionTokens : void 0
|
|
2935
|
-
});
|
|
2936
|
-
const analysis = this.parseAnalysisResponse(result.content);
|
|
2937
|
-
this.log("info", "Incident analysis complete", {
|
|
2938
|
-
id: incident.id,
|
|
2939
|
-
rootCausesCount: analysis.rootCauses.length,
|
|
2940
|
-
recommendationsCount: analysis.recommendations.length
|
|
2941
|
-
});
|
|
2942
|
-
return analysis;
|
|
2943
|
-
} catch (error) {
|
|
2944
|
-
this.log("error", "Incident analysis failed", {
|
|
2945
|
-
id: incident.id,
|
|
2946
|
-
error: error instanceof Error ? error.message : String(error)
|
|
2947
|
-
});
|
|
2948
|
-
return this.getFallbackAnalysis(incident);
|
|
2949
|
-
}
|
|
2950
|
-
}
|
|
2951
|
-
/**
|
|
2952
|
-
* Build analysis prompt from incident data
|
|
2953
|
-
* @private
|
|
2954
|
-
*/
|
|
2955
|
-
buildAnalysisPrompt(incident) {
|
|
2956
|
-
let prompt = `# Incident Analysis Request
|
|
2957
|
-
|
|
2958
|
-
`;
|
|
2959
|
-
prompt += `## Incident Overview
|
|
2960
|
-
`;
|
|
2961
|
-
prompt += `- **ID**: ${incident.id}
|
|
2962
|
-
`;
|
|
2963
|
-
prompt += `- **Type**: ${incident.type}
|
|
2964
|
-
`;
|
|
2965
|
-
prompt += `- **Severity**: ${incident.severity}
|
|
2966
|
-
`;
|
|
2967
|
-
prompt += `- **Title**: ${incident.title}
|
|
2968
|
-
`;
|
|
2969
|
-
prompt += `- **Time**: ${new Date(incident.timestamp).toISOString()}
|
|
2970
|
-
|
|
2971
|
-
`;
|
|
2972
|
-
prompt += `## Description
|
|
2973
|
-
${incident.details}
|
|
2974
|
-
|
|
2975
|
-
`;
|
|
2976
|
-
if (incident.metadata && Object.keys(incident.metadata).length > 0) {
|
|
2977
|
-
prompt += `## Metrics
|
|
2978
|
-
`;
|
|
2979
|
-
prompt += "```json\n";
|
|
2980
|
-
prompt += JSON.stringify(incident.metadata, null, 2);
|
|
2981
|
-
prompt += "\n```\n\n";
|
|
2982
|
-
}
|
|
2983
|
-
if (incident.relatedData?.logs && incident.relatedData.logs.sampleErrors.length > 0) {
|
|
2984
|
-
prompt += `## Related Error Logs (${incident.relatedData.logs.errorCount} total)
|
|
2985
|
-
`;
|
|
2986
|
-
incident.relatedData.logs.sampleErrors.forEach((error, idx) => {
|
|
2987
|
-
prompt += `${idx + 1}. ${error}
|
|
2988
|
-
`;
|
|
2989
|
-
});
|
|
2990
|
-
prompt += "\n";
|
|
2991
|
-
}
|
|
2992
|
-
if (incident.relatedData?.timeline && incident.relatedData.timeline.length > 0) {
|
|
2993
|
-
prompt += `## Event Timeline
|
|
2994
|
-
`;
|
|
2995
|
-
incident.relatedData.timeline.slice(0, 10).forEach((event) => {
|
|
2996
|
-
const time = new Date(event.timestamp).toISOString();
|
|
2997
|
-
prompt += `- [${time}] (${event.source}) ${event.event}
|
|
2998
|
-
`;
|
|
2999
|
-
});
|
|
3000
|
-
prompt += "\n";
|
|
3001
|
-
}
|
|
3002
|
-
if (incident.relatedData?.metrics) {
|
|
3003
|
-
prompt += `## System Metrics
|
|
3004
|
-
`;
|
|
3005
|
-
prompt += "```json\n";
|
|
3006
|
-
prompt += JSON.stringify(incident.relatedData.metrics, null, 2);
|
|
3007
|
-
prompt += "\n```\n\n";
|
|
3008
|
-
}
|
|
3009
|
-
prompt += `## Analysis Instructions
|
|
3010
|
-
`;
|
|
3011
|
-
prompt += `Please analyze this incident and provide:
|
|
3012
|
-
`;
|
|
3013
|
-
prompt += `1. A brief executive summary (2-3 sentences)
|
|
3014
|
-
`;
|
|
3015
|
-
prompt += `2. Root causes with confidence scores (0.0-1.0) and evidence
|
|
3016
|
-
`;
|
|
3017
|
-
prompt += `3. Patterns or trends you observe
|
|
3018
|
-
`;
|
|
3019
|
-
prompt += `4. Actionable recommendations for resolution and prevention
|
|
3020
|
-
|
|
3021
|
-
`;
|
|
3022
|
-
prompt += `Format your response as JSON:
|
|
3023
|
-
`;
|
|
3024
|
-
prompt += "```json\n";
|
|
3025
|
-
prompt += `{
|
|
3026
|
-
`;
|
|
3027
|
-
prompt += ` "summary": "Executive summary here",
|
|
3028
|
-
`;
|
|
3029
|
-
prompt += ` "rootCauses": [
|
|
3030
|
-
`;
|
|
3031
|
-
prompt += ` {
|
|
3032
|
-
`;
|
|
3033
|
-
prompt += ` "factor": "Root cause description",
|
|
3034
|
-
`;
|
|
3035
|
-
prompt += ` "confidence": 0.85,
|
|
3036
|
-
`;
|
|
3037
|
-
prompt += ` "evidence": "Supporting evidence from logs/metrics"
|
|
3038
|
-
`;
|
|
3039
|
-
prompt += ` }
|
|
3040
|
-
`;
|
|
3041
|
-
prompt += ` ],
|
|
3042
|
-
`;
|
|
3043
|
-
prompt += ` "patterns": ["Pattern 1", "Pattern 2"],
|
|
3044
|
-
`;
|
|
3045
|
-
prompt += ` "recommendations": ["Recommendation 1", "Recommendation 2"]
|
|
3046
|
-
`;
|
|
3047
|
-
prompt += `}
|
|
3048
|
-
`;
|
|
3049
|
-
prompt += "```";
|
|
3050
|
-
return prompt;
|
|
3051
|
-
}
|
|
3052
|
-
/**
|
|
3053
|
-
* Get system prompt for incident analysis
|
|
3054
|
-
* @private
|
|
3055
|
-
*/
|
|
3056
|
-
getSystemPrompt() {
|
|
3057
|
-
return `You are an expert SRE (Site Reliability Engineer) and incident response specialist.
|
|
3058
|
-
Your role is to analyze system incidents, identify root causes, and provide actionable recommendations.
|
|
3059
|
-
|
|
3060
|
-
Guidelines:
|
|
3061
|
-
- Base your analysis on the actual data provided (logs, metrics, timeline)
|
|
3062
|
-
- Provide confidence scores based on available evidence (0.0 = no evidence, 1.0 = certain)
|
|
3063
|
-
- Focus on actionable recommendations that can prevent future incidents
|
|
3064
|
-
- Be concise but thorough - this is for production systems
|
|
3065
|
-
- Always respond with valid JSON matching the requested format`;
|
|
3066
|
-
}
|
|
3067
|
-
/**
|
|
3068
|
-
* Parse LLM response into structured analysis
|
|
3069
|
-
* @private
|
|
3070
|
-
*/
|
|
3071
|
-
parseAnalysisResponse(response) {
|
|
3072
|
-
try {
|
|
3073
|
-
const jsonMatch = response.match(/```json\s*([\s\S]*?)\s*```/) ?? response.match(/```\s*([\s\S]*?)\s*```/) ?? [null, response];
|
|
3074
|
-
const jsonStr = jsonMatch[1] || response;
|
|
3075
|
-
const parsed = JSON.parse(jsonStr.trim());
|
|
3076
|
-
return {
|
|
3077
|
-
summary: parsed.summary || "No summary provided",
|
|
3078
|
-
rootCauses: Array.isArray(parsed.rootCauses) ? parsed.rootCauses : [],
|
|
3079
|
-
patterns: Array.isArray(parsed.patterns) ? parsed.patterns : [],
|
|
3080
|
-
recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations : [],
|
|
3081
|
-
analyzedAt: Date.now()
|
|
3082
|
-
};
|
|
3083
|
-
} catch (error) {
|
|
3084
|
-
this.log("warn", "Failed to parse LLM response, using fallback", {
|
|
3085
|
-
error: error instanceof Error ? error.message : String(error)
|
|
3086
|
-
});
|
|
3087
|
-
return {
|
|
3088
|
-
summary: response.substring(0, 200),
|
|
3089
|
-
rootCauses: [],
|
|
3090
|
-
patterns: [],
|
|
3091
|
-
recommendations: [],
|
|
3092
|
-
analyzedAt: Date.now()
|
|
3093
|
-
};
|
|
3094
|
-
}
|
|
3095
|
-
}
|
|
3096
|
-
/**
|
|
3097
|
-
* Get fallback analysis when LLM is unavailable
|
|
3098
|
-
* @private
|
|
3099
|
-
*/
|
|
3100
|
-
getFallbackAnalysis(incident) {
|
|
3101
|
-
const errorCount = incident.relatedData?.logs?.errorCount ?? 0;
|
|
3102
|
-
const sampleErrors = incident.relatedData?.logs?.sampleErrors ?? [];
|
|
3103
|
-
return {
|
|
3104
|
-
summary: `${incident.severity} ${incident.type} incident detected. ${errorCount} errors logged.`,
|
|
3105
|
-
rootCauses: [
|
|
3106
|
-
{
|
|
3107
|
-
factor: incident.title,
|
|
3108
|
-
confidence: 0.5,
|
|
3109
|
-
evidence: incident.details
|
|
3110
|
-
}
|
|
3111
|
-
],
|
|
3112
|
-
patterns: sampleErrors.length > 0 ? [`Multiple error types detected: ${sampleErrors.length} unique errors`] : [],
|
|
3113
|
-
recommendations: [
|
|
3114
|
-
"Review related error logs for stack traces",
|
|
3115
|
-
"Check system metrics during incident timeframe",
|
|
3116
|
-
"Verify external dependencies are operational"
|
|
3117
|
-
],
|
|
3118
|
-
analyzedAt: Date.now()
|
|
3119
|
-
};
|
|
3120
|
-
}
|
|
3121
|
-
log(level, message, meta) {
|
|
3122
|
-
if (level === "debug" && !this.config.debug) {
|
|
3123
|
-
return;
|
|
3124
|
-
}
|
|
3125
|
-
const prefix = "[IncidentAnalyzer]";
|
|
3126
|
-
if (this.logger[level]) {
|
|
3127
|
-
if (meta) {
|
|
3128
|
-
this.logger[level]({ ...meta }, `${prefix} ${message}`);
|
|
3129
|
-
} else {
|
|
3130
|
-
this.logger[level](`${prefix} ${message}`);
|
|
3131
|
-
}
|
|
3132
|
-
} else {
|
|
3133
|
-
console.log(`${prefix} [${level}] ${message}`, meta ?? "");
|
|
3134
|
-
}
|
|
3135
|
-
}
|
|
3136
|
-
};
|
|
3137
|
-
|
|
3138
|
-
// src/routes/observability.ts
|
|
3139
|
-
var execAsync = promisify(exec);
|
|
3140
|
-
var DEVKIT_CACHE_KEY = "observability:devkit-health";
|
|
3141
|
-
var DEVKIT_CACHE_TTL_MS = 10 * 60 * 1e3;
|
|
3142
|
-
async function registerObservabilityRoutes(fastify, config, repoRoot, historicalMetrics, incidentStorage, platform19) {
|
|
2902
|
+
async function registerObservabilityRoutes(fastify, config, repoRoot, historicalMetrics, platform16) {
|
|
3143
2903
|
const basePath = normalizeBasePath(config.basePath);
|
|
3144
2904
|
const stateBrokerPaths = resolvePaths(basePath, "/observability/state-broker");
|
|
3145
|
-
const devkitPaths = resolvePaths(basePath, "/observability/devkit");
|
|
3146
2905
|
const systemMetricsPaths = resolvePaths(basePath, "/observability/system-metrics");
|
|
3147
2906
|
const metricsHistoryPaths = resolvePaths(basePath, "/observability/metrics/history");
|
|
3148
2907
|
const metricsHeatmapPaths = resolvePaths(basePath, "/observability/metrics/heatmap");
|
|
3149
|
-
const incidentsListPaths = resolvePaths(basePath, "/observability/incidents");
|
|
3150
|
-
const incidentsDetailPaths = resolvePaths(basePath, "/observability/incidents/:id");
|
|
3151
|
-
const incidentsCreatePaths = resolvePaths(basePath, "/observability/incidents");
|
|
3152
|
-
const incidentsHistoryPaths = resolvePaths(basePath, "/observability/incidents/history");
|
|
3153
|
-
const incidentsResolvePaths = resolvePaths(basePath, "/observability/incidents/:id/resolve");
|
|
3154
|
-
const incidentsAnalyzePaths = resolvePaths(basePath, "/observability/incidents/:id/analyze");
|
|
3155
2908
|
const insightsChatPaths = resolvePaths(basePath, "/observability/insights/chat");
|
|
3156
2909
|
for (const path3 of stateBrokerPaths) {
|
|
3157
|
-
fastify.get(path3, async (_request, reply) => {
|
|
2910
|
+
fastify.get(path3, { schema: { tags: ["Observability"], summary: "State Broker statistics" } }, async (_request, reply) => {
|
|
3158
2911
|
try {
|
|
3159
2912
|
const stateBrokerUrl = process.env.KB_STATE_DAEMON_URL || "http://localhost:7777";
|
|
3160
2913
|
fastify.log.debug({ url: stateBrokerUrl }, "Fetching State Broker stats");
|
|
@@ -3193,7 +2946,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3193
2946
|
}
|
|
3194
2947
|
};
|
|
3195
2948
|
} catch (error) {
|
|
3196
|
-
|
|
2949
|
+
platform16?.logger.error("Failed to fetch State Broker stats", error instanceof Error ? error : new Error(String(error)));
|
|
3197
2950
|
const isTimeout = error instanceof Error && error.name === "AbortError";
|
|
3198
2951
|
return reply.code(503).send({
|
|
3199
2952
|
ok: false,
|
|
@@ -3208,105 +2961,10 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3208
2961
|
}
|
|
3209
2962
|
});
|
|
3210
2963
|
}
|
|
3211
|
-
for (const path3 of devkitPaths) {
|
|
3212
|
-
fastify.get(path3, async (_request, reply) => {
|
|
3213
|
-
const now = Date.now();
|
|
3214
|
-
if (platform19?.cache) {
|
|
3215
|
-
try {
|
|
3216
|
-
const cached = await platform19.cache.get(DEVKIT_CACHE_KEY);
|
|
3217
|
-
if (cached && now < cached.expiresAt) {
|
|
3218
|
-
const remainingTtl = Math.round((cached.expiresAt - now) / 1e3);
|
|
3219
|
-
fastify.log.debug({ remainingTtl }, "Returning cached DevKit health from platform.cache");
|
|
3220
|
-
return {
|
|
3221
|
-
ok: true,
|
|
3222
|
-
data: cached.data,
|
|
3223
|
-
meta: {
|
|
3224
|
-
source: "devkit-cli",
|
|
3225
|
-
repoRoot,
|
|
3226
|
-
cached: true,
|
|
3227
|
-
cachedAt: cached.timestamp,
|
|
3228
|
-
expiresAt: cached.expiresAt,
|
|
3229
|
-
ttlSeconds: remainingTtl
|
|
3230
|
-
}
|
|
3231
|
-
};
|
|
3232
|
-
}
|
|
3233
|
-
} catch (cacheError) {
|
|
3234
|
-
fastify.log.warn({ err: cacheError }, "Failed to read from platform.cache, proceeding without cache");
|
|
3235
|
-
}
|
|
3236
|
-
}
|
|
3237
|
-
try {
|
|
3238
|
-
fastify.log.debug({ cwd: repoRoot }, "Executing DevKit health check (cache miss)");
|
|
3239
|
-
const { stdout, stderr } = await execAsync("npx kb-devkit-health --json", {
|
|
3240
|
-
cwd: repoRoot,
|
|
3241
|
-
timeout: 3e4,
|
|
3242
|
-
// 30s timeout
|
|
3243
|
-
env: {
|
|
3244
|
-
...process.env,
|
|
3245
|
-
// Ensure DevKit runs in non-interactive mode
|
|
3246
|
-
CI: "true"
|
|
3247
|
-
}
|
|
3248
|
-
});
|
|
3249
|
-
if (stderr) {
|
|
3250
|
-
fastify.log.warn({ stderr }, "DevKit health check produced warnings");
|
|
3251
|
-
}
|
|
3252
|
-
const health = JSON.parse(stdout);
|
|
3253
|
-
const expiresAt = now + DEVKIT_CACHE_TTL_MS;
|
|
3254
|
-
if (platform19?.cache) {
|
|
3255
|
-
try {
|
|
3256
|
-
await platform19.cache.set(DEVKIT_CACHE_KEY, {
|
|
3257
|
-
data: health,
|
|
3258
|
-
timestamp: now,
|
|
3259
|
-
expiresAt
|
|
3260
|
-
}, DEVKIT_CACHE_TTL_MS);
|
|
3261
|
-
fastify.log.debug({
|
|
3262
|
-
healthScore: health.healthScore,
|
|
3263
|
-
grade: health.grade,
|
|
3264
|
-
cachedUntil: new Date(expiresAt).toISOString()
|
|
3265
|
-
}, "DevKit health check completed and cached in platform.cache");
|
|
3266
|
-
} catch (cacheError) {
|
|
3267
|
-
fastify.log.warn({ err: cacheError }, "Failed to write to platform.cache");
|
|
3268
|
-
}
|
|
3269
|
-
}
|
|
3270
|
-
return {
|
|
3271
|
-
ok: true,
|
|
3272
|
-
data: health,
|
|
3273
|
-
meta: {
|
|
3274
|
-
source: "devkit-cli",
|
|
3275
|
-
repoRoot,
|
|
3276
|
-
command: "npx kb-devkit-health --json",
|
|
3277
|
-
cached: false,
|
|
3278
|
-
cachedAt: now,
|
|
3279
|
-
expiresAt,
|
|
3280
|
-
ttlSeconds: DEVKIT_CACHE_TTL_MS / 1e3
|
|
3281
|
-
}
|
|
3282
|
-
};
|
|
3283
|
-
} catch (error) {
|
|
3284
|
-
let partialData = null;
|
|
3285
|
-
if (error && typeof error === "object" && "stdout" in error) {
|
|
3286
|
-
try {
|
|
3287
|
-
partialData = JSON.parse(error.stdout);
|
|
3288
|
-
} catch {
|
|
3289
|
-
partialData = null;
|
|
3290
|
-
}
|
|
3291
|
-
}
|
|
3292
|
-
platform19?.logger.error("Failed to execute DevKit health check", error instanceof Error ? error : new Error(String(error)));
|
|
3293
|
-
return reply.code(500).send({
|
|
3294
|
-
ok: false,
|
|
3295
|
-
error: {
|
|
3296
|
-
code: "DEVKIT_ERROR",
|
|
3297
|
-
message: error instanceof Error ? error.message : "Failed to execute DevKit health check",
|
|
3298
|
-
details: {
|
|
3299
|
-
partialData
|
|
3300
|
-
}
|
|
3301
|
-
}
|
|
3302
|
-
});
|
|
3303
|
-
}
|
|
3304
|
-
});
|
|
3305
|
-
}
|
|
3306
2964
|
for (const path3 of systemMetricsPaths) {
|
|
3307
|
-
fastify.get(path3, async (_request, reply) => {
|
|
2965
|
+
fastify.get(path3, { schema: { tags: ["Observability"], summary: "System resource metrics" } }, async (_request, reply) => {
|
|
3308
2966
|
try {
|
|
3309
|
-
if (!
|
|
2967
|
+
if (!platform16?.cache) {
|
|
3310
2968
|
return reply.code(503).send({
|
|
3311
2969
|
ok: false,
|
|
3312
2970
|
error: {
|
|
@@ -3318,17 +2976,17 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3318
2976
|
fastify.log.debug("Fetching system metrics from all instances");
|
|
3319
2977
|
const allMetrics = [];
|
|
3320
2978
|
try {
|
|
3321
|
-
if ("scan" in
|
|
3322
|
-
const keys = await
|
|
2979
|
+
if ("scan" in platform16.cache && typeof platform16.cache.scan === "function") {
|
|
2980
|
+
const keys = await platform16.cache.scan("system-metrics:*");
|
|
3323
2981
|
for (const key of keys) {
|
|
3324
|
-
const metrics = await
|
|
2982
|
+
const metrics = await platform16.cache.get(key);
|
|
3325
2983
|
if (metrics) {
|
|
3326
2984
|
allMetrics.push(metrics);
|
|
3327
2985
|
}
|
|
3328
2986
|
}
|
|
3329
2987
|
} else {
|
|
3330
2988
|
const currentInstanceId = hostname();
|
|
3331
|
-
const metrics = await
|
|
2989
|
+
const metrics = await platform16.cache.get(`system-metrics:${currentInstanceId}`);
|
|
3332
2990
|
if (metrics) {
|
|
3333
2991
|
allMetrics.push(metrics);
|
|
3334
2992
|
}
|
|
@@ -3337,7 +2995,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3337
2995
|
} catch (scanError) {
|
|
3338
2996
|
fastify.log.warn({ err: scanError }, "Failed to scan platform.cache for system metrics");
|
|
3339
2997
|
const currentInstanceId = hostname();
|
|
3340
|
-
const metrics = await
|
|
2998
|
+
const metrics = await platform16.cache.get(`system-metrics:${currentInstanceId}`);
|
|
3341
2999
|
if (metrics) {
|
|
3342
3000
|
allMetrics.push(metrics);
|
|
3343
3001
|
}
|
|
@@ -3385,7 +3043,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3385
3043
|
}
|
|
3386
3044
|
};
|
|
3387
3045
|
} catch (error) {
|
|
3388
|
-
|
|
3046
|
+
platform16?.logger.error("Failed to fetch system metrics", error instanceof Error ? error : new Error(String(error)));
|
|
3389
3047
|
return reply.code(500).send({
|
|
3390
3048
|
ok: false,
|
|
3391
3049
|
error: {
|
|
@@ -3399,6 +3057,8 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3399
3057
|
for (const path3 of metricsHistoryPaths) {
|
|
3400
3058
|
fastify.get(path3, {
|
|
3401
3059
|
schema: {
|
|
3060
|
+
tags: ["Observability"],
|
|
3061
|
+
summary: "Historical time-series metrics",
|
|
3402
3062
|
querystring: {
|
|
3403
3063
|
type: "object",
|
|
3404
3064
|
properties: {
|
|
@@ -3481,7 +3141,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3481
3141
|
}
|
|
3482
3142
|
};
|
|
3483
3143
|
} catch (error) {
|
|
3484
|
-
|
|
3144
|
+
platform16?.logger.error("Failed to query historical metrics", error instanceof Error ? error : new Error(String(error)), { query });
|
|
3485
3145
|
return reply.code(500).send({
|
|
3486
3146
|
ok: false,
|
|
3487
3147
|
error: {
|
|
@@ -3495,6 +3155,8 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3495
3155
|
for (const path3 of metricsHeatmapPaths) {
|
|
3496
3156
|
fastify.get(path3, {
|
|
3497
3157
|
schema: {
|
|
3158
|
+
tags: ["Observability"],
|
|
3159
|
+
summary: "Metrics heatmap data",
|
|
3498
3160
|
querystring: {
|
|
3499
3161
|
type: "object",
|
|
3500
3162
|
properties: {
|
|
@@ -3570,7 +3232,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3570
3232
|
}
|
|
3571
3233
|
};
|
|
3572
3234
|
} catch (error) {
|
|
3573
|
-
|
|
3235
|
+
platform16?.logger.error("Failed to query heatmap data", error instanceof Error ? error : new Error(String(error)), { query });
|
|
3574
3236
|
return reply.code(500).send({
|
|
3575
3237
|
ok: false,
|
|
3576
3238
|
error: {
|
|
@@ -3581,310 +3243,11 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3581
3243
|
}
|
|
3582
3244
|
});
|
|
3583
3245
|
}
|
|
3584
|
-
for (const path3 of incidentsListPaths) {
|
|
3585
|
-
fastify.get(path3, {
|
|
3586
|
-
schema: {
|
|
3587
|
-
querystring: {
|
|
3588
|
-
type: "object",
|
|
3589
|
-
properties: {
|
|
3590
|
-
limit: { type: "integer", default: 50 },
|
|
3591
|
-
severity: {
|
|
3592
|
-
type: "string",
|
|
3593
|
-
enum: ["critical", "warning", "info"]
|
|
3594
|
-
},
|
|
3595
|
-
type: {
|
|
3596
|
-
type: "string",
|
|
3597
|
-
enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
|
|
3598
|
-
},
|
|
3599
|
-
from: { type: "integer" },
|
|
3600
|
-
to: { type: "integer" },
|
|
3601
|
-
includeResolved: { type: "boolean", default: false }
|
|
3602
|
-
}
|
|
3603
|
-
}
|
|
3604
|
-
}
|
|
3605
|
-
}, async (request, reply) => {
|
|
3606
|
-
if (!incidentStorage) {
|
|
3607
|
-
return reply.code(503).send({
|
|
3608
|
-
ok: false,
|
|
3609
|
-
error: {
|
|
3610
|
-
code: "INCIDENTS_NOT_CONFIGURED",
|
|
3611
|
-
message: "Incident storage is not configured"
|
|
3612
|
-
}
|
|
3613
|
-
});
|
|
3614
|
-
}
|
|
3615
|
-
try {
|
|
3616
|
-
const query = request.query;
|
|
3617
|
-
const incidents = await incidentStorage.queryIncidents({
|
|
3618
|
-
limit: query.limit ?? 50,
|
|
3619
|
-
severity: query.severity,
|
|
3620
|
-
type: query.type,
|
|
3621
|
-
from: query.from,
|
|
3622
|
-
to: query.to,
|
|
3623
|
-
includeResolved: query.includeResolved ?? false
|
|
3624
|
-
});
|
|
3625
|
-
const stats = await incidentStorage.getStats();
|
|
3626
|
-
return {
|
|
3627
|
-
ok: true,
|
|
3628
|
-
data: {
|
|
3629
|
-
incidents,
|
|
3630
|
-
summary: {
|
|
3631
|
-
total: stats.total,
|
|
3632
|
-
unresolved: stats.unresolved,
|
|
3633
|
-
bySeverity: stats.bySeverity,
|
|
3634
|
-
showing: incidents.length
|
|
3635
|
-
}
|
|
3636
|
-
}
|
|
3637
|
-
};
|
|
3638
|
-
} catch (error) {
|
|
3639
|
-
fastify.log.error({ err: error }, "Failed to list incidents");
|
|
3640
|
-
return reply.code(500).send({
|
|
3641
|
-
ok: false,
|
|
3642
|
-
error: {
|
|
3643
|
-
code: "INCIDENTS_LIST_ERROR",
|
|
3644
|
-
message: error instanceof Error ? error.message : "Failed to list incidents"
|
|
3645
|
-
}
|
|
3646
|
-
});
|
|
3647
|
-
}
|
|
3648
|
-
});
|
|
3649
|
-
}
|
|
3650
|
-
for (const path3 of incidentsDetailPaths) {
|
|
3651
|
-
fastify.get(path3, {
|
|
3652
|
-
schema: {
|
|
3653
|
-
params: {
|
|
3654
|
-
type: "object",
|
|
3655
|
-
properties: {
|
|
3656
|
-
id: { type: "string" }
|
|
3657
|
-
},
|
|
3658
|
-
required: ["id"]
|
|
3659
|
-
}
|
|
3660
|
-
}
|
|
3661
|
-
}, async (request, reply) => {
|
|
3662
|
-
if (!incidentStorage) {
|
|
3663
|
-
return reply.code(503).send({
|
|
3664
|
-
ok: false,
|
|
3665
|
-
error: {
|
|
3666
|
-
code: "INCIDENTS_NOT_CONFIGURED",
|
|
3667
|
-
message: "Incident storage is not configured"
|
|
3668
|
-
}
|
|
3669
|
-
});
|
|
3670
|
-
}
|
|
3671
|
-
const { id } = request.params;
|
|
3672
|
-
try {
|
|
3673
|
-
const incident = await incidentStorage.getIncident(id);
|
|
3674
|
-
if (!incident) {
|
|
3675
|
-
return reply.code(404).send({
|
|
3676
|
-
ok: false,
|
|
3677
|
-
error: {
|
|
3678
|
-
code: "INCIDENT_NOT_FOUND",
|
|
3679
|
-
message: `Incident ${id} not found`
|
|
3680
|
-
}
|
|
3681
|
-
});
|
|
3682
|
-
}
|
|
3683
|
-
if (platform19?.analytics) {
|
|
3684
|
-
platform19.analytics.track("incident.viewed", {
|
|
3685
|
-
incidentId: id,
|
|
3686
|
-
type: incident.type,
|
|
3687
|
-
severity: incident.severity,
|
|
3688
|
-
isResolved: !!incident.resolvedAt,
|
|
3689
|
-
hasAIAnalysis: !!incident.aiAnalysis
|
|
3690
|
-
}).catch(() => {
|
|
3691
|
-
});
|
|
3692
|
-
}
|
|
3693
|
-
return {
|
|
3694
|
-
ok: true,
|
|
3695
|
-
data: incident
|
|
3696
|
-
};
|
|
3697
|
-
} catch (error) {
|
|
3698
|
-
fastify.log.error({ err: error }, "Failed to get incident");
|
|
3699
|
-
return reply.code(500).send({
|
|
3700
|
-
ok: false,
|
|
3701
|
-
error: {
|
|
3702
|
-
code: "INCIDENT_GET_ERROR",
|
|
3703
|
-
message: error instanceof Error ? error.message : "Failed to get incident"
|
|
3704
|
-
}
|
|
3705
|
-
});
|
|
3706
|
-
}
|
|
3707
|
-
});
|
|
3708
|
-
}
|
|
3709
|
-
for (const path3 of incidentsCreatePaths) {
|
|
3710
|
-
fastify.post(path3, {
|
|
3711
|
-
schema: {
|
|
3712
|
-
body: {
|
|
3713
|
-
type: "object",
|
|
3714
|
-
properties: {
|
|
3715
|
-
type: {
|
|
3716
|
-
type: "string",
|
|
3717
|
-
enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
|
|
3718
|
-
},
|
|
3719
|
-
severity: {
|
|
3720
|
-
type: "string",
|
|
3721
|
-
enum: ["critical", "warning", "info"]
|
|
3722
|
-
},
|
|
3723
|
-
title: { type: "string" },
|
|
3724
|
-
details: { type: "string" },
|
|
3725
|
-
rootCause: { type: "string" },
|
|
3726
|
-
affectedServices: {
|
|
3727
|
-
type: "array",
|
|
3728
|
-
items: { type: "string" }
|
|
3729
|
-
},
|
|
3730
|
-
timestamp: { type: "number" },
|
|
3731
|
-
metadata: { type: "object" }
|
|
3732
|
-
},
|
|
3733
|
-
required: ["type", "severity", "title", "details"]
|
|
3734
|
-
}
|
|
3735
|
-
}
|
|
3736
|
-
}, async (request, reply) => {
|
|
3737
|
-
if (!incidentStorage) {
|
|
3738
|
-
return reply.code(503).send({
|
|
3739
|
-
ok: false,
|
|
3740
|
-
error: {
|
|
3741
|
-
code: "INCIDENT_STORAGE_UNAVAILABLE",
|
|
3742
|
-
message: "Incident storage is not initialized"
|
|
3743
|
-
}
|
|
3744
|
-
});
|
|
3745
|
-
}
|
|
3746
|
-
try {
|
|
3747
|
-
const payload = request.body;
|
|
3748
|
-
const incident = await incidentStorage.createIncident(payload);
|
|
3749
|
-
return {
|
|
3750
|
-
ok: true,
|
|
3751
|
-
data: incident,
|
|
3752
|
-
meta: {
|
|
3753
|
-
source: "incident-storage"
|
|
3754
|
-
}
|
|
3755
|
-
};
|
|
3756
|
-
} catch (error) {
|
|
3757
|
-
platform19?.logger.error("Failed to create incident", error instanceof Error ? error : new Error(String(error)));
|
|
3758
|
-
return reply.code(500).send({
|
|
3759
|
-
ok: false,
|
|
3760
|
-
error: {
|
|
3761
|
-
code: "INCIDENT_CREATE_ERROR",
|
|
3762
|
-
message: error instanceof Error ? error.message : "Failed to create incident"
|
|
3763
|
-
}
|
|
3764
|
-
});
|
|
3765
|
-
}
|
|
3766
|
-
});
|
|
3767
|
-
}
|
|
3768
|
-
for (const path3 of incidentsHistoryPaths) {
|
|
3769
|
-
fastify.get(path3, {
|
|
3770
|
-
schema: {
|
|
3771
|
-
querystring: {
|
|
3772
|
-
type: "object",
|
|
3773
|
-
properties: {
|
|
3774
|
-
limit: { type: "integer", minimum: 1, maximum: 500 },
|
|
3775
|
-
severity: { type: "string", enum: ["critical", "warning", "info"] },
|
|
3776
|
-
type: {
|
|
3777
|
-
type: "string",
|
|
3778
|
-
enum: ["error_rate", "latency_spike", "plugin_failure", "adapter_failure", "system_health", "custom"]
|
|
3779
|
-
},
|
|
3780
|
-
from: { type: "integer" },
|
|
3781
|
-
to: { type: "integer" },
|
|
3782
|
-
includeResolved: { type: "boolean" }
|
|
3783
|
-
}
|
|
3784
|
-
}
|
|
3785
|
-
}
|
|
3786
|
-
}, async (request, reply) => {
|
|
3787
|
-
if (!incidentStorage) {
|
|
3788
|
-
return reply.code(503).send({
|
|
3789
|
-
ok: false,
|
|
3790
|
-
error: {
|
|
3791
|
-
code: "INCIDENT_STORAGE_UNAVAILABLE",
|
|
3792
|
-
message: "Incident storage is not initialized"
|
|
3793
|
-
}
|
|
3794
|
-
});
|
|
3795
|
-
}
|
|
3796
|
-
try {
|
|
3797
|
-
const query = request.query;
|
|
3798
|
-
const incidents = await incidentStorage.queryIncidents({
|
|
3799
|
-
limit: query.limit ?? 50,
|
|
3800
|
-
severity: query.severity,
|
|
3801
|
-
type: query.type,
|
|
3802
|
-
from: query.from,
|
|
3803
|
-
to: query.to,
|
|
3804
|
-
includeResolved: query.includeResolved ?? false
|
|
3805
|
-
});
|
|
3806
|
-
return {
|
|
3807
|
-
ok: true,
|
|
3808
|
-
data: incidents,
|
|
3809
|
-
meta: {
|
|
3810
|
-
source: "incident-storage",
|
|
3811
|
-
count: incidents.length
|
|
3812
|
-
}
|
|
3813
|
-
};
|
|
3814
|
-
} catch (error) {
|
|
3815
|
-
platform19?.logger.error("Failed to query incidents", error instanceof Error ? error : new Error(String(error)));
|
|
3816
|
-
return reply.code(500).send({
|
|
3817
|
-
ok: false,
|
|
3818
|
-
error: {
|
|
3819
|
-
code: "INCIDENT_QUERY_ERROR",
|
|
3820
|
-
message: error instanceof Error ? error.message : "Failed to query incidents"
|
|
3821
|
-
}
|
|
3822
|
-
});
|
|
3823
|
-
}
|
|
3824
|
-
});
|
|
3825
|
-
}
|
|
3826
|
-
for (const path3 of incidentsResolvePaths) {
|
|
3827
|
-
fastify.post(path3, {
|
|
3828
|
-
schema: {
|
|
3829
|
-
params: {
|
|
3830
|
-
type: "object",
|
|
3831
|
-
properties: {
|
|
3832
|
-
id: { type: "string" }
|
|
3833
|
-
},
|
|
3834
|
-
required: ["id"]
|
|
3835
|
-
},
|
|
3836
|
-
body: {
|
|
3837
|
-
type: "object",
|
|
3838
|
-
properties: {
|
|
3839
|
-
resolutionNotes: { type: "string" }
|
|
3840
|
-
}
|
|
3841
|
-
}
|
|
3842
|
-
}
|
|
3843
|
-
}, async (request, reply) => {
|
|
3844
|
-
if (!incidentStorage) {
|
|
3845
|
-
return reply.code(503).send({
|
|
3846
|
-
ok: false,
|
|
3847
|
-
error: {
|
|
3848
|
-
code: "INCIDENT_STORAGE_UNAVAILABLE",
|
|
3849
|
-
message: "Incident storage is not initialized"
|
|
3850
|
-
}
|
|
3851
|
-
});
|
|
3852
|
-
}
|
|
3853
|
-
try {
|
|
3854
|
-
const { id } = request.params;
|
|
3855
|
-
const body = request.body;
|
|
3856
|
-
const incident = await incidentStorage.resolveIncident(id, body.resolutionNotes);
|
|
3857
|
-
if (!incident) {
|
|
3858
|
-
return reply.code(404).send({
|
|
3859
|
-
ok: false,
|
|
3860
|
-
error: {
|
|
3861
|
-
code: "INCIDENT_NOT_FOUND",
|
|
3862
|
-
message: `Incident with id ${id} not found`
|
|
3863
|
-
}
|
|
3864
|
-
});
|
|
3865
|
-
}
|
|
3866
|
-
return {
|
|
3867
|
-
ok: true,
|
|
3868
|
-
data: incident,
|
|
3869
|
-
meta: {
|
|
3870
|
-
source: "incident-storage"
|
|
3871
|
-
}
|
|
3872
|
-
};
|
|
3873
|
-
} catch (error) {
|
|
3874
|
-
platform19?.logger.error("Failed to resolve incident", error instanceof Error ? error : new Error(String(error)));
|
|
3875
|
-
return reply.code(500).send({
|
|
3876
|
-
ok: false,
|
|
3877
|
-
error: {
|
|
3878
|
-
code: "INCIDENT_RESOLVE_ERROR",
|
|
3879
|
-
message: error instanceof Error ? error.message : "Failed to resolve incident"
|
|
3880
|
-
}
|
|
3881
|
-
});
|
|
3882
|
-
}
|
|
3883
|
-
});
|
|
3884
|
-
}
|
|
3885
3246
|
for (const path3 of insightsChatPaths) {
|
|
3886
3247
|
fastify.post(path3, {
|
|
3887
3248
|
schema: {
|
|
3249
|
+
tags: ["Observability"],
|
|
3250
|
+
summary: "AI-powered observability insights chat",
|
|
3888
3251
|
body: {
|
|
3889
3252
|
type: "object",
|
|
3890
3253
|
properties: {
|
|
@@ -3893,7 +3256,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3893
3256
|
type: "object",
|
|
3894
3257
|
properties: {
|
|
3895
3258
|
includeMetrics: { type: "boolean" },
|
|
3896
|
-
includeIncidents: { type: "boolean" },
|
|
3897
3259
|
includeHistory: { type: "boolean" },
|
|
3898
3260
|
timeRange: { type: "string", enum: ["1h", "6h", "24h", "7d"] },
|
|
3899
3261
|
plugins: { type: "array", items: { type: "string" } }
|
|
@@ -3904,7 +3266,7 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3904
3266
|
}
|
|
3905
3267
|
}
|
|
3906
3268
|
}, async (request, reply) => {
|
|
3907
|
-
if (!
|
|
3269
|
+
if (!platform16?.llm) {
|
|
3908
3270
|
return reply.code(503).send({
|
|
3909
3271
|
ok: false,
|
|
3910
3272
|
error: {
|
|
@@ -3917,7 +3279,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3917
3279
|
const body = request.body;
|
|
3918
3280
|
const contextConfig = {
|
|
3919
3281
|
includeMetrics: body.context?.includeMetrics ?? true,
|
|
3920
|
-
includeIncidents: body.context?.includeIncidents ?? true,
|
|
3921
3282
|
includeHistory: body.context?.includeHistory ?? true,
|
|
3922
3283
|
timeRange: body.context?.timeRange ?? "24h",
|
|
3923
3284
|
plugins: body.context?.plugins ?? []
|
|
@@ -3958,26 +3319,6 @@ async function registerObservabilityRoutes(fastify, config, repoRoot, historical
|
|
|
3958
3319
|
fastify.log.warn({ err: metricsError }, "Failed to build metrics context for insights");
|
|
3959
3320
|
}
|
|
3960
3321
|
}
|
|
3961
|
-
if (contextConfig.includeIncidents && incidentStorage) {
|
|
3962
|
-
try {
|
|
3963
|
-
const incidents = await incidentStorage.queryIncidents({ limit: 10 });
|
|
3964
|
-
if (incidents.length > 0) {
|
|
3965
|
-
contextText += `
|
|
3966
|
-
## Recent Incidents (${incidents.length})
|
|
3967
|
-
`;
|
|
3968
|
-
for (const incident of incidents) {
|
|
3969
|
-
contextText += `- [${incident.severity.toUpperCase()}] ${incident.title}
|
|
3970
|
-
`;
|
|
3971
|
-
if (incident.details) {
|
|
3972
|
-
contextText += ` Details: ${incident.details.slice(0, 100)}${incident.details.length > 100 ? "..." : ""}
|
|
3973
|
-
`;
|
|
3974
|
-
}
|
|
3975
|
-
}
|
|
3976
|
-
}
|
|
3977
|
-
} catch (incidentError) {
|
|
3978
|
-
fastify.log.warn({ err: incidentError }, "Failed to fetch incidents for insights context");
|
|
3979
|
-
}
|
|
3980
|
-
}
|
|
3981
3322
|
const prompt = `You are an AI assistant analyzing a software platform's observability data.
|
|
3982
3323
|
|
|
3983
3324
|
${contextText}
|
|
@@ -3991,15 +3332,15 @@ Provide a clear, actionable response based on the data above. Include:
|
|
|
3991
3332
|
|
|
3992
3333
|
Be concise but thorough. Use markdown formatting.`;
|
|
3993
3334
|
fastify.log.debug({ question: body.question, contextLength: contextText.length }, "Calling LLM for insights");
|
|
3994
|
-
const result = await
|
|
3335
|
+
const result = await platform16.llm.complete(prompt, {
|
|
3995
3336
|
systemPrompt: "You are a DevOps and SRE expert assistant. Analyze system metrics and provide actionable insights. Be concise, technical, and helpful.",
|
|
3996
3337
|
temperature: 0.7,
|
|
3997
3338
|
maxTokens: 1e3
|
|
3998
3339
|
});
|
|
3999
3340
|
const totalTokens = result.usage.promptTokens + result.usage.completionTokens;
|
|
4000
3341
|
fastify.log.debug({ tokensUsed: totalTokens }, "LLM response received for insights");
|
|
4001
|
-
if (
|
|
4002
|
-
|
|
3342
|
+
if (platform16.analytics) {
|
|
3343
|
+
platform16.analytics.track("ai_insights.chat", {
|
|
4003
3344
|
questionLength: body.question.length,
|
|
4004
3345
|
contextIncluded: Object.keys(contextConfig).filter((k) => contextConfig[k]),
|
|
4005
3346
|
timeRange: contextConfig.timeRange,
|
|
@@ -4026,246 +3367,27 @@ Be concise but thorough. Use markdown formatting.`;
|
|
|
4026
3367
|
source: "llm-insights",
|
|
4027
3368
|
model: result.model
|
|
4028
3369
|
}
|
|
4029
|
-
};
|
|
4030
|
-
} catch (error) {
|
|
4031
|
-
|
|
4032
|
-
if (
|
|
4033
|
-
|
|
4034
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
4035
|
-
questionLength: request.body?.question?.length ?? 0
|
|
4036
|
-
}).catch(() => {
|
|
4037
|
-
});
|
|
4038
|
-
}
|
|
4039
|
-
return reply.code(500).send({
|
|
4040
|
-
ok: false,
|
|
4041
|
-
error: {
|
|
4042
|
-
code: "INSIGHTS_ERROR",
|
|
4043
|
-
message: error instanceof Error ? error.message : "Failed to generate insights"
|
|
4044
|
-
}
|
|
4045
|
-
});
|
|
4046
|
-
}
|
|
4047
|
-
});
|
|
4048
|
-
}
|
|
4049
|
-
for (const path3 of incidentsAnalyzePaths) {
|
|
4050
|
-
fastify.post(path3, async (request, reply) => {
|
|
4051
|
-
if (!incidentStorage) {
|
|
4052
|
-
return reply.code(503).send({
|
|
4053
|
-
ok: false,
|
|
4054
|
-
error: {
|
|
4055
|
-
code: "INCIDENTS_NOT_CONFIGURED",
|
|
4056
|
-
message: "Incident storage is not configured"
|
|
4057
|
-
}
|
|
4058
|
-
});
|
|
4059
|
-
}
|
|
4060
|
-
const { id } = request.params;
|
|
4061
|
-
try {
|
|
4062
|
-
const incident = await incidentStorage.getIncident(id);
|
|
4063
|
-
if (!incident) {
|
|
4064
|
-
return reply.code(404).send({
|
|
4065
|
-
ok: false,
|
|
4066
|
-
error: {
|
|
4067
|
-
code: "INCIDENT_NOT_FOUND",
|
|
4068
|
-
message: `Incident ${id} not found`
|
|
4069
|
-
}
|
|
4070
|
-
});
|
|
4071
|
-
}
|
|
4072
|
-
if (incident.aiAnalysis && incident.aiAnalyzedAt) {
|
|
4073
|
-
const ageMs = Date.now() - incident.aiAnalyzedAt;
|
|
4074
|
-
const ageMinutes = Math.floor(ageMs / 6e4);
|
|
4075
|
-
if (ageMs < 60 * 60 * 1e3) {
|
|
4076
|
-
return {
|
|
4077
|
-
ok: true,
|
|
4078
|
-
data: {
|
|
4079
|
-
...incident.aiAnalysis,
|
|
4080
|
-
cached: true,
|
|
4081
|
-
analyzedAt: incident.aiAnalyzedAt,
|
|
4082
|
-
ageMinutes
|
|
4083
|
-
}
|
|
4084
|
-
};
|
|
4085
|
-
}
|
|
4086
|
-
}
|
|
4087
|
-
const analyzer = new IncidentAnalyzer(
|
|
4088
|
-
{
|
|
4089
|
-
debug: process.env.NODE_ENV !== "production"
|
|
4090
|
-
},
|
|
4091
|
-
fastify.log
|
|
4092
|
-
);
|
|
4093
|
-
const analysis = await analyzer.analyze(incident);
|
|
4094
|
-
await incidentStorage.updateAIAnalysis(id, analysis);
|
|
4095
|
-
if (platform19?.analytics) {
|
|
4096
|
-
platform19.analytics.track("incident.analyzed", {
|
|
4097
|
-
incidentId: id,
|
|
4098
|
-
type: incident.type,
|
|
4099
|
-
severity: incident.severity,
|
|
4100
|
-
rootCausesCount: analysis.rootCauses.length,
|
|
4101
|
-
recommendationsCount: analysis.recommendations.length
|
|
4102
|
-
}).catch(() => {
|
|
4103
|
-
});
|
|
4104
|
-
}
|
|
4105
|
-
return {
|
|
4106
|
-
ok: true,
|
|
4107
|
-
data: {
|
|
4108
|
-
...analysis,
|
|
4109
|
-
cached: false
|
|
4110
|
-
}
|
|
4111
|
-
};
|
|
4112
|
-
} catch (error) {
|
|
4113
|
-
fastify.log.error({ err: error }, "Failed to analyze incident");
|
|
4114
|
-
if (platform19?.analytics) {
|
|
4115
|
-
platform19.analytics.track("incident.analysis_error", {
|
|
4116
|
-
incidentId: id,
|
|
4117
|
-
error: error instanceof Error ? error.message : "Unknown error"
|
|
4118
|
-
}).catch(() => {
|
|
4119
|
-
});
|
|
4120
|
-
}
|
|
4121
|
-
return reply.code(500).send({
|
|
4122
|
-
ok: false,
|
|
4123
|
-
error: {
|
|
4124
|
-
code: "ANALYSIS_ERROR",
|
|
4125
|
-
message: error instanceof Error ? error.message : "Failed to analyze incident"
|
|
4126
|
-
}
|
|
4127
|
-
});
|
|
4128
|
-
}
|
|
4129
|
-
});
|
|
4130
|
-
}
|
|
4131
|
-
const testCreateIncidentPaths = resolvePaths(basePath, "/test/create-incident");
|
|
4132
|
-
for (const path3 of testCreateIncidentPaths) {
|
|
4133
|
-
fastify.post(path3, async (request, reply) => {
|
|
4134
|
-
if (!incidentStorage) {
|
|
4135
|
-
return reply.code(500).send({
|
|
4136
|
-
ok: false,
|
|
4137
|
-
error: { code: "INCIDENT_STORAGE_NOT_INITIALIZED", message: "Incident storage not initialized" }
|
|
4138
|
-
});
|
|
4139
|
-
}
|
|
4140
|
-
try {
|
|
4141
|
-
const {
|
|
4142
|
-
type = "custom",
|
|
4143
|
-
severity = "warning",
|
|
4144
|
-
title = "Test Incident",
|
|
4145
|
-
details,
|
|
4146
|
-
relatedData
|
|
4147
|
-
} = request.body;
|
|
4148
|
-
const incident = await incidentStorage.createIncident({
|
|
4149
|
-
type,
|
|
4150
|
-
severity,
|
|
4151
|
-
title,
|
|
4152
|
-
details: details || "This is a test incident created manually for testing purposes.",
|
|
4153
|
-
timestamp: Date.now(),
|
|
4154
|
-
metadata: {
|
|
4155
|
-
testMode: true,
|
|
4156
|
-
createdVia: "test-endpoint"
|
|
4157
|
-
},
|
|
4158
|
-
relatedData: relatedData || {
|
|
4159
|
-
timeline: [
|
|
4160
|
-
{
|
|
4161
|
-
timestamp: Date.now(),
|
|
4162
|
-
event: "Test incident created via /test/create-incident",
|
|
4163
|
-
source: "manual"
|
|
4164
|
-
}
|
|
4165
|
-
]
|
|
4166
|
-
}
|
|
4167
|
-
});
|
|
4168
|
-
fastify.log.info({ id: incident.id }, "Test incident created");
|
|
4169
|
-
return { ok: true, data: incident };
|
|
4170
|
-
} catch (error) {
|
|
4171
|
-
fastify.log.error({ err: error }, "Failed to create test incident");
|
|
4172
|
-
return reply.code(500).send({
|
|
4173
|
-
ok: false,
|
|
4174
|
-
error: {
|
|
4175
|
-
code: "TEST_INCIDENT_CREATION_FAILED",
|
|
4176
|
-
message: error instanceof Error ? error.message : "Failed to create test incident"
|
|
4177
|
-
}
|
|
4178
|
-
});
|
|
4179
|
-
}
|
|
4180
|
-
});
|
|
4181
|
-
}
|
|
4182
|
-
const testTriggerErrorsPaths = resolvePaths(basePath, "/test/trigger-errors");
|
|
4183
|
-
for (const path3 of testTriggerErrorsPaths) {
|
|
4184
|
-
fastify.get(path3, async (request, reply) => {
|
|
4185
|
-
const { count = 10 } = request.query;
|
|
4186
|
-
const errorCount = Math.min(Math.max(1, parseInt(count, 10) || 10), 100);
|
|
4187
|
-
fastify.log.info(`Triggering ${errorCount} test errors`);
|
|
4188
|
-
for (let i = 0; i < errorCount; i++) {
|
|
4189
|
-
platform19?.logger.error(`Test error ${i + 1}/${errorCount}`, void 0, {
|
|
4190
|
-
testMode: true,
|
|
4191
|
-
errorNumber: i + 1,
|
|
4192
|
-
totalErrors: errorCount
|
|
4193
|
-
});
|
|
4194
|
-
if (i === errorCount - 1) {
|
|
4195
|
-
return reply.code(500).send({
|
|
4196
|
-
ok: false,
|
|
4197
|
-
error: {
|
|
4198
|
-
code: "TEST_ERROR_TRIGGERED",
|
|
4199
|
-
message: `Generated ${errorCount} test errors. Check /observability/incidents in ~30 seconds for auto-created incident.`
|
|
4200
|
-
}
|
|
3370
|
+
};
|
|
3371
|
+
} catch (error) {
|
|
3372
|
+
platform16?.logger.error("Failed to generate insights", error instanceof Error ? error : new Error(String(error)));
|
|
3373
|
+
if (platform16?.analytics) {
|
|
3374
|
+
platform16.analytics.track("ai_insights.error", {
|
|
3375
|
+
error: error instanceof Error ? error.message : "Unknown error",
|
|
3376
|
+
questionLength: request.body?.question?.length ?? 0
|
|
3377
|
+
}).catch(() => {
|
|
4201
3378
|
});
|
|
4202
3379
|
}
|
|
4203
|
-
|
|
4204
|
-
|
|
4205
|
-
|
|
4206
|
-
|
|
4207
|
-
|
|
4208
|
-
|
|
4209
|
-
}
|
|
4210
|
-
const testSimulateLatencyPaths = resolvePaths(basePath, "/test/simulate-latency");
|
|
4211
|
-
for (const path3 of testSimulateLatencyPaths) {
|
|
4212
|
-
fastify.get(path3, async (request, reply) => {
|
|
4213
|
-
const { delay = 2e3 } = request.query;
|
|
4214
|
-
const delayMs = Math.min(Math.max(100, parseInt(delay, 10) || 2e3), 1e4);
|
|
4215
|
-
fastify.log.info(`Simulating ${delayMs}ms latency`);
|
|
4216
|
-
await new Promise((resolve3) => {
|
|
4217
|
-
setTimeout(resolve3, delayMs);
|
|
4218
|
-
});
|
|
4219
|
-
return {
|
|
4220
|
-
ok: true,
|
|
4221
|
-
message: `Simulated ${delayMs}ms latency. Call this endpoint multiple times to trigger latency_spike incident.`,
|
|
4222
|
-
actualDelay: delayMs
|
|
4223
|
-
};
|
|
4224
|
-
});
|
|
4225
|
-
}
|
|
4226
|
-
const testBulkErrorsPaths = resolvePaths(basePath, "/test/bulk-errors");
|
|
4227
|
-
for (const path3 of testBulkErrorsPaths) {
|
|
4228
|
-
fastify.post(path3, async (request, reply) => {
|
|
4229
|
-
const { successCount = 10, errorCount = 50 } = request.body;
|
|
4230
|
-
fastify.log.info({ successCount, errorCount }, "Generating bulk requests for incident testing");
|
|
4231
|
-
for (let i = 0; i < successCount; i++) {
|
|
4232
|
-
platform19?.logger.info(`Bulk test - success ${i + 1}/${successCount}`);
|
|
4233
|
-
}
|
|
4234
|
-
const testEndpoints = [
|
|
4235
|
-
"POST /api/v1/test/endpoint-a",
|
|
4236
|
-
"GET /api/v1/test/endpoint-b",
|
|
4237
|
-
"PUT /api/v1/test/endpoint-c",
|
|
4238
|
-
"DELETE /api/v1/test/endpoint-d",
|
|
4239
|
-
"POST /api/v1/test/endpoint-e"
|
|
4240
|
-
];
|
|
4241
|
-
for (let i = 0; i < errorCount; i++) {
|
|
4242
|
-
const endpoint = testEndpoints[i % testEndpoints.length];
|
|
4243
|
-
const errorTypes = [
|
|
4244
|
-
"Connection timeout",
|
|
4245
|
-
"Validation failed",
|
|
4246
|
-
"Database query error",
|
|
4247
|
-
"Authentication failed",
|
|
4248
|
-
"Rate limit exceeded"
|
|
4249
|
-
];
|
|
4250
|
-
const errorType = errorTypes[i % errorTypes.length];
|
|
4251
|
-
platform19?.logger.error(`${errorType}: ${endpoint}`, void 0, {
|
|
4252
|
-
testMode: true,
|
|
4253
|
-
bulkTest: true,
|
|
4254
|
-
errorIndex: i,
|
|
4255
|
-
endpoint
|
|
3380
|
+
return reply.code(500).send({
|
|
3381
|
+
ok: false,
|
|
3382
|
+
error: {
|
|
3383
|
+
code: "INSIGHTS_ERROR",
|
|
3384
|
+
message: error instanceof Error ? error.message : "Failed to generate insights"
|
|
3385
|
+
}
|
|
4256
3386
|
});
|
|
4257
3387
|
}
|
|
4258
|
-
const totalRequests = successCount + errorCount;
|
|
4259
|
-
const errorRate = errorCount / totalRequests * 100;
|
|
4260
|
-
return {
|
|
4261
|
-
ok: true,
|
|
4262
|
-
message: `Generated ${totalRequests} requests (${successCount} success, ${errorCount} errors)`,
|
|
4263
|
-
errorRate: `${errorRate.toFixed(1)}%`,
|
|
4264
|
-
note: "Incident should be auto-created in next detection cycle (~30s)"
|
|
4265
|
-
};
|
|
4266
3388
|
});
|
|
4267
3389
|
}
|
|
4268
|
-
fastify.log.info("Observability routes registered
|
|
3390
|
+
fastify.log.info("Observability routes registered");
|
|
4269
3391
|
}
|
|
4270
3392
|
async function registerAnalyticsRoutes(fastify, config) {
|
|
4271
3393
|
const basePath = normalizeBasePath(config.basePath);
|
|
@@ -5598,7 +4720,7 @@ function generateFallbackSummary(stats, logs) {
|
|
|
5598
4720
|
return summary;
|
|
5599
4721
|
}
|
|
5600
4722
|
async function registerLogRoutes(server, config, eventHub) {
|
|
5601
|
-
server.get("/api/v1/logs", async (request, reply) => {
|
|
4723
|
+
server.get("/api/v1/logs", { schema: { tags: ["Logs"], summary: "Query logs with filters" } }, async (request, reply) => {
|
|
5602
4724
|
try {
|
|
5603
4725
|
const limit = request.query.limit ? parseInt(request.query.limit, 10) : 100;
|
|
5604
4726
|
const offset = request.query.offset ? parseInt(request.query.offset, 10) : 0;
|
|
@@ -5656,6 +4778,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5656
4778
|
});
|
|
5657
4779
|
server.get(
|
|
5658
4780
|
"/api/v1/logs/:id",
|
|
4781
|
+
{ schema: { tags: ["Logs"], summary: "Get log entry by ID" } },
|
|
5659
4782
|
async (request, reply) => {
|
|
5660
4783
|
try {
|
|
5661
4784
|
const log = await platform.logs.getById(request.params.id);
|
|
@@ -5689,6 +4812,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5689
4812
|
);
|
|
5690
4813
|
server.get(
|
|
5691
4814
|
"/api/v1/logs/:id/related",
|
|
4815
|
+
{ schema: { tags: ["Logs"], summary: "Get logs related to a specific entry" } },
|
|
5692
4816
|
async (request, reply) => {
|
|
5693
4817
|
try {
|
|
5694
4818
|
const log = await platform.logs.getById(request.params.id);
|
|
@@ -5717,7 +4841,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5717
4841
|
}
|
|
5718
4842
|
}
|
|
5719
4843
|
);
|
|
5720
|
-
server.get("/api/v1/logs/stream", async (request, reply) => {
|
|
4844
|
+
server.get("/api/v1/logs/stream", { schema: { hide: true } }, async (request, reply) => {
|
|
5721
4845
|
const caps = platform.logs.getCapabilities();
|
|
5722
4846
|
if (!caps.hasStreaming) {
|
|
5723
4847
|
return reply.code(503).send({
|
|
@@ -5775,7 +4899,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5775
4899
|
await new Promise(() => {
|
|
5776
4900
|
});
|
|
5777
4901
|
});
|
|
5778
|
-
server.get("/api/v1/logs/stats", async (request, reply) => {
|
|
4902
|
+
server.get("/api/v1/logs/stats", { schema: { tags: ["Logs"], summary: "Get log storage statistics" } }, async (request, reply) => {
|
|
5779
4903
|
try {
|
|
5780
4904
|
const stats = await platform.logs.getStats();
|
|
5781
4905
|
const caps = platform.logs.getCapabilities();
|
|
@@ -5805,7 +4929,7 @@ async function registerLogRoutes(server, config, eventHub) {
|
|
|
5805
4929
|
});
|
|
5806
4930
|
}
|
|
5807
4931
|
});
|
|
5808
|
-
server.post("/api/v1/logs/summarize", async (request, reply) => {
|
|
4932
|
+
server.post("/api/v1/logs/summarize", { schema: { tags: ["Logs"], summary: "AI-powered log summarization" } }, async (request, reply) => {
|
|
5809
4933
|
const { timeRange, filters, groupBy, question, includeContext } = request.body;
|
|
5810
4934
|
try {
|
|
5811
4935
|
const query = {
|
|
@@ -6197,7 +5321,7 @@ async function registerDebugRoutes(fastify, config) {
|
|
|
6197
5321
|
}
|
|
6198
5322
|
|
|
6199
5323
|
// src/services/historical-metrics.ts
|
|
6200
|
-
var
|
|
5324
|
+
var DEFAULT_CONFIG = {
|
|
6201
5325
|
intervalMs: 5e3,
|
|
6202
5326
|
maxPoints: {
|
|
6203
5327
|
"1m": 12,
|
|
@@ -6216,7 +5340,7 @@ var HistoricalMetricsCollector = class {
|
|
|
6216
5340
|
startTimeMs = Date.now();
|
|
6217
5341
|
constructor(cache, config = {}, logger = console) {
|
|
6218
5342
|
this.cache = cache;
|
|
6219
|
-
this.config = { ...
|
|
5343
|
+
this.config = { ...DEFAULT_CONFIG, ...config, maxPoints: { ...DEFAULT_CONFIG.maxPoints, ...config.maxPoints } };
|
|
6220
5344
|
this.logger = logger;
|
|
6221
5345
|
}
|
|
6222
5346
|
/**
|
|
@@ -6516,754 +5640,6 @@ var HistoricalMetricsCollector = class {
|
|
|
6516
5640
|
}
|
|
6517
5641
|
}
|
|
6518
5642
|
};
|
|
6519
|
-
var DEFAULT_CONFIG3 = {
|
|
6520
|
-
ttlMs: 30 * 24 * 60 * 60 * 1e3,
|
|
6521
|
-
// 30 days (not used with SQLite, kept for compatibility)
|
|
6522
|
-
maxIncidents: 1e3,
|
|
6523
|
-
// Not enforced with SQLite, use retention policy instead
|
|
6524
|
-
debug: false
|
|
6525
|
-
};
|
|
6526
|
-
var IncidentStorage = class {
|
|
6527
|
-
db;
|
|
6528
|
-
config;
|
|
6529
|
-
logger;
|
|
6530
|
-
constructor(db, config = {}, logger = console) {
|
|
6531
|
-
this.db = db;
|
|
6532
|
-
this.config = { ...DEFAULT_CONFIG3, ...config };
|
|
6533
|
-
this.logger = logger;
|
|
6534
|
-
}
|
|
6535
|
-
/**
|
|
6536
|
-
* Create a new incident record
|
|
6537
|
-
*/
|
|
6538
|
-
async createIncident(payload) {
|
|
6539
|
-
const id = `inc-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
|
|
6540
|
-
const timestamp = payload.timestamp ?? Date.now();
|
|
6541
|
-
if (!payload.type || !payload.severity || !payload.title) {
|
|
6542
|
-
throw new Error("Incident must have type, severity, and title");
|
|
6543
|
-
}
|
|
6544
|
-
const relatedLogsCount = payload.relatedData?.logs?.errorCount ?? 0;
|
|
6545
|
-
const relatedLogsSample = payload.relatedData?.logs ? JSON.stringify(payload.relatedData.logs) : null;
|
|
6546
|
-
const relatedMetrics = payload.relatedData?.metrics ? JSON.stringify(payload.relatedData.metrics) : null;
|
|
6547
|
-
const timeline = payload.relatedData?.timeline ? JSON.stringify(payload.relatedData.timeline) : null;
|
|
6548
|
-
await this.db.query(
|
|
6549
|
-
`INSERT INTO incidents (
|
|
6550
|
-
id, type, severity, title, details, timestamp,
|
|
6551
|
-
resolved_at, resolution_notes,
|
|
6552
|
-
affected_services, metadata,
|
|
6553
|
-
related_logs_count, related_logs_sample, related_metrics, timeline
|
|
6554
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
6555
|
-
[
|
|
6556
|
-
id,
|
|
6557
|
-
payload.type,
|
|
6558
|
-
payload.severity,
|
|
6559
|
-
payload.title,
|
|
6560
|
-
payload.details,
|
|
6561
|
-
timestamp,
|
|
6562
|
-
payload.resolvedAt ?? null,
|
|
6563
|
-
payload.resolutionNotes ?? null,
|
|
6564
|
-
payload.affectedServices ? JSON.stringify(payload.affectedServices) : null,
|
|
6565
|
-
payload.metadata ? JSON.stringify(payload.metadata) : null,
|
|
6566
|
-
relatedLogsCount,
|
|
6567
|
-
relatedLogsSample,
|
|
6568
|
-
relatedMetrics,
|
|
6569
|
-
timeline
|
|
6570
|
-
]
|
|
6571
|
-
);
|
|
6572
|
-
this.log("info", "Incident created", {
|
|
6573
|
-
id,
|
|
6574
|
-
type: payload.type,
|
|
6575
|
-
severity: payload.severity
|
|
6576
|
-
});
|
|
6577
|
-
const created = await this.getIncident(id);
|
|
6578
|
-
if (!created) {
|
|
6579
|
-
throw new Error("Failed to retrieve created incident");
|
|
6580
|
-
}
|
|
6581
|
-
return created;
|
|
6582
|
-
}
|
|
6583
|
-
/**
|
|
6584
|
-
* Get incident by ID
|
|
6585
|
-
*/
|
|
6586
|
-
async getIncident(id) {
|
|
6587
|
-
const result = await this.db.query(
|
|
6588
|
-
"SELECT * FROM incidents WHERE id = ?",
|
|
6589
|
-
[id]
|
|
6590
|
-
);
|
|
6591
|
-
if (result.rows.length === 0) {
|
|
6592
|
-
return null;
|
|
6593
|
-
}
|
|
6594
|
-
return this.rowToIncident(result.rows[0]);
|
|
6595
|
-
}
|
|
6596
|
-
/**
|
|
6597
|
-
* Query incidents with filters
|
|
6598
|
-
*/
|
|
6599
|
-
async queryIncidents(options = {}) {
|
|
6600
|
-
const {
|
|
6601
|
-
limit = 50,
|
|
6602
|
-
severity,
|
|
6603
|
-
type,
|
|
6604
|
-
from,
|
|
6605
|
-
to,
|
|
6606
|
-
includeResolved = false
|
|
6607
|
-
} = options;
|
|
6608
|
-
const conditions = [];
|
|
6609
|
-
const params = [];
|
|
6610
|
-
if (severity) {
|
|
6611
|
-
const severityList = Array.isArray(severity) ? severity : [severity];
|
|
6612
|
-
conditions.push(`severity IN (${severityList.map(() => "?").join(", ")})`);
|
|
6613
|
-
params.push(...severityList);
|
|
6614
|
-
}
|
|
6615
|
-
if (type) {
|
|
6616
|
-
const typeList = Array.isArray(type) ? type : [type];
|
|
6617
|
-
conditions.push(`type IN (${typeList.map(() => "?").join(", ")})`);
|
|
6618
|
-
params.push(...typeList);
|
|
6619
|
-
}
|
|
6620
|
-
if (from) {
|
|
6621
|
-
conditions.push("timestamp >= ?");
|
|
6622
|
-
params.push(from);
|
|
6623
|
-
}
|
|
6624
|
-
if (to) {
|
|
6625
|
-
conditions.push("timestamp <= ?");
|
|
6626
|
-
params.push(to);
|
|
6627
|
-
}
|
|
6628
|
-
if (!includeResolved) {
|
|
6629
|
-
conditions.push("resolved_at IS NULL");
|
|
6630
|
-
}
|
|
6631
|
-
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
6632
|
-
const sql = `
|
|
6633
|
-
SELECT * FROM incidents
|
|
6634
|
-
${whereClause}
|
|
6635
|
-
ORDER BY timestamp DESC
|
|
6636
|
-
LIMIT ?
|
|
6637
|
-
`;
|
|
6638
|
-
const result = await this.db.query(sql, [...params, limit]);
|
|
6639
|
-
return result.rows.map((row) => this.rowToIncident(row));
|
|
6640
|
-
}
|
|
6641
|
-
/**
|
|
6642
|
-
* Resolve an incident
|
|
6643
|
-
*/
|
|
6644
|
-
async resolveIncident(id, resolutionNotes) {
|
|
6645
|
-
const resolvedAt = Date.now();
|
|
6646
|
-
const result = await this.db.query(
|
|
6647
|
-
`UPDATE incidents
|
|
6648
|
-
SET resolved_at = ?, resolution_notes = ?
|
|
6649
|
-
WHERE id = ?`,
|
|
6650
|
-
[resolvedAt, resolutionNotes ?? null, id]
|
|
6651
|
-
);
|
|
6652
|
-
if (result.rowCount === 0) {
|
|
6653
|
-
return null;
|
|
6654
|
-
}
|
|
6655
|
-
this.log("info", "Incident resolved", { id, resolvedAt });
|
|
6656
|
-
const incident = await this.getIncident(id);
|
|
6657
|
-
if (incident && platform.analytics) {
|
|
6658
|
-
const durationMs = resolvedAt - incident.timestamp;
|
|
6659
|
-
const durationMinutes = Math.floor(durationMs / 6e4);
|
|
6660
|
-
platform.analytics.track("incident.resolved", {
|
|
6661
|
-
incidentId: id,
|
|
6662
|
-
type: incident.type,
|
|
6663
|
-
severity: incident.severity,
|
|
6664
|
-
durationMs,
|
|
6665
|
-
durationMinutes,
|
|
6666
|
-
hasResolutionNotes: !!resolutionNotes,
|
|
6667
|
-
wasAnalyzed: !!incident.aiAnalysis
|
|
6668
|
-
}).catch(() => {
|
|
6669
|
-
});
|
|
6670
|
-
}
|
|
6671
|
-
return incident;
|
|
6672
|
-
}
|
|
6673
|
-
/**
|
|
6674
|
-
* Delete an incident
|
|
6675
|
-
*/
|
|
6676
|
-
async deleteIncident(id) {
|
|
6677
|
-
const incident = await this.getIncident(id);
|
|
6678
|
-
const result = await this.db.query("DELETE FROM incidents WHERE id = ?", [id]);
|
|
6679
|
-
const deleted = result.rowCount > 0;
|
|
6680
|
-
if (deleted) {
|
|
6681
|
-
this.log("info", "Incident deleted", { id });
|
|
6682
|
-
if (incident && platform.analytics) {
|
|
6683
|
-
platform.analytics.track("incident.deleted", {
|
|
6684
|
-
incidentId: id,
|
|
6685
|
-
type: incident.type,
|
|
6686
|
-
severity: incident.severity,
|
|
6687
|
-
wasResolved: !!incident.resolvedAt,
|
|
6688
|
-
wasAnalyzed: !!incident.aiAnalysis
|
|
6689
|
-
}).catch(() => {
|
|
6690
|
-
});
|
|
6691
|
-
}
|
|
6692
|
-
}
|
|
6693
|
-
return deleted;
|
|
6694
|
-
}
|
|
6695
|
-
/**
|
|
6696
|
-
* Get statistics about stored incidents
|
|
6697
|
-
*/
|
|
6698
|
-
async getStats() {
|
|
6699
|
-
const totalResult = await this.db.query(`
|
|
6700
|
-
SELECT
|
|
6701
|
-
COUNT(*) as total,
|
|
6702
|
-
MIN(timestamp) as oldest,
|
|
6703
|
-
MAX(timestamp) as newest
|
|
6704
|
-
FROM incidents
|
|
6705
|
-
`);
|
|
6706
|
-
const { total, oldest, newest } = totalResult.rows[0];
|
|
6707
|
-
const severityResult = await this.db.query("SELECT severity, COUNT(*) as count FROM incidents GROUP BY severity");
|
|
6708
|
-
const bySeverity = {
|
|
6709
|
-
critical: 0,
|
|
6710
|
-
warning: 0,
|
|
6711
|
-
info: 0
|
|
6712
|
-
};
|
|
6713
|
-
for (const row of severityResult.rows) {
|
|
6714
|
-
bySeverity[row.severity] = row.count;
|
|
6715
|
-
}
|
|
6716
|
-
const typeResult = await this.db.query("SELECT type, COUNT(*) as count FROM incidents GROUP BY type");
|
|
6717
|
-
const byType = {};
|
|
6718
|
-
for (const row of typeResult.rows) {
|
|
6719
|
-
byType[row.type] = row.count;
|
|
6720
|
-
}
|
|
6721
|
-
const resolvedResult = await this.db.query(`
|
|
6722
|
-
SELECT
|
|
6723
|
-
SUM(CASE WHEN resolved_at IS NOT NULL THEN 1 ELSE 0 END) as resolved,
|
|
6724
|
-
SUM(CASE WHEN resolved_at IS NULL THEN 1 ELSE 0 END) as unresolved
|
|
6725
|
-
FROM incidents
|
|
6726
|
-
`);
|
|
6727
|
-
const { resolved, unresolved } = resolvedResult.rows[0];
|
|
6728
|
-
return {
|
|
6729
|
-
total,
|
|
6730
|
-
bySeverity,
|
|
6731
|
-
byType,
|
|
6732
|
-
resolved: resolved ?? 0,
|
|
6733
|
-
unresolved: unresolved ?? 0,
|
|
6734
|
-
oldestTimestamp: oldest,
|
|
6735
|
-
newestTimestamp: newest
|
|
6736
|
-
};
|
|
6737
|
-
}
|
|
6738
|
-
/**
|
|
6739
|
-
* Clear all incidents (admin function)
|
|
6740
|
-
*/
|
|
6741
|
-
async clearAll() {
|
|
6742
|
-
await this.db.query("DELETE FROM incidents");
|
|
6743
|
-
this.log("warn", "All incidents cleared");
|
|
6744
|
-
}
|
|
6745
|
-
/**
|
|
6746
|
-
* Update AI analysis for an incident
|
|
6747
|
-
*/
|
|
6748
|
-
async updateAIAnalysis(id, analysis) {
|
|
6749
|
-
await this.db.query(
|
|
6750
|
-
`UPDATE incidents
|
|
6751
|
-
SET ai_analysis = ?, ai_analyzed_at = ?
|
|
6752
|
-
WHERE id = ?`,
|
|
6753
|
-
[JSON.stringify(analysis), Date.now(), id]
|
|
6754
|
-
);
|
|
6755
|
-
this.log("debug", "AI analysis updated", { id });
|
|
6756
|
-
}
|
|
6757
|
-
/**
|
|
6758
|
-
* Convert database row to Incident object
|
|
6759
|
-
* @private
|
|
6760
|
-
*/
|
|
6761
|
-
rowToIncident(row) {
|
|
6762
|
-
const affectedServices = row.affected_services ? JSON.parse(row.affected_services) : void 0;
|
|
6763
|
-
const metadata = row.metadata ? JSON.parse(row.metadata) : void 0;
|
|
6764
|
-
const aiAnalysis = row.ai_analysis ? JSON.parse(row.ai_analysis) : void 0;
|
|
6765
|
-
const relatedData = row.related_logs_sample || row.related_metrics || row.timeline ? {
|
|
6766
|
-
logs: row.related_logs_sample ? JSON.parse(row.related_logs_sample) : void 0,
|
|
6767
|
-
metrics: row.related_metrics ? JSON.parse(row.related_metrics) : void 0,
|
|
6768
|
-
timeline: row.timeline ? JSON.parse(row.timeline) : void 0
|
|
6769
|
-
} : void 0;
|
|
6770
|
-
return {
|
|
6771
|
-
id: row.id,
|
|
6772
|
-
type: row.type,
|
|
6773
|
-
severity: row.severity,
|
|
6774
|
-
title: row.title,
|
|
6775
|
-
details: row.details,
|
|
6776
|
-
timestamp: row.timestamp,
|
|
6777
|
-
resolvedAt: row.resolved_at ?? void 0,
|
|
6778
|
-
resolutionNotes: row.resolution_notes ?? void 0,
|
|
6779
|
-
affectedServices,
|
|
6780
|
-
metadata,
|
|
6781
|
-
relatedData,
|
|
6782
|
-
aiAnalysis,
|
|
6783
|
-
aiAnalyzedAt: row.ai_analyzed_at ?? void 0
|
|
6784
|
-
};
|
|
6785
|
-
}
|
|
6786
|
-
log(level, message, meta) {
|
|
6787
|
-
if (level === "debug" && !this.config.debug) {
|
|
6788
|
-
return;
|
|
6789
|
-
}
|
|
6790
|
-
if (this.logger[level]) {
|
|
6791
|
-
this.logger[level](`[IncidentStorage] ${message}`, meta);
|
|
6792
|
-
} else {
|
|
6793
|
-
console.log(`[IncidentStorage] [${level}] ${message}`, meta);
|
|
6794
|
-
}
|
|
6795
|
-
}
|
|
6796
|
-
};
|
|
6797
|
-
var DEFAULT_THRESHOLDS = {
|
|
6798
|
-
errorRateWarning: 5,
|
|
6799
|
-
errorRateCritical: 10,
|
|
6800
|
-
latencyP99Warning: 2e3,
|
|
6801
|
-
latencyP99Critical: 5e3,
|
|
6802
|
-
latencyP95Warning: 1e3,
|
|
6803
|
-
minRequestsForDetection: 10,
|
|
6804
|
-
pluginErrorRateWarning: 10,
|
|
6805
|
-
pluginErrorRateCritical: 25
|
|
6806
|
-
};
|
|
6807
|
-
var DEFAULT_CONFIG4 = {
|
|
6808
|
-
intervalMs: 3e4,
|
|
6809
|
-
// 30 seconds
|
|
6810
|
-
thresholds: DEFAULT_THRESHOLDS,
|
|
6811
|
-
cooldownMs: 5 * 60 * 1e3,
|
|
6812
|
-
// 5 minutes
|
|
6813
|
-
debug: false
|
|
6814
|
-
};
|
|
6815
|
-
var IncidentDetector = class {
|
|
6816
|
-
incidentStorage;
|
|
6817
|
-
config;
|
|
6818
|
-
logger;
|
|
6819
|
-
intervalHandle = null;
|
|
6820
|
-
recentIncidents = [];
|
|
6821
|
-
isRunning = false;
|
|
6822
|
-
// Metrics history for "before" comparison (last 10 snapshots)
|
|
6823
|
-
metricsHistory = [];
|
|
6824
|
-
constructor(incidentStorage, config = {}, logger = console) {
|
|
6825
|
-
this.incidentStorage = incidentStorage;
|
|
6826
|
-
this.config = {
|
|
6827
|
-
...DEFAULT_CONFIG4,
|
|
6828
|
-
...config,
|
|
6829
|
-
thresholds: { ...DEFAULT_THRESHOLDS, ...config.thresholds }
|
|
6830
|
-
};
|
|
6831
|
-
this.logger = logger;
|
|
6832
|
-
}
|
|
6833
|
-
/**
|
|
6834
|
-
* Start automatic detection
|
|
6835
|
-
*/
|
|
6836
|
-
start() {
|
|
6837
|
-
if (this.isRunning) {
|
|
6838
|
-
this.log("warn", "Detector already running");
|
|
6839
|
-
return;
|
|
6840
|
-
}
|
|
6841
|
-
this.isRunning = true;
|
|
6842
|
-
this.log("info", "Starting incident detector", {
|
|
6843
|
-
intervalMs: this.config.intervalMs,
|
|
6844
|
-
thresholds: this.config.thresholds
|
|
6845
|
-
});
|
|
6846
|
-
this.runDetection().catch((err) => {
|
|
6847
|
-
this.log("error", "Initial detection failed", { error: err.message });
|
|
6848
|
-
});
|
|
6849
|
-
this.intervalHandle = setInterval(() => {
|
|
6850
|
-
this.runDetection().catch((err) => {
|
|
6851
|
-
this.log("error", "Detection cycle failed", { error: err.message });
|
|
6852
|
-
});
|
|
6853
|
-
}, this.config.intervalMs);
|
|
6854
|
-
}
|
|
6855
|
-
/**
|
|
6856
|
-
* Stop automatic detection
|
|
6857
|
-
*/
|
|
6858
|
-
stop() {
|
|
6859
|
-
if (this.intervalHandle) {
|
|
6860
|
-
clearInterval(this.intervalHandle);
|
|
6861
|
-
this.intervalHandle = null;
|
|
6862
|
-
}
|
|
6863
|
-
this.isRunning = false;
|
|
6864
|
-
this.log("info", "Incident detector stopped");
|
|
6865
|
-
}
|
|
6866
|
-
/**
|
|
6867
|
-
* Run a single detection cycle
|
|
6868
|
-
*/
|
|
6869
|
-
async runDetection() {
|
|
6870
|
-
const metrics = metricsCollector.getMetrics();
|
|
6871
|
-
const now = Date.now();
|
|
6872
|
-
const totalErrors = (metrics.requests.clientErrors ?? 0) + (metrics.requests.serverErrors ?? 0);
|
|
6873
|
-
const errorRate = metrics.requests.total > 0 ? totalErrors / metrics.requests.total * 100 : 0;
|
|
6874
|
-
this.metricsHistory.push({
|
|
6875
|
-
timestamp: now,
|
|
6876
|
-
errorRate,
|
|
6877
|
-
avgLatency: metrics.latency.average ?? 0,
|
|
6878
|
-
totalRequests: metrics.requests.total
|
|
6879
|
-
});
|
|
6880
|
-
if (this.metricsHistory.length > 10) {
|
|
6881
|
-
this.metricsHistory.shift();
|
|
6882
|
-
}
|
|
6883
|
-
this.recentIncidents = this.recentIncidents.filter(
|
|
6884
|
-
(ri) => now - ri.timestamp < this.config.cooldownMs
|
|
6885
|
-
);
|
|
6886
|
-
if (metrics.requests.total < this.config.thresholds.minRequestsForDetection) {
|
|
6887
|
-
this.log("debug", "Not enough requests for detection", {
|
|
6888
|
-
total: metrics.requests.total,
|
|
6889
|
-
min: this.config.thresholds.minRequestsForDetection
|
|
6890
|
-
});
|
|
6891
|
-
return;
|
|
6892
|
-
}
|
|
6893
|
-
await this.detectErrorRate(metrics, now);
|
|
6894
|
-
await this.detectLatencyIssues(metrics, now);
|
|
6895
|
-
await this.detectPluginFailures(metrics, now);
|
|
6896
|
-
this.log("debug", "Detection cycle complete", {
|
|
6897
|
-
recentIncidentsCount: this.recentIncidents.length
|
|
6898
|
-
});
|
|
6899
|
-
}
|
|
6900
|
-
/**
|
|
6901
|
-
* Detect high error rate
|
|
6902
|
-
*/
|
|
6903
|
-
async detectErrorRate(metrics, now) {
|
|
6904
|
-
const totalErrors = (metrics.requests.clientErrors ?? 0) + (metrics.requests.serverErrors ?? 0);
|
|
6905
|
-
const errorRate = metrics.requests.total > 0 ? totalErrors / metrics.requests.total * 100 : 0;
|
|
6906
|
-
const key = "error_rate";
|
|
6907
|
-
if (this.isInCooldown(key)) {
|
|
6908
|
-
return;
|
|
6909
|
-
}
|
|
6910
|
-
let severity = null;
|
|
6911
|
-
let title = "";
|
|
6912
|
-
let details = "";
|
|
6913
|
-
if (errorRate >= this.config.thresholds.errorRateCritical) {
|
|
6914
|
-
severity = "critical";
|
|
6915
|
-
title = `Critical Error Rate: ${errorRate.toFixed(1)}%`;
|
|
6916
|
-
details = `Error rate has exceeded critical threshold of ${this.config.thresholds.errorRateCritical}%. Current: ${errorRate.toFixed(2)}% (${totalErrors}/${metrics.requests.total} requests). Client errors: ${metrics.requests.clientErrors ?? 0}, Server errors: ${metrics.requests.serverErrors ?? 0}.`;
|
|
6917
|
-
} else if (errorRate >= this.config.thresholds.errorRateWarning) {
|
|
6918
|
-
severity = "warning";
|
|
6919
|
-
title = `High Error Rate: ${errorRate.toFixed(1)}%`;
|
|
6920
|
-
details = `Error rate has exceeded warning threshold of ${this.config.thresholds.errorRateWarning}%. Current: ${errorRate.toFixed(2)}% (${totalErrors}/${metrics.requests.total} requests).`;
|
|
6921
|
-
}
|
|
6922
|
-
if (severity) {
|
|
6923
|
-
await this.createIncident({
|
|
6924
|
-
type: "error_rate",
|
|
6925
|
-
severity,
|
|
6926
|
-
title,
|
|
6927
|
-
details,
|
|
6928
|
-
metadata: {
|
|
6929
|
-
errorRate,
|
|
6930
|
-
totalRequests: metrics.requests.total,
|
|
6931
|
-
clientErrors: metrics.requests.clientErrors,
|
|
6932
|
-
serverErrors: metrics.requests.serverErrors,
|
|
6933
|
-
threshold: severity === "critical" ? this.config.thresholds.errorRateCritical : this.config.thresholds.errorRateWarning
|
|
6934
|
-
}
|
|
6935
|
-
}, key, now);
|
|
6936
|
-
}
|
|
6937
|
-
}
|
|
6938
|
-
/**
|
|
6939
|
-
* Detect latency spikes
|
|
6940
|
-
*/
|
|
6941
|
-
async detectLatencyIssues(metrics, now) {
|
|
6942
|
-
const avgLatency = metrics.latency.average ?? 0;
|
|
6943
|
-
let p99 = avgLatency * 3;
|
|
6944
|
-
let p95 = avgLatency * 2;
|
|
6945
|
-
if (metrics.latency.histogram && metrics.latency.histogram.length > 0) {
|
|
6946
|
-
const sorted = [...metrics.latency.histogram].sort((a, b) => b.max - a.max);
|
|
6947
|
-
if (sorted.length > 0) {
|
|
6948
|
-
p99 = sorted[0]?.max ?? p99;
|
|
6949
|
-
p95 = sorted[Math.floor(sorted.length * 0.05)]?.max ?? p95;
|
|
6950
|
-
}
|
|
6951
|
-
}
|
|
6952
|
-
const keyP99 = "latency_p99";
|
|
6953
|
-
const keyP95 = "latency_p95";
|
|
6954
|
-
if (!this.isInCooldown(keyP99)) {
|
|
6955
|
-
let severity = null;
|
|
6956
|
-
let title = "";
|
|
6957
|
-
let details = "";
|
|
6958
|
-
if (p99 >= this.config.thresholds.latencyP99Critical) {
|
|
6959
|
-
severity = "critical";
|
|
6960
|
-
title = `Critical P99 Latency: ${p99.toFixed(0)}ms`;
|
|
6961
|
-
details = `P99 latency has exceeded critical threshold of ${this.config.thresholds.latencyP99Critical}ms. Current P99: ${p99.toFixed(0)}ms, Average: ${avgLatency.toFixed(0)}ms. This indicates severe performance degradation affecting tail latencies.`;
|
|
6962
|
-
} else if (p99 >= this.config.thresholds.latencyP99Warning) {
|
|
6963
|
-
severity = "warning";
|
|
6964
|
-
title = `High P99 Latency: ${p99.toFixed(0)}ms`;
|
|
6965
|
-
details = `P99 latency has exceeded warning threshold of ${this.config.thresholds.latencyP99Warning}ms. Current P99: ${p99.toFixed(0)}ms, Average: ${avgLatency.toFixed(0)}ms.`;
|
|
6966
|
-
}
|
|
6967
|
-
if (severity) {
|
|
6968
|
-
await this.createIncident({
|
|
6969
|
-
type: "latency_spike",
|
|
6970
|
-
severity,
|
|
6971
|
-
title,
|
|
6972
|
-
details,
|
|
6973
|
-
metadata: {
|
|
6974
|
-
p99,
|
|
6975
|
-
p95,
|
|
6976
|
-
average: avgLatency,
|
|
6977
|
-
threshold: severity === "critical" ? this.config.thresholds.latencyP99Critical : this.config.thresholds.latencyP99Warning
|
|
6978
|
-
}
|
|
6979
|
-
}, keyP99, now);
|
|
6980
|
-
}
|
|
6981
|
-
}
|
|
6982
|
-
if (!this.isInCooldown(keyP95) && p95 >= this.config.thresholds.latencyP95Warning) {
|
|
6983
|
-
await this.createIncident({
|
|
6984
|
-
type: "latency_spike",
|
|
6985
|
-
severity: "warning",
|
|
6986
|
-
title: `Elevated P95 Latency: ${p95.toFixed(0)}ms`,
|
|
6987
|
-
details: `P95 latency has exceeded threshold of ${this.config.thresholds.latencyP95Warning}ms. Current P95: ${p95.toFixed(0)}ms. Consider investigating slow endpoints.`,
|
|
6988
|
-
metadata: {
|
|
6989
|
-
p95,
|
|
6990
|
-
average: avgLatency,
|
|
6991
|
-
threshold: this.config.thresholds.latencyP95Warning
|
|
6992
|
-
}
|
|
6993
|
-
}, keyP95, now);
|
|
6994
|
-
}
|
|
6995
|
-
}
|
|
6996
|
-
/**
|
|
6997
|
-
* Detect plugin failures
|
|
6998
|
-
*/
|
|
6999
|
-
async detectPluginFailures(metrics, now) {
|
|
7000
|
-
if (!metrics.perPlugin || !Array.isArray(metrics.perPlugin)) {
|
|
7001
|
-
return;
|
|
7002
|
-
}
|
|
7003
|
-
for (const plugin of metrics.perPlugin) {
|
|
7004
|
-
if (!plugin.pluginId || plugin.requests < 5) {
|
|
7005
|
-
continue;
|
|
7006
|
-
}
|
|
7007
|
-
const errorRate = plugin.requests > 0 ? (plugin.errors ?? 0) / plugin.requests * 100 : 0;
|
|
7008
|
-
const key = `plugin:${plugin.pluginId}`;
|
|
7009
|
-
if (this.isInCooldown(key)) {
|
|
7010
|
-
continue;
|
|
7011
|
-
}
|
|
7012
|
-
let severity = null;
|
|
7013
|
-
let title = "";
|
|
7014
|
-
let details = "";
|
|
7015
|
-
if (errorRate >= this.config.thresholds.pluginErrorRateCritical) {
|
|
7016
|
-
severity = "critical";
|
|
7017
|
-
title = `Plugin Failure: ${plugin.pluginId}`;
|
|
7018
|
-
details = `Plugin ${plugin.pluginId} has critical error rate of ${errorRate.toFixed(1)}% (${plugin.errors ?? 0}/${plugin.requests} requests). This exceeds the critical threshold of ${this.config.thresholds.pluginErrorRateCritical}%.`;
|
|
7019
|
-
} else if (errorRate >= this.config.thresholds.pluginErrorRateWarning) {
|
|
7020
|
-
severity = "warning";
|
|
7021
|
-
title = `Plugin Issues: ${plugin.pluginId}`;
|
|
7022
|
-
details = `Plugin ${plugin.pluginId} has elevated error rate of ${errorRate.toFixed(1)}% (${plugin.errors ?? 0}/${plugin.requests} requests).`;
|
|
7023
|
-
}
|
|
7024
|
-
if (severity) {
|
|
7025
|
-
await this.createIncident({
|
|
7026
|
-
type: "plugin_failure",
|
|
7027
|
-
severity,
|
|
7028
|
-
title,
|
|
7029
|
-
details,
|
|
7030
|
-
affectedServices: [plugin.pluginId],
|
|
7031
|
-
metadata: {
|
|
7032
|
-
pluginId: plugin.pluginId,
|
|
7033
|
-
errorRate,
|
|
7034
|
-
requests: plugin.requests,
|
|
7035
|
-
errors: plugin.errors,
|
|
7036
|
-
avgLatency: plugin.latency?.average
|
|
7037
|
-
}
|
|
7038
|
-
}, key, now);
|
|
7039
|
-
}
|
|
7040
|
-
}
|
|
7041
|
-
}
|
|
7042
|
-
/**
|
|
7043
|
-
* Check if incident key is in cooldown
|
|
7044
|
-
*/
|
|
7045
|
-
isInCooldown(key) {
|
|
7046
|
-
return this.recentIncidents.some((ri) => ri.key === key);
|
|
7047
|
-
}
|
|
7048
|
-
/**
|
|
7049
|
-
* Gather related data (logs, metrics, timeline) for incident
|
|
7050
|
-
* @private
|
|
7051
|
-
*/
|
|
7052
|
-
async gatherRelatedData(incidentType, timeWindow = 5 * 60 * 1e3) {
|
|
7053
|
-
const now = Date.now();
|
|
7054
|
-
const from = now - timeWindow;
|
|
7055
|
-
const relatedData = {
|
|
7056
|
-
timeline: []
|
|
7057
|
-
};
|
|
7058
|
-
try {
|
|
7059
|
-
const [errorLogsResult, fatalLogsResult] = await Promise.all([
|
|
7060
|
-
platform.logs.query({ level: "error", from, to: now }, { limit: 50 }),
|
|
7061
|
-
platform.logs.query({ level: "fatal", from, to: now }, { limit: 50 })
|
|
7062
|
-
]);
|
|
7063
|
-
const allErrorLogs = [...errorLogsResult.logs, ...fatalLogsResult.logs];
|
|
7064
|
-
if (allErrorLogs.length > 0) {
|
|
7065
|
-
allErrorLogs.sort((a, b) => b.timestamp - a.timestamp);
|
|
7066
|
-
const uniqueErrors = /* @__PURE__ */ new Set();
|
|
7067
|
-
const endpointErrorCount = /* @__PURE__ */ new Map();
|
|
7068
|
-
for (const log of allErrorLogs) {
|
|
7069
|
-
let errorMsg = typeof log.message === "string" ? log.message : JSON.stringify(log.message);
|
|
7070
|
-
if (log.err) {
|
|
7071
|
-
const err = log.err;
|
|
7072
|
-
const stack = err.stack ? `
|
|
7073
|
-
${err.stack.split("\n").slice(0, 3).join("\n")}` : "";
|
|
7074
|
-
errorMsg = `${err.message || errorMsg}${stack}`;
|
|
7075
|
-
if (log.plugin) {
|
|
7076
|
-
errorMsg = `[${log.plugin}] ${errorMsg}`;
|
|
7077
|
-
}
|
|
7078
|
-
if (log.command) {
|
|
7079
|
-
errorMsg += `
|
|
7080
|
-
Command: ${log.command}`;
|
|
7081
|
-
}
|
|
7082
|
-
}
|
|
7083
|
-
if (errorMsg && uniqueErrors.size < 5) {
|
|
7084
|
-
uniqueErrors.add(errorMsg.substring(0, 500));
|
|
7085
|
-
}
|
|
7086
|
-
const endpoint = log.endpoint || log.url || "unknown";
|
|
7087
|
-
const existing = endpointErrorCount.get(endpoint);
|
|
7088
|
-
if (existing) {
|
|
7089
|
-
existing.count++;
|
|
7090
|
-
} else {
|
|
7091
|
-
endpointErrorCount.set(endpoint, {
|
|
7092
|
-
count: 1,
|
|
7093
|
-
sample: errorMsg.substring(0, 200)
|
|
7094
|
-
});
|
|
7095
|
-
}
|
|
7096
|
-
}
|
|
7097
|
-
const topEndpoints = Array.from(endpointErrorCount.entries()).sort((a, b) => b[1].count - a[1].count).slice(0, 5).map(([endpoint, data]) => ({
|
|
7098
|
-
endpoint,
|
|
7099
|
-
count: data.count,
|
|
7100
|
-
sample: data.sample
|
|
7101
|
-
}));
|
|
7102
|
-
relatedData.logs = {
|
|
7103
|
-
errorCount: allErrorLogs.filter((l) => l.level === "error").length,
|
|
7104
|
-
warnCount: 0,
|
|
7105
|
-
// Could query warnings separately if needed
|
|
7106
|
-
timeRange: [from, now],
|
|
7107
|
-
sampleErrors: Array.from(uniqueErrors),
|
|
7108
|
-
topEndpoints: topEndpoints.length > 0 ? topEndpoints : void 0
|
|
7109
|
-
};
|
|
7110
|
-
for (const log of allErrorLogs.slice(0, 10)) {
|
|
7111
|
-
const msg = typeof log.message === "string" ? log.message : "Error occurred";
|
|
7112
|
-
const plugin = log.plugin ? `[${log.plugin}] ` : "";
|
|
7113
|
-
relatedData.timeline.push({
|
|
7114
|
-
timestamp: log.timestamp,
|
|
7115
|
-
event: `${plugin}${msg.substring(0, 100)}`,
|
|
7116
|
-
source: "logs"
|
|
7117
|
-
});
|
|
7118
|
-
}
|
|
7119
|
-
}
|
|
7120
|
-
} catch (error) {
|
|
7121
|
-
this.log("warn", "Failed to gather related logs", {
|
|
7122
|
-
error: error instanceof Error ? error.message : String(error)
|
|
7123
|
-
});
|
|
7124
|
-
}
|
|
7125
|
-
const currentMetrics = metricsCollector.getMetrics();
|
|
7126
|
-
const duringMetrics = {
|
|
7127
|
-
errorRate: currentMetrics.requests.total > 0 ? (currentMetrics.requests.clientErrors + currentMetrics.requests.serverErrors) / currentMetrics.requests.total * 100 : 0,
|
|
7128
|
-
avgLatency: currentMetrics.latency.average ?? 0,
|
|
7129
|
-
totalRequests: currentMetrics.requests.total,
|
|
7130
|
-
totalErrors: currentMetrics.requests.clientErrors + currentMetrics.requests.serverErrors
|
|
7131
|
-
};
|
|
7132
|
-
let beforeMetrics;
|
|
7133
|
-
if (this.metricsHistory.length >= 2) {
|
|
7134
|
-
const beforeSnapshots = this.metricsHistory.slice(0, -1);
|
|
7135
|
-
if (beforeSnapshots.length > 0) {
|
|
7136
|
-
const avgErrorRate = beforeSnapshots.reduce((sum, s) => sum + s.errorRate, 0) / beforeSnapshots.length;
|
|
7137
|
-
const avgLatency = beforeSnapshots.reduce((sum, s) => sum + s.avgLatency, 0) / beforeSnapshots.length;
|
|
7138
|
-
const avgRequests = beforeSnapshots.reduce((sum, s) => sum + s.totalRequests, 0) / beforeSnapshots.length;
|
|
7139
|
-
beforeMetrics = {
|
|
7140
|
-
errorRate: avgErrorRate,
|
|
7141
|
-
avgLatency,
|
|
7142
|
-
totalRequests: avgRequests
|
|
7143
|
-
};
|
|
7144
|
-
this.log("debug", "Calculated before metrics", {
|
|
7145
|
-
before: beforeMetrics,
|
|
7146
|
-
during: duringMetrics,
|
|
7147
|
-
snapshotsUsed: beforeSnapshots.length
|
|
7148
|
-
});
|
|
7149
|
-
}
|
|
7150
|
-
}
|
|
7151
|
-
let topSlowest;
|
|
7152
|
-
let affectedEndpoints;
|
|
7153
|
-
if (incidentType === "latency_spike") {
|
|
7154
|
-
const histogram = currentMetrics.latency.histogram;
|
|
7155
|
-
if (histogram && histogram.length > 0) {
|
|
7156
|
-
const slowestRoutes = histogram.filter((h) => h.max > 100).sort((a, b) => b.max - a.max).slice(0, 10);
|
|
7157
|
-
topSlowest = slowestRoutes.map((h) => {
|
|
7158
|
-
const [method, ...pathParts] = h.route.split(" ");
|
|
7159
|
-
const endpoint = pathParts.join(" ");
|
|
7160
|
-
const statusCodes = Object.keys(h.byStatus);
|
|
7161
|
-
const mostCommonStatus = statusCodes.length > 0 ? parseInt(statusCodes[0], 10) : void 0;
|
|
7162
|
-
return {
|
|
7163
|
-
endpoint,
|
|
7164
|
-
method: method || "GET",
|
|
7165
|
-
durationMs: Math.round(h.max),
|
|
7166
|
-
statusCode: mostCommonStatus
|
|
7167
|
-
};
|
|
7168
|
-
});
|
|
7169
|
-
affectedEndpoints = [...new Set(slowestRoutes.map((h) => h.route))];
|
|
7170
|
-
this.log("debug", "Collected slow requests for latency incident", {
|
|
7171
|
-
topSlowestCount: topSlowest.length,
|
|
7172
|
-
affectedEndpointsCount: affectedEndpoints.length
|
|
7173
|
-
});
|
|
7174
|
-
}
|
|
7175
|
-
}
|
|
7176
|
-
relatedData.metrics = {
|
|
7177
|
-
before: beforeMetrics,
|
|
7178
|
-
during: duringMetrics,
|
|
7179
|
-
topSlowest,
|
|
7180
|
-
affectedEndpoints
|
|
7181
|
-
};
|
|
7182
|
-
relatedData.timeline.unshift({
|
|
7183
|
-
timestamp: now,
|
|
7184
|
-
event: `Incident detected: ${incidentType}`,
|
|
7185
|
-
source: "detector"
|
|
7186
|
-
});
|
|
7187
|
-
relatedData.timeline.sort((a, b) => b.timestamp - a.timestamp);
|
|
7188
|
-
return relatedData;
|
|
7189
|
-
}
|
|
7190
|
-
/**
|
|
7191
|
-
* Create incident and track it
|
|
7192
|
-
*/
|
|
7193
|
-
async createIncident(payload, key, timestamp) {
|
|
7194
|
-
try {
|
|
7195
|
-
const relatedData = await this.gatherRelatedData(payload.type);
|
|
7196
|
-
const enrichedPayload = {
|
|
7197
|
-
...payload,
|
|
7198
|
-
relatedData
|
|
7199
|
-
};
|
|
7200
|
-
const incident = await this.incidentStorage.createIncident(enrichedPayload);
|
|
7201
|
-
this.recentIncidents.push({
|
|
7202
|
-
type: payload.type,
|
|
7203
|
-
key,
|
|
7204
|
-
timestamp
|
|
7205
|
-
});
|
|
7206
|
-
this.log("info", "Auto-created incident with context", {
|
|
7207
|
-
id: incident.id,
|
|
7208
|
-
type: incident.type,
|
|
7209
|
-
severity: incident.severity,
|
|
7210
|
-
title: incident.title,
|
|
7211
|
-
errorLogsCount: relatedData.logs?.errorCount ?? 0,
|
|
7212
|
-
timelineEventsCount: relatedData.timeline?.length ?? 0
|
|
7213
|
-
});
|
|
7214
|
-
if (platform.analytics) {
|
|
7215
|
-
platform.analytics.track("incident.created", {
|
|
7216
|
-
incidentId: incident.id,
|
|
7217
|
-
type: incident.type,
|
|
7218
|
-
severity: incident.severity,
|
|
7219
|
-
source: "auto-detector",
|
|
7220
|
-
errorLogsCount: relatedData.logs?.errorCount ?? 0,
|
|
7221
|
-
timelineEventsCount: relatedData.timeline?.length ?? 0,
|
|
7222
|
-
hasBeforeMetrics: !!relatedData.metrics?.before,
|
|
7223
|
-
affectedServicesCount: payload.affectedServices?.length ?? 0
|
|
7224
|
-
}).catch(() => {
|
|
7225
|
-
});
|
|
7226
|
-
}
|
|
7227
|
-
} catch (error) {
|
|
7228
|
-
this.log("error", "Failed to create incident", {
|
|
7229
|
-
key,
|
|
7230
|
-
error: error instanceof Error ? error.message : String(error)
|
|
7231
|
-
});
|
|
7232
|
-
}
|
|
7233
|
-
}
|
|
7234
|
-
/**
|
|
7235
|
-
* Get current detector status
|
|
7236
|
-
*/
|
|
7237
|
-
getStatus() {
|
|
7238
|
-
return {
|
|
7239
|
-
running: this.isRunning,
|
|
7240
|
-
config: this.config,
|
|
7241
|
-
recentIncidentsCount: this.recentIncidents.length
|
|
7242
|
-
};
|
|
7243
|
-
}
|
|
7244
|
-
/**
|
|
7245
|
-
* Update thresholds at runtime
|
|
7246
|
-
*/
|
|
7247
|
-
updateThresholds(thresholds) {
|
|
7248
|
-
this.config.thresholds = { ...this.config.thresholds, ...thresholds };
|
|
7249
|
-
this.log("info", "Thresholds updated", { thresholds: this.config.thresholds });
|
|
7250
|
-
}
|
|
7251
|
-
log(level, message, meta) {
|
|
7252
|
-
if (level === "debug" && !this.config.debug) {
|
|
7253
|
-
return;
|
|
7254
|
-
}
|
|
7255
|
-
const prefix = "[IncidentDetector]";
|
|
7256
|
-
if (this.logger[level]) {
|
|
7257
|
-
if (meta) {
|
|
7258
|
-
this.logger[level]({ ...meta }, `${prefix} ${message}`);
|
|
7259
|
-
} else {
|
|
7260
|
-
this.logger[level](`${prefix} ${message}`);
|
|
7261
|
-
}
|
|
7262
|
-
} else {
|
|
7263
|
-
console.log(`${prefix} [${level}] ${message}`, meta ?? "");
|
|
7264
|
-
}
|
|
7265
|
-
}
|
|
7266
|
-
};
|
|
7267
5643
|
|
|
7268
5644
|
// src/routes/index.ts
|
|
7269
5645
|
function normalizeBasePath2(basePath) {
|
|
@@ -7459,71 +5835,7 @@ async function registerRoutes(server, config, repoRoot, registry) {
|
|
|
7459
5835
|
historicalCollector.stop();
|
|
7460
5836
|
platform.logger.info("Historical metrics collector stopped");
|
|
7461
5837
|
});
|
|
7462
|
-
|
|
7463
|
-
const db = platform.getAdapter("db");
|
|
7464
|
-
if (db) {
|
|
7465
|
-
try {
|
|
7466
|
-
const { readFileSync: readFileSync2 } = await import('fs');
|
|
7467
|
-
const { fileURLToPath } = await import('url');
|
|
7468
|
-
const { dirname: dirname2, join: join3 } = await import('path');
|
|
7469
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
7470
|
-
const __dirname = dirname2(__filename);
|
|
7471
|
-
const schemaPath = join3(__dirname, "..", "services", "incident-schema.sql");
|
|
7472
|
-
const schema = readFileSync2(schemaPath, "utf-8");
|
|
7473
|
-
if ("exec" in db && typeof db.exec === "function") {
|
|
7474
|
-
await db.exec(schema);
|
|
7475
|
-
} else {
|
|
7476
|
-
const statements = schema.split(";").map((s) => s.trim()).filter((s) => s.length > 0 && !s.startsWith("--"));
|
|
7477
|
-
for (const statement of statements) {
|
|
7478
|
-
if (statement) {
|
|
7479
|
-
await db.query(statement);
|
|
7480
|
-
}
|
|
7481
|
-
}
|
|
7482
|
-
}
|
|
7483
|
-
platform.logger.info("Incidents schema initialized");
|
|
7484
|
-
incidentStorage = new IncidentStorage(
|
|
7485
|
-
db,
|
|
7486
|
-
{ debug: process.env.NODE_ENV !== "production" },
|
|
7487
|
-
platformServices.logger
|
|
7488
|
-
);
|
|
7489
|
-
platform.logger.info("Incident storage initialized");
|
|
7490
|
-
} catch (err) {
|
|
7491
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
7492
|
-
platform.logger.warn("Failed to initialize incident storage, continuing without it", { error: error.message });
|
|
7493
|
-
}
|
|
7494
|
-
} else {
|
|
7495
|
-
platform.logger.warn("Database adapter (db) not configured \u2014 incident storage disabled. Configure db adapter in kb.config.json to enable.");
|
|
7496
|
-
}
|
|
7497
|
-
if (incidentStorage) {
|
|
7498
|
-
const incidentDetector = new IncidentDetector(
|
|
7499
|
-
incidentStorage,
|
|
7500
|
-
{
|
|
7501
|
-
intervalMs: 3e4,
|
|
7502
|
-
// Check every 30 seconds
|
|
7503
|
-
cooldownMs: 5 * 60 * 1e3,
|
|
7504
|
-
// 5 minute cooldown between same incidents
|
|
7505
|
-
debug: process.env.NODE_ENV !== "production",
|
|
7506
|
-
thresholds: {
|
|
7507
|
-
errorRateWarning: 5,
|
|
7508
|
-
errorRateCritical: 10,
|
|
7509
|
-
latencyP99Warning: 2e3,
|
|
7510
|
-
latencyP99Critical: 5e3,
|
|
7511
|
-
latencyP95Warning: 1e3,
|
|
7512
|
-
minRequestsForDetection: 10,
|
|
7513
|
-
pluginErrorRateWarning: 10,
|
|
7514
|
-
pluginErrorRateCritical: 25
|
|
7515
|
-
}
|
|
7516
|
-
},
|
|
7517
|
-
platformServices.logger
|
|
7518
|
-
);
|
|
7519
|
-
incidentDetector.start();
|
|
7520
|
-
platform.logger.info("Incident detector started");
|
|
7521
|
-
server.addHook("onClose", async () => {
|
|
7522
|
-
incidentDetector.stop();
|
|
7523
|
-
platform.logger.info("Incident detector stopped");
|
|
7524
|
-
});
|
|
7525
|
-
}
|
|
7526
|
-
await registerObservabilityRoutes(server, config, repoRoot, historicalCollector, incidentStorage, platformServices);
|
|
5838
|
+
await registerObservabilityRoutes(server, config, repoRoot, historicalCollector, platformServices);
|
|
7527
5839
|
await registerAnalyticsRoutes(server, config);
|
|
7528
5840
|
await registerAdaptersRoutes(server, config);
|
|
7529
5841
|
await registerLogRoutes(server);
|
|
@@ -7614,6 +5926,9 @@ function registerEnvelopeMiddleware(server, config) {
|
|
|
7614
5926
|
if (parsedPayload && typeof parsedPayload === "object" && "ok" in parsedPayload) {
|
|
7615
5927
|
return payload;
|
|
7616
5928
|
}
|
|
5929
|
+
if (reply.getHeader("x-openapi-spec")) {
|
|
5930
|
+
return payload;
|
|
5931
|
+
}
|
|
7617
5932
|
let dataToWrap = parsedPayload;
|
|
7618
5933
|
if (parsedPayload === null || parsedPayload === void 0) {
|
|
7619
5934
|
dataToWrap = null;
|