@oneuptime/common 11.3.28 → 11.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/DatabaseModels/AIRun.ts +50 -0
- package/Models/DatabaseModels/Project.ts +60 -0
- package/Server/API/AIInvestigationAPI.ts +148 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1783598145111-AddEnableAutomaticIncidentInvestigationToProject.ts +19 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1783615884106-AddInvestigationSubjectToAIRun.ts +33 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1783632447975-AddEnableAutomaticAlertInvestigationToProject.ts +19 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +6 -0
- package/Server/Services/AlertService.ts +25 -0
- package/Server/Services/IncidentService.ts +25 -0
- package/Server/Services/IncidentStateTimelineService.ts +19 -0
- package/Server/Utils/AI/Chat/ObservabilityAssistant.ts +119 -11
- package/Server/Utils/AI/Sentinel/AlertInvestigationRunner.ts +141 -0
- package/Server/Utils/AI/Sentinel/IncidentInvestigationRunner.ts +178 -0
- package/Server/Utils/AI/Sentinel/IncidentPostmortemRunner.ts +105 -0
- package/Server/Utils/AI/Sentinel/SentinelInvestigationEngine.ts +405 -0
- package/Server/Utils/AI/Sentinel/SentinelMemory.ts +167 -0
- package/Server/Utils/AI/Toolbox/Index.ts +11 -0
- package/Server/Utils/AI/Toolbox/SentinelActionTools.ts +577 -0
- package/Tests/Server/API/BaseAPI.test.ts +1 -0
- package/Tests/Types/OnCallDutyPolicy/ScheduleShiftUtil.test.ts +260 -0
- package/Types/Date.ts +46 -0
- package/Types/OnCallDutyPolicy/ScheduleShiftUtil.ts +257 -0
- package/UI/Components/Calendar/Calendar.tsx +8 -2
- package/UI/Components/Charts/ChartGroup/ChartGroup.tsx +5 -5
- package/build/dist/Models/DatabaseModels/AIRun.js +52 -0
- package/build/dist/Models/DatabaseModels/AIRun.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Project.js +62 -0
- package/build/dist/Models/DatabaseModels/Project.js.map +1 -1
- package/build/dist/Server/API/AIInvestigationAPI.js +107 -0
- package/build/dist/Server/API/AIInvestigationAPI.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783598145111-AddEnableAutomaticIncidentInvestigationToProject.js +12 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783598145111-AddEnableAutomaticIncidentInvestigationToProject.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783615884106-AddInvestigationSubjectToAIRun.js +18 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783615884106-AddInvestigationSubjectToAIRun.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783632447975-AddEnableAutomaticAlertInvestigationToProject.js +12 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783632447975-AddEnableAutomaticAlertInvestigationToProject.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +6 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/AlertService.js +24 -0
- package/build/dist/Server/Services/AlertService.js.map +1 -1
- package/build/dist/Server/Services/IncidentService.js +24 -0
- package/build/dist/Server/Services/IncidentService.js.map +1 -1
- package/build/dist/Server/Services/IncidentStateTimelineService.js +19 -0
- package/build/dist/Server/Services/IncidentStateTimelineService.js.map +1 -1
- package/build/dist/Server/Utils/AI/Chat/ObservabilityAssistant.js +59 -11
- package/build/dist/Server/Utils/AI/Chat/ObservabilityAssistant.js.map +1 -1
- package/build/dist/Server/Utils/AI/Sentinel/AlertInvestigationRunner.js +119 -0
- package/build/dist/Server/Utils/AI/Sentinel/AlertInvestigationRunner.js.map +1 -0
- package/build/dist/Server/Utils/AI/Sentinel/IncidentInvestigationRunner.js +147 -0
- package/build/dist/Server/Utils/AI/Sentinel/IncidentInvestigationRunner.js.map +1 -0
- package/build/dist/Server/Utils/AI/Sentinel/IncidentPostmortemRunner.js +93 -0
- package/build/dist/Server/Utils/AI/Sentinel/IncidentPostmortemRunner.js.map +1 -0
- package/build/dist/Server/Utils/AI/Sentinel/SentinelInvestigationEngine.js +311 -0
- package/build/dist/Server/Utils/AI/Sentinel/SentinelInvestigationEngine.js.map +1 -0
- package/build/dist/Server/Utils/AI/Sentinel/SentinelMemory.js +143 -0
- package/build/dist/Server/Utils/AI/Sentinel/SentinelMemory.js.map +1 -0
- package/build/dist/Server/Utils/AI/Toolbox/Index.js +6 -0
- package/build/dist/Server/Utils/AI/Toolbox/Index.js.map +1 -1
- package/build/dist/Server/Utils/AI/Toolbox/SentinelActionTools.js +451 -0
- package/build/dist/Server/Utils/AI/Toolbox/SentinelActionTools.js.map +1 -0
- package/build/dist/Types/Date.js +32 -0
- package/build/dist/Types/Date.js.map +1 -1
- package/build/dist/Types/OnCallDutyPolicy/ScheduleShiftUtil.js +159 -0
- package/build/dist/Types/OnCallDutyPolicy/ScheduleShiftUtil.js.map +1 -0
- package/build/dist/UI/Components/Calendar/Calendar.js +2 -2
- package/build/dist/UI/Components/Calendar/Calendar.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js +2 -2
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js.map +1 -1
- package/package.json +1 -1
|
@@ -23,19 +23,43 @@ const MAX_OUTPUT_TOKENS = 1500;
|
|
|
23
23
|
const TEMPERATURE = 0.2;
|
|
24
24
|
export default class ObservabilityAssistant {
|
|
25
25
|
static async answerQuestion(request) {
|
|
26
|
+
var _a, _b, _c, _d;
|
|
26
27
|
const startedAtMs = Date.now();
|
|
28
|
+
const maxLlmCalls = (_a = request.maxLlmCalls) !== null && _a !== void 0 ? _a : MAX_LLM_CALLS;
|
|
29
|
+
const maxToolCalls = (_b = request.maxToolCalls) !== null && _b !== void 0 ? _b : MAX_TOOL_CALLS;
|
|
30
|
+
const maxWallClockMs = (_c = request.maxWallClockMs) !== null && _c !== void 0 ? _c : MAX_WALL_CLOCK_MS;
|
|
31
|
+
const maxOutputTokens = (_d = request.maxOutputTokens) !== null && _d !== void 0 ? _d : MAX_OUTPUT_TOKENS;
|
|
32
|
+
// Best-effort live-narration emitter — never throws into the loop.
|
|
33
|
+
const emitStep = async (step) => {
|
|
34
|
+
if (!request.onStep) {
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
await request.onStep(step);
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
logger.error(`ObservabilityAssistant onStep failed: ${err}`);
|
|
42
|
+
}
|
|
43
|
+
};
|
|
27
44
|
const toolContext = {
|
|
28
45
|
projectId: request.projectId,
|
|
29
46
|
props: request.props,
|
|
30
47
|
};
|
|
48
|
+
let systemPromptContent = buildObservabilityChatSystemPrompt({
|
|
49
|
+
currentTime: OneUptimeDate.getCurrentDate(),
|
|
50
|
+
/*
|
|
51
|
+
* Slack/Teams and autonomous investigations have no approval UI, so this
|
|
52
|
+
* surface stays strictly read-only.
|
|
53
|
+
*/
|
|
54
|
+
permissionMode: AIChatPermissionMode.ReadOnly,
|
|
55
|
+
});
|
|
56
|
+
if (request.systemInstructions) {
|
|
57
|
+
systemPromptContent += `\n\n${request.systemInstructions}`;
|
|
58
|
+
}
|
|
31
59
|
const messages = [
|
|
32
60
|
{
|
|
33
61
|
role: "system",
|
|
34
|
-
content:
|
|
35
|
-
currentTime: OneUptimeDate.getCurrentDate(),
|
|
36
|
-
// Slack/Teams have no approval UI, so this surface stays read-only.
|
|
37
|
-
permissionMode: AIChatPermissionMode.ReadOnly,
|
|
38
|
-
}),
|
|
62
|
+
content: systemPromptContent,
|
|
39
63
|
},
|
|
40
64
|
];
|
|
41
65
|
for (const turn of (request.history || []).slice(-MAX_HISTORY_TURNS)) {
|
|
@@ -52,15 +76,16 @@ export default class ObservabilityAssistant {
|
|
|
52
76
|
let modelName = undefined;
|
|
53
77
|
let finalContent = "";
|
|
54
78
|
while (true) {
|
|
55
|
-
const budgetExhausted = llmCallCount >=
|
|
56
|
-
toolCallCount >=
|
|
57
|
-
Date.now() - startedAtMs >=
|
|
79
|
+
const budgetExhausted = llmCallCount >= maxLlmCalls - 1 ||
|
|
80
|
+
toolCallCount >= maxToolCalls ||
|
|
81
|
+
Date.now() - startedAtMs >= maxWallClockMs;
|
|
58
82
|
if (budgetExhausted) {
|
|
59
83
|
messages.push({
|
|
60
84
|
role: "user",
|
|
61
85
|
content: "Your query budget for this turn is exhausted. Answer now with the findings so far, clearly stating what you could and could not verify. Do not request more tools.",
|
|
62
86
|
});
|
|
63
87
|
}
|
|
88
|
+
await emitStep({ type: "llm_started" });
|
|
64
89
|
const response = await AIService.executeWithLogging({
|
|
65
90
|
projectId: request.projectId,
|
|
66
91
|
userId: request.userId,
|
|
@@ -70,13 +95,17 @@ export default class ObservabilityAssistant {
|
|
|
70
95
|
tools: budgetExhausted
|
|
71
96
|
? undefined
|
|
72
97
|
: AIToolbox.getLlmToolDefinitions(AIChatPermissionMode.ReadOnly),
|
|
73
|
-
maxTokens:
|
|
98
|
+
maxTokens: maxOutputTokens,
|
|
74
99
|
temperature: TEMPERATURE,
|
|
75
100
|
// Chat-ops content is per-user — do not persist previews to LlmLog.
|
|
76
101
|
storeContentPreviews: false,
|
|
77
102
|
});
|
|
78
103
|
llmCallCount++;
|
|
79
104
|
totalTokens += response.llmLog.totalTokens || 0;
|
|
105
|
+
await emitStep({
|
|
106
|
+
type: "llm_completed",
|
|
107
|
+
totalTokens: response.llmLog.totalTokens || 0,
|
|
108
|
+
});
|
|
80
109
|
if (!providerName) {
|
|
81
110
|
providerName = response.llmLog.llmProviderName;
|
|
82
111
|
modelName = response.llmLog.modelName;
|
|
@@ -90,8 +119,8 @@ export default class ObservabilityAssistant {
|
|
|
90
119
|
toolCalls: response.toolCalls,
|
|
91
120
|
});
|
|
92
121
|
for (const toolCall of response.toolCalls) {
|
|
93
|
-
const overBudget = toolCallCount >=
|
|
94
|
-
Date.now() - startedAtMs >=
|
|
122
|
+
const overBudget = toolCallCount >= maxToolCalls ||
|
|
123
|
+
Date.now() - startedAtMs >= maxWallClockMs;
|
|
95
124
|
if (overBudget) {
|
|
96
125
|
messages.push({
|
|
97
126
|
role: "tool",
|
|
@@ -109,12 +138,24 @@ export default class ObservabilityAssistant {
|
|
|
109
138
|
});
|
|
110
139
|
continue;
|
|
111
140
|
}
|
|
141
|
+
const toolStartedAtMs = Date.now();
|
|
142
|
+
await emitStep({
|
|
143
|
+
type: "tool_started",
|
|
144
|
+
toolName: toolCall.name,
|
|
145
|
+
toolArguments: toolCall.arguments,
|
|
146
|
+
});
|
|
112
147
|
const outcome = await AIToolbox.executeTool({
|
|
113
148
|
name: toolCall.name,
|
|
114
149
|
args: toolCall.arguments,
|
|
115
150
|
ctx: toolContext,
|
|
116
151
|
});
|
|
117
152
|
if (!outcome.success || !outcome.result) {
|
|
153
|
+
await emitStep({
|
|
154
|
+
type: "tool_failed",
|
|
155
|
+
toolName: toolCall.name,
|
|
156
|
+
durationMs: Date.now() - toolStartedAtMs,
|
|
157
|
+
errorMessage: outcome.errorMessage || outcome.textForLlm,
|
|
158
|
+
});
|
|
118
159
|
messages.push({
|
|
119
160
|
role: "tool",
|
|
120
161
|
toolCallId: toolCall.id,
|
|
@@ -131,6 +172,13 @@ export default class ObservabilityAssistant {
|
|
|
131
172
|
rowCount: outcome.result.rowCount,
|
|
132
173
|
target: outcome.result.citationTarget,
|
|
133
174
|
});
|
|
175
|
+
await emitStep({
|
|
176
|
+
type: "tool_completed",
|
|
177
|
+
toolName: toolCall.name,
|
|
178
|
+
durationMs: Date.now() - toolStartedAtMs,
|
|
179
|
+
rowCount: outcome.result.rowCount,
|
|
180
|
+
citationId: citationId,
|
|
181
|
+
});
|
|
134
182
|
const escapedText = escapeToolResultContent(outcome.textForLlm);
|
|
135
183
|
messages.push({
|
|
136
184
|
role: "tool",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ObservabilityAssistant.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Chat/ObservabilityAssistant.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"ObservabilityAssistant.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Chat/ObservabilityAssistant.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AAInD,OAAO,SAA4B,MAAM,6BAA6B,CAAC;AACvE,OAAO,MAAM,MAAM,cAAc,CAAC;AAElC,OAAO,SAA8B,MAAM,kBAAkB,CAAC;AAE9D,OAAO,oBAAoB,MAAM,2CAA2C,CAAC;AAC7E,OAAO,EAAE,kCAAkC,EAAE,MAAM,2BAA2B,CAAC;AAC/E,OAAO,EACL,uBAAuB,EACvB,8BAA8B,GAC/B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAyFtD,MAAM,aAAa,GAAW,CAAC,CAAC;AAChC,MAAM,cAAc,GAAW,CAAC,CAAC;AACjC,MAAM,iBAAiB,GAAW,EAAE,GAAG,IAAI,CAAC;AAC5C,MAAM,iBAAiB,GAAW,CAAC,CAAC;AACpC,MAAM,iBAAiB,GAAW,IAAI,CAAC;AACvC,MAAM,WAAW,GAAW,GAAG,CAAC;AAEhC,MAAM,CAAC,OAAO,OAAO,sBAAsB;IAErB,AAAb,MAAM,CAAC,KAAK,CAAC,cAAc,CAChC,OAAsC;;QAEtC,MAAM,WAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvC,MAAM,WAAW,GAAW,MAAA,OAAO,CAAC,WAAW,mCAAI,aAAa,CAAC;QACjE,MAAM,YAAY,GAAW,MAAA,OAAO,CAAC,YAAY,mCAAI,cAAc,CAAC;QACpE,MAAM,cAAc,GAAW,MAAA,OAAO,CAAC,cAAc,mCAAI,iBAAiB,CAAC;QAC3E,MAAM,eAAe,GACnB,MAAA,OAAO,CAAC,eAAe,mCAAI,iBAAiB,CAAC;QAE/C,mEAAmE;QACnE,MAAM,QAAQ,GAEO,KAAK,EACxB,IAAgC,EACjB,EAAE;YACjB,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,OAAO;YACT,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAgB;YAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC;QAEF,IAAI,mBAAmB,GAAW,kCAAkC,CAAC;YACnE,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;YAC3C;;;eAGG;YACH,cAAc,EAAE,oBAAoB,CAAC,QAAQ;SAC9C,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;YAC/B,mBAAmB,IAAI,OAAO,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC7D,CAAC;QAED,MAAM,QAAQ,GAAsB;YAClC;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,mBAAmB;aAC7B;SACF,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACrE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE3D,MAAM,SAAS,GAA0B,EAAE,CAAC;QAC5C,IAAI,YAAY,GAAW,CAAC,CAAC;QAC7B,IAAI,aAAa,GAAW,CAAC,CAAC;QAC9B,IAAI,WAAW,GAAW,CAAC,CAAC;QAC5B,IAAI,YAAY,GAAuB,SAAS,CAAC;QACjD,IAAI,SAAS,GAAuB,SAAS,CAAC;QAC9C,IAAI,YAAY,GAAW,EAAE,CAAC;QAE9B,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,eAAe,GACnB,YAAY,IAAI,WAAW,GAAG,CAAC;gBAC/B,aAAa,IAAI,YAAY;gBAC7B,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,IAAI,cAAc,CAAC;YAE7C,IAAI,eAAe,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,MAAM;oBACZ,OAAO,EACL,oKAAoK;iBACvK,CAAC,CAAC;YACL,CAAC;YAED,MAAM,QAAQ,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;YAExC,MAAM,QAAQ,GAAkB,MAAM,SAAS,CAAC,kBAAkB,CAAC;gBACjE,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,QAAQ,EAAE,QAAQ;gBAClB,KAAK,EAAE,eAAe;oBACpB,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,oBAAoB,CAAC,QAAQ,CAAC;gBAClE,SAAS,EAAE,eAAe;gBAC1B,WAAW,EAAE,WAAW;gBACxB,oEAAoE;gBACpE,oBAAoB,EAAE,KAAK;aAC5B,CAAC,CAAC;YAEH,YAAY,EAAE,CAAC;YACf,WAAW,IAAI,QAAQ,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;YAEhD,MAAM,QAAQ,CAAC;gBACb,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC;aAC9C,CAAC,CAAC;YAEH,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,eAAe,CAAC;gBAC/C,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC;YACxC,CAAC;YAED,IACE,CAAC,eAAe;gBAChB,QAAQ,CAAC,SAAS;gBAClB,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAC7B,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,SAAS,EAAE,QAAQ,CAAC,SAAS;iBAC9B,CAAC,CAAC;gBAEH,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;oBAC1C,MAAM,UAAU,GACd,aAAa,IAAI,YAAY;wBAC7B,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,IAAI,cAAc,CAAC;oBAE7C,IAAI,UAAU,EAAE,CAAC;wBACf,QAAQ,CAAC,IAAI,CAAC;4BACZ,IAAI,EAAE,MAAM;4BACZ,UAAU,EAAE,QAAQ,CAAC,EAAE;4BACvB,OAAO,EACL,8FAA8F;yBACjG,CAAC,CAAC;wBACH,SAAS;oBACX,CAAC;oBAED,aAAa,EAAE,CAAC;oBAEhB,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;wBACjC,QAAQ,CAAC,IAAI,CAAC;4BACZ,IAAI,EAAE,MAAM;4BACZ,UAAU,EAAE,QAAQ,CAAC,EAAE;4BACvB,OAAO,EAAE,iBAAiB,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,mBAAmB,sDAAsD;yBAC/H,CAAC,CAAC;wBACH,SAAS;oBACX,CAAC;oBAED,MAAM,eAAe,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE3C,MAAM,QAAQ,CAAC;wBACb,IAAI,EAAE,cAAc;wBACpB,QAAQ,EAAE,QAAQ,CAAC,IAAI;wBACvB,aAAa,EAAE,QAAQ,CAAC,SAAS;qBAClC,CAAC,CAAC;oBAEH,MAAM,OAAO,GAAoB,MAAM,SAAS,CAAC,WAAW,CAAC;wBAC3D,IAAI,EAAE,QAAQ,CAAC,IAAI;wBACnB,IAAI,EAAE,QAAQ,CAAC,SAAS;wBACxB,GAAG,EAAE,WAAW;qBACjB,CAAC,CAAC;oBAEH,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;wBACxC,MAAM,QAAQ,CAAC;4BACb,IAAI,EAAE,aAAa;4BACnB,QAAQ,EAAE,QAAQ,CAAC,IAAI;4BACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe;4BACxC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,UAAU;yBACzD,CAAC,CAAC;wBAEH,QAAQ,CAAC,IAAI,CAAC;4BACZ,IAAI,EAAE,MAAM;4BACZ,UAAU,EAAE,QAAQ,CAAC,EAAE;4BACvB,OAAO,EAAE,OAAO,CAAC,UAAU;yBAC5B,CAAC,CAAC;wBACH,SAAS;oBACX,CAAC;oBAED,MAAM,UAAU,GAAW,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAEtD,SAAS,CAAC,IAAI,CAAC;wBACb,EAAE,EAAE,UAAU;wBACd,QAAQ,EAAE,QAAQ,CAAC,IAAI;wBACvB,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,aAAa;wBACnC,cAAc,EAAE,QAAQ,CAAC,SAAS;wBAClC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;wBACjC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,cAAc;qBACtC,CAAC,CAAC;oBAEH,MAAM,QAAQ,CAAC;wBACb,IAAI,EAAE,gBAAgB;wBACtB,QAAQ,EAAE,QAAQ,CAAC,IAAI;wBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe;wBACxC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;wBACjC,UAAU,EAAE,UAAU;qBACvB,CAAC,CAAC;oBAEH,MAAM,WAAW,GAAW,uBAAuB,CACjD,OAAO,CAAC,UAAU,CACnB,CAAC;oBAEF,QAAQ,CAAC,IAAI,CAAC;wBACZ,IAAI,EAAE,MAAM;wBACZ,UAAU,EAAE,QAAQ,CAAC,EAAE;wBACvB,OAAO,EAAE,4DAA4D,UAAU,WAAW,OAAO,CAAC,MAAM,CAAC,QAAQ,OAAO,WAAW,qDAAqD,UAAU,+CAA+C;qBAClP,CAAC,CAAC;gBACL,CAAC;gBAED,SAAS;YACX,CAAC;YAED,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC;YAChC,MAAM;QACR,CAAC;QAED,YAAY,GAAG,8BAA8B,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEvE,MAAM,CAAC,KAAK,CACV,sCAAsC,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,OAAO,YAAY,eAAe,aAAa,UAAU,CACxH,CAAC;QAEF,OAAO;YACL,iBAAiB,EAAE,YAAY;YAC/B,SAAS,EAAE,SAAS;YACpB,WAAW,EAAE,WAAW;YACxB,YAAY,EAAE,YAAY;YAC1B,aAAa,EAAE,aAAa;YAC5B,YAAY,EAAE,YAAY;YAC1B,SAAS,EAAE,SAAS;SACrB,CAAC;IACJ,CAAC;CACF;AAxOqB;IADnB,WAAW,EAAE;;;;kDAwOb"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
|
+
};
|
|
10
|
+
import OneUptimeDate from "../../../../Types/Date";
|
|
11
|
+
import { Blue500 } from "../../../../Types/BrandColors";
|
|
12
|
+
import { AlertFeedEventType } from "../../../../Models/DatabaseModels/AlertFeed";
|
|
13
|
+
import AlertFeedService from "../../../Services/AlertFeedService";
|
|
14
|
+
import AlertAIContextBuilder from "../AlertAIContextBuilder";
|
|
15
|
+
import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
|
|
16
|
+
import logger from "../../Logger";
|
|
17
|
+
import CaptureSpan from "../../Telemetry/CaptureSpan";
|
|
18
|
+
/*
|
|
19
|
+
* Sentinel — alert investigation.
|
|
20
|
+
*
|
|
21
|
+
* Alerts are the earlier, "left of boom" signal. When a new alert is declared,
|
|
22
|
+
* Sentinel investigates it (read-only) and posts a cited root-cause analysis to
|
|
23
|
+
* the alert timeline + Slack/Teams. Symmetric to incident investigation and
|
|
24
|
+
* built on the same shared SentinelInvestigationEngine.
|
|
25
|
+
*
|
|
26
|
+
* Gated by the same per-project opt-in as incidents. Alert volume can be higher
|
|
27
|
+
* than incidents, so severity / rate gating is a deliberate follow-up.
|
|
28
|
+
*/
|
|
29
|
+
export default class SentinelAlertInvestigationRunner {
|
|
30
|
+
/*
|
|
31
|
+
* Entry point called (fire-and-forget) from AlertService.onCreateSuccess.
|
|
32
|
+
* Never throws — all failures are logged and recorded on the AIRun.
|
|
33
|
+
*/
|
|
34
|
+
static async investigateNewAlert(data) {
|
|
35
|
+
const { alertId, projectId } = data;
|
|
36
|
+
try {
|
|
37
|
+
if (!(await SentinelInvestigationEngine.isEnabledForProject(projectId, "Alert"))) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const contextData = await AlertAIContextBuilder.buildAlertContext({
|
|
41
|
+
alertId,
|
|
42
|
+
});
|
|
43
|
+
await SentinelInvestigationEngine.investigate({
|
|
44
|
+
projectId,
|
|
45
|
+
feature: "Sentinel Alert Investigation",
|
|
46
|
+
subjectAlertId: alertId,
|
|
47
|
+
contextSummary: this.buildAlertSummary(contextData),
|
|
48
|
+
postAnalysis: async (postData) => {
|
|
49
|
+
await AlertFeedService.createAlertFeedItem({
|
|
50
|
+
alertId,
|
|
51
|
+
projectId,
|
|
52
|
+
alertFeedEventType: AlertFeedEventType.RootCause,
|
|
53
|
+
displayColor: Blue500,
|
|
54
|
+
feedInfoInMarkdown: postData.analysisMarkdown,
|
|
55
|
+
/*
|
|
56
|
+
* Quiet mode: an inconclusive investigation posts to the timeline
|
|
57
|
+
* but does NOT loudly ping the workspace / on-call.
|
|
58
|
+
*/
|
|
59
|
+
workspaceNotification: {
|
|
60
|
+
sendWorkspaceNotification: postData.isConfident,
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
logger.error(`Sentinel: unexpected error investigating alert ${alertId.toString()}: ${error}`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Build a compact alert record to seed the investigation.
|
|
71
|
+
static buildAlertSummary(contextData) {
|
|
72
|
+
var _a, _b, _c;
|
|
73
|
+
const { alert, internalNotes } = contextData;
|
|
74
|
+
const lines = [];
|
|
75
|
+
lines.push("# Alert");
|
|
76
|
+
lines.push(`Title: ${alert.title || "N/A"}`);
|
|
77
|
+
if (alert.description) {
|
|
78
|
+
lines.push(`Description: ${alert.description}`);
|
|
79
|
+
}
|
|
80
|
+
lines.push(`Severity: ${((_a = alert.alertSeverity) === null || _a === void 0 ? void 0 : _a.name) || "N/A"}`);
|
|
81
|
+
lines.push(`Current state: ${((_b = alert.currentAlertState) === null || _b === void 0 ? void 0 : _b.name) || "N/A"}`);
|
|
82
|
+
lines.push(`Declared at: ${alert.createdAt
|
|
83
|
+
? OneUptimeDate.getDateAsFormattedString(alert.createdAt)
|
|
84
|
+
: "N/A"}`);
|
|
85
|
+
if ((_c = alert.monitor) === null || _c === void 0 ? void 0 : _c.name) {
|
|
86
|
+
lines.push(`Affected monitor: ${alert.monitor.name}`);
|
|
87
|
+
}
|
|
88
|
+
if (alert.labels && alert.labels.length > 0) {
|
|
89
|
+
lines.push(`Labels: ${alert.labels
|
|
90
|
+
.map((l) => {
|
|
91
|
+
return l.name;
|
|
92
|
+
})
|
|
93
|
+
.filter(Boolean)
|
|
94
|
+
.join(", ")}`);
|
|
95
|
+
}
|
|
96
|
+
if (alert.rootCause) {
|
|
97
|
+
lines.push(`Root cause (as recorded so far): ${alert.rootCause}`);
|
|
98
|
+
}
|
|
99
|
+
const recentNotes = (internalNotes || [])
|
|
100
|
+
.slice(-5)
|
|
101
|
+
.map((note) => {
|
|
102
|
+
return note.note ? `- ${note.note}` : "";
|
|
103
|
+
})
|
|
104
|
+
.filter(Boolean);
|
|
105
|
+
if (recentNotes.length > 0) {
|
|
106
|
+
lines.push("");
|
|
107
|
+
lines.push("Recent internal notes:");
|
|
108
|
+
lines.push(...recentNotes);
|
|
109
|
+
}
|
|
110
|
+
return lines.join("\n");
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
__decorate([
|
|
114
|
+
CaptureSpan(),
|
|
115
|
+
__metadata("design:type", Function),
|
|
116
|
+
__metadata("design:paramtypes", [Object]),
|
|
117
|
+
__metadata("design:returntype", Promise)
|
|
118
|
+
], SentinelAlertInvestigationRunner, "investigateNewAlert", null);
|
|
119
|
+
//# sourceMappingURL=AlertInvestigationRunner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AlertInvestigationRunner.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Sentinel/AlertInvestigationRunner.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,+BAA+B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,6CAA6C,CAAC;AACjF,OAAO,gBAAgB,MAAM,oCAAoC,CAAC;AAClE,OAAO,qBAEN,MAAM,0BAA0B,CAAC;AAClC,OAAO,2BAA2B,MAAM,+BAA+B,CAAC;AAExE,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,OAAO,OAAO,gCAAgC;IACnD;;;OAGG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAGvC;QACC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEpC,IAAI,CAAC;YACH,IACE,CAAC,CAAC,MAAM,2BAA2B,CAAC,mBAAmB,CACrD,SAAS,EACT,OAAO,CACR,CAAC,EACF,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,WAAW,GACf,MAAM,qBAAqB,CAAC,iBAAiB,CAAC;gBAC5C,OAAO;aACR,CAAC,CAAC;YAEL,MAAM,2BAA2B,CAAC,WAAW,CAAC;gBAC5C,SAAS;gBACT,OAAO,EAAE,8BAA8B;gBACvC,cAAc,EAAE,OAAO;gBACvB,cAAc,EAAE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC;gBACnD,YAAY,EAAE,KAAK,EAAE,QAIpB,EAAiB,EAAE;oBAClB,MAAM,gBAAgB,CAAC,mBAAmB,CAAC;wBACzC,OAAO;wBACP,SAAS;wBACT,kBAAkB,EAAE,kBAAkB,CAAC,SAAS;wBAChD,YAAY,EAAE,OAAO;wBACrB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;wBAC7C;;;2BAGG;wBACH,qBAAqB,EAAE;4BACrB,yBAAyB,EAAE,QAAQ,CAAC,WAAW;yBAChD;qBACF,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,kDAAkD,OAAO,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACjF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,0DAA0D;IAClD,MAAM,CAAC,iBAAiB,CAAC,WAA6B;;QAC5D,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,WAAW,CAAC;QAE7C,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;QAE7C,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,aAAa,CAAA,MAAA,KAAK,CAAC,aAAa,0CAAE,IAAI,KAAI,KAAK,EAAE,CAAC,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAA,MAAA,KAAK,CAAC,iBAAiB,0CAAE,IAAI,KAAI,KAAK,EAAE,CAAC,CAAC;QACvE,KAAK,CAAC,IAAI,CACR,gBACE,KAAK,CAAC,SAAS;YACb,CAAC,CAAC,aAAa,CAAC,wBAAwB,CAAC,KAAK,CAAC,SAAS,CAAC;YACzD,CAAC,CAAC,KACN,EAAE,CACH,CAAC;QAEF,IAAI,MAAA,KAAK,CAAC,OAAO,0CAAE,IAAI,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,KAAK,CAAC,IAAI,CACR,WAAW,KAAK,CAAC,MAAM;iBACpB,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,oCAAoC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,WAAW,GAAkB,CAAC,aAAa,IAAI,EAAE,CAAC;aACrD,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,IAAuB,EAAE,EAAE;YAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AA9GqB;IADnB,WAAW,EAAE;;;;iEAqDb"}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
|
+
};
|
|
10
|
+
import OneUptimeDate from "../../../../Types/Date";
|
|
11
|
+
import { Blue500 } from "../../../../Types/BrandColors";
|
|
12
|
+
import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
|
|
13
|
+
import IncidentFeedService from "../../../Services/IncidentFeedService";
|
|
14
|
+
import IncidentAIContextBuilder from "../IncidentAIContextBuilder";
|
|
15
|
+
import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
|
|
16
|
+
import SentinelMemory from "./SentinelMemory";
|
|
17
|
+
import logger from "../../Logger";
|
|
18
|
+
import CaptureSpan from "../../Telemetry/CaptureSpan";
|
|
19
|
+
/*
|
|
20
|
+
* Sentinel — incident investigation.
|
|
21
|
+
*
|
|
22
|
+
* When a new incident is declared, wakes Sentinel to investigate it (read-only)
|
|
23
|
+
* and post a cited root-cause analysis into the incident timeline + Slack/Teams
|
|
24
|
+
* — before the on-call engineer has finished reading the page. The heavy lifting
|
|
25
|
+
* (run lifecycle, the agent loop, confidence gating) lives in the shared
|
|
26
|
+
* SentinelInvestigationEngine; this file only assembles incident context and
|
|
27
|
+
* posts the result to the incident's feed.
|
|
28
|
+
*
|
|
29
|
+
* Built on the fire-and-forget async pattern already used throughout
|
|
30
|
+
* IncidentService.onCreateSuccess. See Internal/Roadmap/AISentinelReliabilityBrain.md.
|
|
31
|
+
*/
|
|
32
|
+
export default class SentinelIncidentInvestigationRunner {
|
|
33
|
+
/*
|
|
34
|
+
* Entry point called (fire-and-forget) from IncidentService.onCreateSuccess.
|
|
35
|
+
* Never throws — all failures are logged and recorded on the AIRun.
|
|
36
|
+
*/
|
|
37
|
+
static async investigateNewIncident(data) {
|
|
38
|
+
const { incidentId, projectId } = data;
|
|
39
|
+
try {
|
|
40
|
+
if (!(await SentinelInvestigationEngine.isEnabledForProject(projectId, "Incident"))) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const contextData = await IncidentAIContextBuilder.buildIncidentContext({
|
|
44
|
+
incidentId,
|
|
45
|
+
includeWorkspaceMessages: false,
|
|
46
|
+
});
|
|
47
|
+
/*
|
|
48
|
+
* Recurrence memory: pull in past resolved incidents that share this
|
|
49
|
+
* incident's monitors/labels so Sentinel can recognise "I've seen this
|
|
50
|
+
* before" and reference the prior fix.
|
|
51
|
+
*/
|
|
52
|
+
const priorCasesContext = await SentinelMemory.getPriorSimilarIncidentsContext({
|
|
53
|
+
projectId,
|
|
54
|
+
currentIncidentId: incidentId,
|
|
55
|
+
monitorNames: (contextData.incident.monitors || [])
|
|
56
|
+
.map((m) => {
|
|
57
|
+
return m.name || "";
|
|
58
|
+
})
|
|
59
|
+
.filter(Boolean),
|
|
60
|
+
labelNames: (contextData.incident.labels || [])
|
|
61
|
+
.map((l) => {
|
|
62
|
+
return l.name || "";
|
|
63
|
+
})
|
|
64
|
+
.filter(Boolean),
|
|
65
|
+
});
|
|
66
|
+
await SentinelInvestigationEngine.investigate({
|
|
67
|
+
projectId,
|
|
68
|
+
feature: "Sentinel Incident Investigation",
|
|
69
|
+
subjectIncidentId: incidentId,
|
|
70
|
+
contextSummary: this.buildIncidentSummary(contextData) + priorCasesContext,
|
|
71
|
+
postAnalysis: async (postData) => {
|
|
72
|
+
await IncidentFeedService.createIncidentFeedItem({
|
|
73
|
+
incidentId,
|
|
74
|
+
projectId,
|
|
75
|
+
incidentFeedEventType: IncidentFeedEventType.RootCause,
|
|
76
|
+
displayColor: Blue500,
|
|
77
|
+
feedInfoInMarkdown: postData.analysisMarkdown,
|
|
78
|
+
/*
|
|
79
|
+
* Quiet mode: an inconclusive investigation posts to the timeline
|
|
80
|
+
* but does NOT loudly ping the workspace / on-call.
|
|
81
|
+
*/
|
|
82
|
+
workspaceNotification: {
|
|
83
|
+
sendWorkspaceNotification: postData.isConfident,
|
|
84
|
+
},
|
|
85
|
+
});
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
logger.error(`Sentinel: unexpected error investigating incident ${incidentId.toString()}: ${error}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// Build a compact incident record to seed the investigation.
|
|
94
|
+
static buildIncidentSummary(contextData) {
|
|
95
|
+
var _a, _b;
|
|
96
|
+
const { incident, internalNotes } = contextData;
|
|
97
|
+
const lines = [];
|
|
98
|
+
lines.push("# Incident");
|
|
99
|
+
lines.push(`Title: ${incident.title || "N/A"}`);
|
|
100
|
+
if (incident.description) {
|
|
101
|
+
lines.push(`Description: ${incident.description}`);
|
|
102
|
+
}
|
|
103
|
+
lines.push(`Severity: ${((_a = incident.incidentSeverity) === null || _a === void 0 ? void 0 : _a.name) || "N/A"}`);
|
|
104
|
+
lines.push(`Current state: ${((_b = incident.currentIncidentState) === null || _b === void 0 ? void 0 : _b.name) || "N/A"}`);
|
|
105
|
+
lines.push(`Declared at: ${incident.createdAt
|
|
106
|
+
? OneUptimeDate.getDateAsFormattedString(incident.createdAt)
|
|
107
|
+
: "N/A"}`);
|
|
108
|
+
if (incident.monitors && incident.monitors.length > 0) {
|
|
109
|
+
lines.push(`Affected monitors: ${incident.monitors
|
|
110
|
+
.map((m) => {
|
|
111
|
+
return m.name;
|
|
112
|
+
})
|
|
113
|
+
.filter(Boolean)
|
|
114
|
+
.join(", ")}`);
|
|
115
|
+
}
|
|
116
|
+
if (incident.labels && incident.labels.length > 0) {
|
|
117
|
+
lines.push(`Labels: ${incident.labels
|
|
118
|
+
.map((l) => {
|
|
119
|
+
return l.name;
|
|
120
|
+
})
|
|
121
|
+
.filter(Boolean)
|
|
122
|
+
.join(", ")}`);
|
|
123
|
+
}
|
|
124
|
+
if (incident.rootCause) {
|
|
125
|
+
lines.push(`Root cause (as recorded so far): ${incident.rootCause}`);
|
|
126
|
+
}
|
|
127
|
+
const recentNotes = (internalNotes || [])
|
|
128
|
+
.slice(-5)
|
|
129
|
+
.map((note) => {
|
|
130
|
+
return note.note ? `- ${note.note}` : "";
|
|
131
|
+
})
|
|
132
|
+
.filter(Boolean);
|
|
133
|
+
if (recentNotes.length > 0) {
|
|
134
|
+
lines.push("");
|
|
135
|
+
lines.push("Recent internal notes:");
|
|
136
|
+
lines.push(...recentNotes);
|
|
137
|
+
}
|
|
138
|
+
return lines.join("\n");
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
__decorate([
|
|
142
|
+
CaptureSpan(),
|
|
143
|
+
__metadata("design:type", Function),
|
|
144
|
+
__metadata("design:paramtypes", [Object]),
|
|
145
|
+
__metadata("design:returntype", Promise)
|
|
146
|
+
], SentinelIncidentInvestigationRunner, "investigateNewIncident", null);
|
|
147
|
+
//# sourceMappingURL=IncidentInvestigationRunner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IncidentInvestigationRunner.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Sentinel/IncidentInvestigationRunner.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,+BAA+B,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gDAAgD,CAAC;AACvF,OAAO,mBAAmB,MAAM,uCAAuC,CAAC;AACxE,OAAO,wBAEN,MAAM,6BAA6B,CAAC;AACrC,OAAO,2BAA2B,MAAM,+BAA+B,CAAC;AACxE,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAE9C,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,OAAO,OAAO,mCAAmC;IACtD;;;OAGG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAG1C;QACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEvC,IAAI,CAAC;YACH,IACE,CAAC,CAAC,MAAM,2BAA2B,CAAC,mBAAmB,CACrD,SAAS,EACT,UAAU,CACX,CAAC,EACF,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,WAAW,GACf,MAAM,wBAAwB,CAAC,oBAAoB,CAAC;gBAClD,UAAU;gBACV,wBAAwB,EAAE,KAAK;aAChC,CAAC,CAAC;YAEL;;;;eAIG;YACH,MAAM,iBAAiB,GACrB,MAAM,cAAc,CAAC,+BAA+B,CAAC;gBACnD,SAAS;gBACT,iBAAiB,EAAE,UAAU;gBAC7B,YAAY,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;qBAChD,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;oBAC5B,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;gBAClB,UAAU,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC;qBAC5C,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;oBAC5B,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;aACnB,CAAC,CAAC;YAEL,MAAM,2BAA2B,CAAC,WAAW,CAAC;gBAC5C,SAAS;gBACT,OAAO,EAAE,iCAAiC;gBAC1C,iBAAiB,EAAE,UAAU;gBAC7B,cAAc,EACZ,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,iBAAiB;gBAC5D,YAAY,EAAE,KAAK,EAAE,QAIpB,EAAiB,EAAE;oBAClB,MAAM,mBAAmB,CAAC,sBAAsB,CAAC;wBAC/C,UAAU;wBACV,SAAS;wBACT,qBAAqB,EAAE,qBAAqB,CAAC,SAAS;wBACtD,YAAY,EAAE,OAAO;wBACrB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;wBAC7C;;;2BAGG;wBACH,qBAAqB,EAAE;4BACrB,yBAAyB,EAAE,QAAQ,CAAC,WAAW;yBAChD;qBACF,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,qDAAqD,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,6DAA6D;IACrD,MAAM,CAAC,oBAAoB,CACjC,WAAgC;;QAEhC,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,WAAW,CAAC;QAEhD,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;QAEhD,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,aAAa,CAAA,MAAA,QAAQ,CAAC,gBAAgB,0CAAE,IAAI,KAAI,KAAK,EAAE,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CACR,kBAAkB,CAAA,MAAA,QAAQ,CAAC,oBAAoB,0CAAE,IAAI,KAAI,KAAK,EAAE,CACjE,CAAC;QACF,KAAK,CAAC,IAAI,CACR,gBACE,QAAQ,CAAC,SAAS;YAChB,CAAC,CAAC,aAAa,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5D,CAAC,CAAC,KACN,EAAE,CACH,CAAC;QAEF,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,KAAK,CAAC,IAAI,CACR,sBAAsB,QAAQ,CAAC,QAAQ;iBACpC,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,KAAK,CAAC,IAAI,CACR,WAAW,QAAQ,CAAC,MAAM;iBACvB,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,oCAAoC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,WAAW,GAAkB,CAAC,aAAa,IAAI,EAAE,CAAC;aACrD,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,IAAuB,EAAE,EAAE;YAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AAhJqB;IADnB,WAAW,EAAE;;;;uEA4Eb"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
|
+
};
|
|
10
|
+
import { Blue500 } from "../../../../Types/BrandColors";
|
|
11
|
+
import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
|
|
12
|
+
import IncidentService from "../../../Services/IncidentService";
|
|
13
|
+
import IncidentFeedService from "../../../Services/IncidentFeedService";
|
|
14
|
+
import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
|
|
15
|
+
import logger from "../../Logger";
|
|
16
|
+
import CaptureSpan from "../../Telemetry/CaptureSpan";
|
|
17
|
+
/*
|
|
18
|
+
* Sentinel — auto-draft postmortem on resolve (Phase 3, "close the loop").
|
|
19
|
+
*
|
|
20
|
+
* When an incident moves to a resolved state, Sentinel drafts a postmortem from
|
|
21
|
+
* the incident timeline + telemetry (reusing the existing
|
|
22
|
+
* IncidentService.generatePostmortemFromAI) and saves it to the incident for a
|
|
23
|
+
* human to review and edit — turning a ~90-minute manual writeup into a review.
|
|
24
|
+
*
|
|
25
|
+
* It NEVER overwrites a postmortem that already exists (human work wins), is
|
|
26
|
+
* gated by the same per-project opt-in + LLM provider as investigations, and is
|
|
27
|
+
* fire-and-forget: failures are logged, never surfaced to the resolve flow.
|
|
28
|
+
*/
|
|
29
|
+
const MAX_FEED_PREVIEW_CHARS = 6000;
|
|
30
|
+
export default class SentinelIncidentPostmortemRunner {
|
|
31
|
+
static async draftPostmortemOnResolve(data) {
|
|
32
|
+
const { incidentId, projectId } = data;
|
|
33
|
+
try {
|
|
34
|
+
if (!(await SentinelInvestigationEngine.isEnabledForProject(projectId, "Incident"))) {
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
const incident = await IncidentService.findOneById({
|
|
38
|
+
id: incidentId,
|
|
39
|
+
select: {
|
|
40
|
+
_id: true,
|
|
41
|
+
incidentNumber: true,
|
|
42
|
+
postmortemNote: true,
|
|
43
|
+
},
|
|
44
|
+
props: { isRoot: true },
|
|
45
|
+
});
|
|
46
|
+
if (!incident) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
// Never clobber an existing postmortem (human or previously drafted).
|
|
50
|
+
if (incident.postmortemNote &&
|
|
51
|
+
incident.postmortemNote.trim().length > 0) {
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const draft = await IncidentService.generatePostmortemFromAI({
|
|
55
|
+
incidentId,
|
|
56
|
+
});
|
|
57
|
+
if (!draft || !draft.trim()) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
await IncidentService.updateOneById({
|
|
61
|
+
id: incidentId,
|
|
62
|
+
data: {
|
|
63
|
+
postmortemNote: draft,
|
|
64
|
+
},
|
|
65
|
+
props: { isRoot: true },
|
|
66
|
+
});
|
|
67
|
+
await IncidentFeedService.createIncidentFeedItem({
|
|
68
|
+
incidentId,
|
|
69
|
+
projectId,
|
|
70
|
+
incidentFeedEventType: IncidentFeedEventType.PostmortemNote,
|
|
71
|
+
displayColor: Blue500,
|
|
72
|
+
feedInfoInMarkdown: `## 🧠 Sentinel — Draft Postmortem\n\nSentinel drafted a postmortem for incident #${incident.incidentNumber} from the incident timeline and telemetry. It has been saved on the incident for you to review and edit.`,
|
|
73
|
+
moreInformationInMarkdown: draft.length > MAX_FEED_PREVIEW_CHARS
|
|
74
|
+
? `${draft.substring(0, MAX_FEED_PREVIEW_CHARS)}\n\n…(truncated — the full draft is saved on the incident.)`
|
|
75
|
+
: draft,
|
|
76
|
+
workspaceNotification: {
|
|
77
|
+
sendWorkspaceNotification: true,
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
logger.debug(`Sentinel: drafted postmortem for incident ${incidentId.toString()}.`);
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
logger.error(`Sentinel: failed to draft postmortem for incident ${incidentId.toString()}: ${error}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
__decorate([
|
|
88
|
+
CaptureSpan(),
|
|
89
|
+
__metadata("design:type", Function),
|
|
90
|
+
__metadata("design:paramtypes", [Object]),
|
|
91
|
+
__metadata("design:returntype", Promise)
|
|
92
|
+
], SentinelIncidentPostmortemRunner, "draftPostmortemOnResolve", null);
|
|
93
|
+
//# sourceMappingURL=IncidentPostmortemRunner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IncidentPostmortemRunner.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Sentinel/IncidentPostmortemRunner.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,EAAE,OAAO,EAAE,MAAM,+BAA+B,CAAC;AAExD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gDAAgD,CAAC;AACvF,OAAO,eAAe,MAAM,mCAAmC,CAAC;AAChE,OAAO,mBAAmB,MAAM,uCAAuC,CAAC;AACxE,OAAO,2BAA2B,MAAM,+BAA+B,CAAC;AACxE,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;;GAWG;AACH,MAAM,sBAAsB,GAAW,IAAI,CAAC;AAE5C,MAAM,CAAC,OAAO,OAAO,gCAAgC;IAE/B,AAAb,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,IAG5C;QACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEvC,IAAI,CAAC;YACH,IACE,CAAC,CAAC,MAAM,2BAA2B,CAAC,mBAAmB,CACrD,SAAS,EACT,UAAU,CACX,CAAC,EACF,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,QAAQ,GAAoB,MAAM,eAAe,CAAC,WAAW,CAAC;gBAClE,EAAE,EAAE,UAAU;gBACd,MAAM,EAAE;oBACN,GAAG,EAAE,IAAI;oBACT,cAAc,EAAE,IAAI;oBACpB,cAAc,EAAE,IAAI;iBACrB;gBACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO;YACT,CAAC;YAED,sEAAsE;YACtE,IACE,QAAQ,CAAC,cAAc;gBACvB,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EACzC,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,KAAK,GAAW,MAAM,eAAe,CAAC,wBAAwB,CAAC;gBACnE,UAAU;aACX,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC5B,OAAO;YACT,CAAC;YAED,MAAM,eAAe,CAAC,aAAa,CAAC;gBAClC,EAAE,EAAE,UAAU;gBACd,IAAI,EAAE;oBACJ,cAAc,EAAE,KAAK;iBACtB;gBACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YAEH,MAAM,mBAAmB,CAAC,sBAAsB,CAAC;gBAC/C,UAAU;gBACV,SAAS;gBACT,qBAAqB,EAAE,qBAAqB,CAAC,cAAc;gBAC3D,YAAY,EAAE,OAAO;gBACrB,kBAAkB,EAAE,oFAAoF,QAAQ,CAAC,cAAc,0GAA0G;gBACzO,yBAAyB,EACvB,KAAK,CAAC,MAAM,GAAG,sBAAsB;oBACnC,CAAC,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,sBAAsB,CAAC,6DAA6D;oBAC5G,CAAC,CAAC,KAAK;gBACX,qBAAqB,EAAE;oBACrB,yBAAyB,EAAE,IAAI;iBAChC;aACF,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CACV,6CAA6C,UAAU,CAAC,QAAQ,EAAE,GAAG,CACtE,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,qDAAqD,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AA9EqB;IADnB,WAAW,EAAE;;;;sEA8Eb"}
|