@oneuptime/common 11.5.11 → 11.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/NetworkFlow.ts +12 -1
- package/Models/DatabaseModels/AIInsight.ts +30 -0
- package/Models/DatabaseModels/Project.ts +30 -0
- package/Models/DatabaseModels/TelemetryException.ts +128 -0
- package/Server/API/AIAgentDataAPI.ts +129 -5
- package/Server/API/AIInvestigationAPI.ts +87 -0
- package/Server/API/StatusPageAPI.ts +14 -1
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784640000000-AddExceptionTriageAndPrivacyColumns.ts +56 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +2 -2
- package/Server/Services/AIRunService.ts +8 -0
- package/Server/Services/IncidentService.ts +62 -67
- package/Server/Services/MonitorGroupService.ts +15 -2
- package/Server/Services/MonitorService.ts +91 -3
- package/Server/Services/MonitorStatusTimelineService.ts +285 -100
- package/Server/Services/StatusPageService.ts +59 -34
- package/Server/Services/TelemetryExceptionService.ts +320 -24
- package/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.ts +15 -4
- package/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.ts +15 -4
- package/Server/Utils/AI/SRE/Insights/InsightScanner.ts +222 -24
- package/Server/Utils/AI/SRE/Insights/InsightTriageRunner.ts +225 -9
- package/Server/Utils/AI/SRE/SubjectCodeFixRun.ts +39 -0
- package/Server/Utils/AI/SRE/TelemetryImprovementTaskTrigger.ts +114 -0
- package/Server/Utils/Monitor/MonitorResource.ts +43 -14
- package/Server/Utils/Monitor/MonitorStatusTimeline.ts +32 -7
- package/Server/Utils/Monitor/MonitorStatusTimelineReconciler.ts +362 -0
- package/Server/Utils/Telemetry/ExceptionSanitizer.ts +222 -0
- package/Tests/Server/Services/MonitorStatusTimelineService.test.ts +235 -0
- package/Tests/Server/Services/TelemetryExceptionCodeFixRun.test.ts +205 -1
- package/Tests/Server/Utils/AI/Insights/InsightScanner.test.ts +150 -141
- package/Tests/Server/Utils/AI/Insights/InsightTriageRunner.test.ts +277 -1
- package/Tests/Server/Utils/ExceptionSanitizer.test.ts +70 -0
- package/Tests/Server/Utils/InsightTriageClassification.test.ts +57 -0
- package/Tests/Server/Utils/Monitor/MonitorStatusTimelineReconciler.test.ts +555 -0
- package/Tests/Server/Utils/TelemetryImprovementTaskTrigger.test.ts +141 -0
- package/Tests/Types/Database/CompareOperatorWireSerialization.test.ts +216 -0
- package/Tests/Types/Database/GreaterThan.test.ts +5 -1
- package/Tests/Types/Database/GreaterThanOrEqual.test.ts +5 -1
- package/Tests/Types/Database/GreaterThanOrNull.test.ts +24 -1
- package/Tests/Types/Database/LessThan.test.ts +5 -1
- package/Tests/Types/Database/LessThanOrEqual.test.ts +24 -1
- package/Tests/Types/Database/LessThanOrNull.test.ts +5 -1
- package/Tests/UI/Components/ModelTableSelectFromColumns.test.ts +259 -0
- package/Tests/Utils/Uptime/UptimeUtil.test.ts +726 -0
- package/Types/AI/CodeFixTaskContext.ts +6 -0
- package/Types/AI/CodeFixTaskType.ts +38 -1
- package/Types/AI/ExceptionAIClassification.ts +27 -0
- package/Types/BaseDatabase/GreaterThan.ts +8 -1
- package/Types/BaseDatabase/GreaterThanOrEqual.ts +8 -1
- package/Types/BaseDatabase/GreaterThanOrNull.ts +9 -1
- package/Types/BaseDatabase/LessThan.ts +8 -1
- package/Types/BaseDatabase/LessThanOrEqual.ts +9 -1
- package/Types/BaseDatabase/LessThanOrNull.ts +8 -1
- package/Types/Log/LogScrubPatternType.ts +7 -0
- package/Types/Trace/TraceScrubPatternType.ts +7 -0
- package/UI/Components/ModelTable/BaseModelTable.tsx +15 -46
- package/UI/Components/ModelTable/SelectFromColumns.ts +129 -0
- package/Utils/StatusPage/ResourceUptime.ts +11 -1
- package/Utils/Uptime/UptimeUtil.ts +142 -36
- package/build/dist/Models/AnalyticsModels/NetworkFlow.js +12 -1
- package/build/dist/Models/AnalyticsModels/NetworkFlow.js.map +1 -1
- package/build/dist/Models/DatabaseModels/AIInsight.js +31 -0
- package/build/dist/Models/DatabaseModels/AIInsight.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Project.js +31 -0
- package/build/dist/Models/DatabaseModels/Project.js.map +1 -1
- package/build/dist/Models/DatabaseModels/TelemetryException.js +134 -0
- package/build/dist/Models/DatabaseModels/TelemetryException.js.map +1 -1
- package/build/dist/Server/API/AIAgentDataAPI.js +97 -8
- package/build/dist/Server/API/AIAgentDataAPI.js.map +1 -1
- package/build/dist/Server/API/AIInvestigationAPI.js +55 -0
- package/build/dist/Server/API/AIInvestigationAPI.js.map +1 -1
- package/build/dist/Server/API/StatusPageAPI.js +12 -2
- package/build/dist/Server/API/StatusPageAPI.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784640000000-AddExceptionTriageAndPrivacyColumns.js +33 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784640000000-AddExceptionTriageAndPrivacyColumns.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +2 -2
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/AIRunService.js +9 -2
- package/build/dist/Server/Services/AIRunService.js.map +1 -1
- package/build/dist/Server/Services/IncidentService.js +47 -56
- package/build/dist/Server/Services/IncidentService.js.map +1 -1
- package/build/dist/Server/Services/MonitorGroupService.js +15 -2
- package/build/dist/Server/Services/MonitorGroupService.js.map +1 -1
- package/build/dist/Server/Services/MonitorService.js +66 -3
- package/build/dist/Server/Services/MonitorService.js.map +1 -1
- package/build/dist/Server/Services/MonitorStatusTimelineService.js +244 -102
- package/build/dist/Server/Services/MonitorStatusTimelineService.js.map +1 -1
- package/build/dist/Server/Services/StatusPageService.js +57 -33
- package/build/dist/Server/Services/StatusPageService.js.map +1 -1
- package/build/dist/Server/Services/TelemetryExceptionService.js +263 -22
- package/build/dist/Server/Services/TelemetryExceptionService.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.js +12 -4
- package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.js +12 -4
- package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/Insights/InsightScanner.js +193 -22
- package/build/dist/Server/Utils/AI/SRE/Insights/InsightScanner.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/Insights/InsightTriageRunner.js +178 -9
- package/build/dist/Server/Utils/AI/SRE/Insights/InsightTriageRunner.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/SubjectCodeFixRun.js +37 -0
- package/build/dist/Server/Utils/AI/SRE/SubjectCodeFixRun.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/TelemetryImprovementTaskTrigger.js +98 -0
- package/build/dist/Server/Utils/AI/SRE/TelemetryImprovementTaskTrigger.js.map +1 -0
- package/build/dist/Server/Utils/Monitor/MonitorResource.js +34 -12
- package/build/dist/Server/Utils/Monitor/MonitorResource.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStatusTimeline.js +25 -8
- package/build/dist/Server/Utils/Monitor/MonitorStatusTimeline.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStatusTimelineReconciler.js +268 -0
- package/build/dist/Server/Utils/Monitor/MonitorStatusTimelineReconciler.js.map +1 -0
- package/build/dist/Server/Utils/Telemetry/ExceptionSanitizer.js +149 -0
- package/build/dist/Server/Utils/Telemetry/ExceptionSanitizer.js.map +1 -0
- package/build/dist/Types/AI/CodeFixTaskType.js +38 -1
- package/build/dist/Types/AI/CodeFixTaskType.js.map +1 -1
- package/build/dist/Types/AI/ExceptionAIClassification.js +28 -0
- package/build/dist/Types/AI/ExceptionAIClassification.js.map +1 -0
- package/build/dist/Types/BaseDatabase/GreaterThan.js +8 -1
- package/build/dist/Types/BaseDatabase/GreaterThan.js.map +1 -1
- package/build/dist/Types/BaseDatabase/GreaterThanOrEqual.js +8 -1
- package/build/dist/Types/BaseDatabase/GreaterThanOrEqual.js.map +1 -1
- package/build/dist/Types/BaseDatabase/GreaterThanOrNull.js +9 -1
- package/build/dist/Types/BaseDatabase/GreaterThanOrNull.js.map +1 -1
- package/build/dist/Types/BaseDatabase/LessThan.js +8 -1
- package/build/dist/Types/BaseDatabase/LessThan.js.map +1 -1
- package/build/dist/Types/BaseDatabase/LessThanOrEqual.js +9 -1
- package/build/dist/Types/BaseDatabase/LessThanOrEqual.js.map +1 -1
- package/build/dist/Types/BaseDatabase/LessThanOrNull.js +8 -1
- package/build/dist/Types/BaseDatabase/LessThanOrNull.js.map +1 -1
- package/build/dist/Types/Log/LogScrubPatternType.js +7 -0
- package/build/dist/Types/Log/LogScrubPatternType.js.map +1 -1
- package/build/dist/Types/Trace/TraceScrubPatternType.js +7 -0
- package/build/dist/Types/Trace/TraceScrubPatternType.js.map +1 -1
- package/build/dist/UI/Components/ModelTable/BaseModelTable.js +12 -37
- package/build/dist/UI/Components/ModelTable/BaseModelTable.js.map +1 -1
- package/build/dist/UI/Components/ModelTable/SelectFromColumns.js +82 -0
- package/build/dist/UI/Components/ModelTable/SelectFromColumns.js.map +1 -0
- package/build/dist/Utils/StatusPage/ResourceUptime.js +4 -1
- package/build/dist/Utils/StatusPage/ResourceUptime.js.map +1 -1
- package/build/dist/Utils/Uptime/UptimeUtil.js +86 -33
- package/build/dist/Utils/Uptime/UptimeUtil.js.map +1 -1
- package/package.json +1 -1
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784401962564-MigrationName.ts +0 -101
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784401962564-MigrationName.js +0 -40
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784401962564-MigrationName.js.map +0 -1
|
@@ -11,6 +11,12 @@ import InsightStore, { UpsertCandidatesResult } from "./InsightStore";
|
|
|
11
11
|
import InsightFixRouting, { InsightFixRoutingResult } from "./FixRouting";
|
|
12
12
|
import InsightTriage, { InsightTriageResult } from "./Triage";
|
|
13
13
|
import { InsightCandidate, InsightDetector, InsightScanContext } from "./Types";
|
|
14
|
+
import AIInsightType from "../../../../../Types/AI/AIInsightType";
|
|
15
|
+
import AIRunType from "../../../../../Types/AI/AIRunType";
|
|
16
|
+
import AIRunStatus from "../../../../../Types/AI/AIRunStatus";
|
|
17
|
+
import ExceptionAIClassification from "../../../../../Types/AI/ExceptionAIClassification";
|
|
18
|
+
import AIRunService from "../../../../Services/AIRunService";
|
|
19
|
+
import QueryHelper from "../../../../Types/Database/QueryHelper";
|
|
14
20
|
import logger from "../../../Logger";
|
|
15
21
|
import CaptureSpan from "../../../Telemetry/CaptureSpan";
|
|
16
22
|
|
|
@@ -19,8 +25,9 @@ import CaptureSpan from "../../../Telemetry/CaptureSpan";
|
|
|
19
25
|
*
|
|
20
26
|
* THE RULE: no LLM here. Detectors are deterministic statistical sensors;
|
|
21
27
|
* the LLM only engages per-finding AFTERWARDS through InsightTriage
|
|
22
|
-
* (budgeted, read-only)
|
|
23
|
-
*
|
|
28
|
+
* (budgeted, read-only). The automatic fix decision happens after triage
|
|
29
|
+
* completes (InsightTriageRunner → InsightFixRouting) and only for
|
|
30
|
+
* insights the triage classified as code faults.
|
|
24
31
|
*
|
|
25
32
|
* Everything is opt-in and quiet: only projects with the default-false
|
|
26
33
|
* enableAiInsights flag are scanned, insights never page and never
|
|
@@ -73,8 +80,8 @@ export default class InsightScanner {
|
|
|
73
80
|
* One project's scan: self-heal any insight an earlier tick left unrouted,
|
|
74
81
|
* run every registered detector (isolated — one failing sensor must not
|
|
75
82
|
* blind the others), dedupe/upsert the candidates through InsightStore,
|
|
76
|
-
* then route each NEWLY created insight to
|
|
77
|
-
*
|
|
83
|
+
* then route each NEWLY created insight to ActionRequired and enqueue its
|
|
84
|
+
* triage (which, on a code-fault verdict, routes the fix).
|
|
78
85
|
*/
|
|
79
86
|
@CaptureSpan()
|
|
80
87
|
public static async scanProjectForInsights(project: Project): Promise<void> {
|
|
@@ -211,19 +218,200 @@ export default class InsightScanner {
|
|
|
211
218
|
);
|
|
212
219
|
}
|
|
213
220
|
}
|
|
221
|
+
|
|
222
|
+
await this.resweepUntriagedInsights({ projectId });
|
|
223
|
+
await this.resweepUnroutedCodeFaults({ projectId, project });
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/*
|
|
227
|
+
* Stranded shape 2: ActionRequired insights that never got a triage
|
|
228
|
+
* verdict. With triage gating the automatic fix, a triage that was
|
|
229
|
+
* quiet-skipped (token budget exhausted), expired in the queue, or
|
|
230
|
+
* failed would otherwise strand the insight verdict-less FOREVER — and
|
|
231
|
+
* with it any automatic fix. Re-enqueueing is safe and cheap: the
|
|
232
|
+
* triage enqueue dedupes to one non-terminal run per insight and
|
|
233
|
+
* quiet-skips when the budget still has no headroom. Bounded two ways:
|
|
234
|
+
* only insights from the last 7 days (older rows predate the verdict
|
|
235
|
+
* gate or were consciously left alone), and only while fewer than 3
|
|
236
|
+
* triage attempts have ended in failure (a permanently failing triage
|
|
237
|
+
* must not retry every 15 minutes for a week).
|
|
238
|
+
*/
|
|
239
|
+
private static async resweepUntriagedInsights(data: {
|
|
240
|
+
projectId: ObjectID;
|
|
241
|
+
}): Promise<void> {
|
|
242
|
+
const { projectId } = data;
|
|
243
|
+
|
|
244
|
+
try {
|
|
245
|
+
const untriaged: Array<AIInsight> = await AIInsightService.findBy({
|
|
246
|
+
query: {
|
|
247
|
+
projectId: projectId,
|
|
248
|
+
status: AIInsightStatus.ActionRequired,
|
|
249
|
+
triageCompletedAt: QueryHelper.isNull(),
|
|
250
|
+
createdAt: QueryHelper.greaterThanEqualTo(
|
|
251
|
+
OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -7),
|
|
252
|
+
),
|
|
253
|
+
},
|
|
254
|
+
select: {
|
|
255
|
+
_id: true,
|
|
256
|
+
projectId: true,
|
|
257
|
+
insightType: true,
|
|
258
|
+
serviceName: true,
|
|
259
|
+
traceId: true,
|
|
260
|
+
telemetryExceptionId: true,
|
|
261
|
+
evidence: true,
|
|
262
|
+
},
|
|
263
|
+
limit: LIMIT_MAX,
|
|
264
|
+
skip: 0,
|
|
265
|
+
props: { isRoot: true },
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
for (const insight of untriaged) {
|
|
269
|
+
try {
|
|
270
|
+
const failedTriageAttempts: number = (
|
|
271
|
+
await AIRunService.countBy({
|
|
272
|
+
query: {
|
|
273
|
+
runType: AIRunType.Investigation,
|
|
274
|
+
triggeredByAiInsightId: insight.id!,
|
|
275
|
+
status: QueryHelper.any([
|
|
276
|
+
AIRunStatus.Error,
|
|
277
|
+
AIRunStatus.Cancelled,
|
|
278
|
+
AIRunStatus.Stale,
|
|
279
|
+
]),
|
|
280
|
+
},
|
|
281
|
+
props: { isRoot: true },
|
|
282
|
+
})
|
|
283
|
+
).toNumber();
|
|
284
|
+
|
|
285
|
+
if (failedTriageAttempts >= 3) {
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
const triageResult: InsightTriageResult =
|
|
290
|
+
await InsightTriage.enqueueInsightTriage({ insight: insight });
|
|
291
|
+
|
|
292
|
+
if (triageResult.triageAiRunId) {
|
|
293
|
+
await AIInsightService.updateOneById({
|
|
294
|
+
id: insight.id!,
|
|
295
|
+
data: {
|
|
296
|
+
triageAiRunId: triageResult.triageAiRunId,
|
|
297
|
+
},
|
|
298
|
+
props: { isRoot: true },
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
} catch (error) {
|
|
302
|
+
logger.error(
|
|
303
|
+
`AI Insights: triage re-enqueue failed for insight ${insight.id?.toString()} — continuing: ${error}`,
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
} catch (error) {
|
|
308
|
+
logger.error(
|
|
309
|
+
`AI Insights: failed to sweep untriaged insights for project ${projectId.toString()} — continuing with the scan: ${error}`,
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/*
|
|
315
|
+
* Stranded shape 3: a code-fault verdict was recorded but the fix was
|
|
316
|
+
* never routed — the pod died between the verdict write and the fix
|
|
317
|
+
* routing, or routing quiet-skipped on a transient gate (daily fix
|
|
318
|
+
* budget, readiness). Re-routing is idempotent: the creation path's
|
|
319
|
+
* per-(exception, recipe) dedupe and the cross-group guard refuse
|
|
320
|
+
* duplicates, and the conditional status flip only moves
|
|
321
|
+
* ActionRequired → FixOpened. Bounded to verdicts from the last 24
|
|
322
|
+
* hours so a permanently refused group (declined by a human, capped
|
|
323
|
+
* repo) does not retry forever. Latency insights are excluded — they
|
|
324
|
+
* are fix-routed deterministically at scan time.
|
|
325
|
+
*/
|
|
326
|
+
private static async resweepUnroutedCodeFaults(data: {
|
|
327
|
+
projectId: ObjectID;
|
|
328
|
+
project: Project;
|
|
329
|
+
}): Promise<void> {
|
|
330
|
+
const { projectId, project } = data;
|
|
331
|
+
|
|
332
|
+
try {
|
|
333
|
+
const unrouted: Array<AIInsight> = await AIInsightService.findBy({
|
|
334
|
+
query: {
|
|
335
|
+
projectId: projectId,
|
|
336
|
+
status: AIInsightStatus.ActionRequired,
|
|
337
|
+
classification: ExceptionAIClassification.CodeFault,
|
|
338
|
+
fixAiRunId: QueryHelper.isNull(),
|
|
339
|
+
triageCompletedAt: QueryHelper.greaterThanEqualTo(
|
|
340
|
+
OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1),
|
|
341
|
+
),
|
|
342
|
+
},
|
|
343
|
+
select: {
|
|
344
|
+
_id: true,
|
|
345
|
+
projectId: true,
|
|
346
|
+
insightType: true,
|
|
347
|
+
serviceName: true,
|
|
348
|
+
traceId: true,
|
|
349
|
+
telemetryExceptionId: true,
|
|
350
|
+
evidence: true,
|
|
351
|
+
},
|
|
352
|
+
limit: LIMIT_MAX,
|
|
353
|
+
skip: 0,
|
|
354
|
+
props: { isRoot: true },
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
for (const insight of unrouted) {
|
|
358
|
+
if (insight.insightType === AIInsightType.TraceLatencyRegression) {
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
try {
|
|
363
|
+
const fixResult: InsightFixRoutingResult =
|
|
364
|
+
await InsightFixRouting.routeInsightFix({
|
|
365
|
+
insight: insight,
|
|
366
|
+
project: project,
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
if (fixResult.fixAiRunId) {
|
|
370
|
+
await AIInsightService.updateOneBy({
|
|
371
|
+
query: {
|
|
372
|
+
_id: insight.id!.toString(),
|
|
373
|
+
status: AIInsightStatus.ActionRequired,
|
|
374
|
+
},
|
|
375
|
+
data: {
|
|
376
|
+
status: AIInsightStatus.FixOpened,
|
|
377
|
+
fixAiRunId: fixResult.fixAiRunId,
|
|
378
|
+
},
|
|
379
|
+
props: { isRoot: true },
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
} catch (error) {
|
|
383
|
+
logger.error(
|
|
384
|
+
`AI Insights: fix re-routing failed for insight ${insight.id?.toString()} — continuing: ${error}`,
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
} catch (error) {
|
|
389
|
+
logger.error(
|
|
390
|
+
`AI Insights: failed to sweep unrouted code-fault insights for project ${projectId.toString()} — continuing with the scan: ${error}`,
|
|
391
|
+
);
|
|
392
|
+
}
|
|
214
393
|
}
|
|
215
394
|
|
|
216
395
|
/*
|
|
217
396
|
* Route one insight — newly created this tick, or swept up as stranded
|
|
218
|
-
* from an earlier one — out of the defensive Detected state
|
|
219
|
-
*
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
*
|
|
397
|
+
* from an earlier one — out of the defensive Detected state.
|
|
398
|
+
*
|
|
399
|
+
* TRIAGE GATES THE FIX. The automatic fix decision no longer happens
|
|
400
|
+
* here: every new insight goes to ActionRequired and gets its triage
|
|
401
|
+
* enqueued; when the triage run completes with a code-fault
|
|
402
|
+
* classification, InsightTriageRunner routes the fix (via
|
|
403
|
+
* InsightFixRouting) and flips the insight to FixOpened. This exists
|
|
404
|
+
* because roughly half of real exception insights turn out to be
|
|
405
|
+
* expected user errors or intentional denials (auth, paywalls, scanner
|
|
406
|
+
* probes) — opening an unreviewed fix PR for those is worse than
|
|
407
|
+
* opening none: the agent's only "fix" is to weaken the check that
|
|
408
|
+
* correctly fired. Fail-closed by design: no triage verdict, no
|
|
409
|
+
* automatic PR. The manual "Fix with AI" button on the exception page
|
|
410
|
+
* is unaffected.
|
|
411
|
+
*
|
|
412
|
+
* The collaborator is never-throws by contract, but it is wrapped
|
|
413
|
+
* anyway: a contract breach must degrade (no triage), not leave the
|
|
414
|
+
* insight stuck in Detected or fail the scan.
|
|
227
415
|
*/
|
|
228
416
|
private static async routeNewInsight(data: {
|
|
229
417
|
insight: AIInsight;
|
|
@@ -231,19 +419,29 @@ export default class InsightScanner {
|
|
|
231
419
|
}): Promise<void> {
|
|
232
420
|
const { insight, project } = data;
|
|
233
421
|
|
|
422
|
+
/*
|
|
423
|
+
* TraceLatencyRegression keeps DETERMINISTIC scan-time fix routing:
|
|
424
|
+
* its evidence is the SpanTreeAnalyzer's own findings (N+1, dominant
|
|
425
|
+
* span, sequential fan-out) — no LLM verdict adds signal there, and
|
|
426
|
+
* the exception-centric classification taxonomy would mislabel most
|
|
427
|
+
* latency regressions as "infrastructure". Exception insights go
|
|
428
|
+
* through the verdict gate instead.
|
|
429
|
+
*/
|
|
234
430
|
let fixAiRunId: ObjectID | undefined = undefined;
|
|
235
431
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
432
|
+
if (insight.insightType === AIInsightType.TraceLatencyRegression) {
|
|
433
|
+
try {
|
|
434
|
+
const fixResult: InsightFixRoutingResult =
|
|
435
|
+
await InsightFixRouting.routeInsightFix({
|
|
436
|
+
insight: insight,
|
|
437
|
+
project: project,
|
|
438
|
+
});
|
|
439
|
+
fixAiRunId = fixResult.fixAiRunId;
|
|
440
|
+
} catch (error) {
|
|
441
|
+
logger.error(
|
|
442
|
+
`AI Insights: fix routing threw for latency insight ${insight.id?.toString()} (treating as no fix opened): ${error}`,
|
|
443
|
+
);
|
|
444
|
+
}
|
|
247
445
|
}
|
|
248
446
|
|
|
249
447
|
if (fixAiRunId) {
|
|
@@ -1,13 +1,23 @@
|
|
|
1
1
|
import ObjectID from "../../../../../Types/ObjectID";
|
|
2
2
|
import OneUptimeDate from "../../../../../Types/Date";
|
|
3
3
|
import AIInsight from "../../../../../Models/DatabaseModels/AIInsight";
|
|
4
|
+
import Project from "../../../../../Models/DatabaseModels/Project";
|
|
4
5
|
import AIInsightEvidence from "../../../../../Types/AI/AIInsightEvidence";
|
|
6
|
+
import AIInsightStatus, {
|
|
7
|
+
AIInsightStatusHelper,
|
|
8
|
+
} from "../../../../../Types/AI/AIInsightStatus";
|
|
9
|
+
import AIInsightType from "../../../../../Types/AI/AIInsightType";
|
|
10
|
+
import ExceptionAIClassification from "../../../../../Types/AI/ExceptionAIClassification";
|
|
5
11
|
import AIInsightService from "../../../../Services/AIInsightService";
|
|
12
|
+
import ProjectService from "../../../../Services/ProjectService";
|
|
13
|
+
import TelemetryExceptionService from "../../../../Services/TelemetryExceptionService";
|
|
6
14
|
import { AI_INSIGHT_TRIAGE_FEATURE } from "../../../../Services/AIService";
|
|
7
15
|
import AIInvestigationEngine from "../AIInvestigationEngine";
|
|
8
16
|
import AIInvestigationQueue from "../InvestigationQueue";
|
|
17
|
+
import InsightFixRouting, { InsightFixRoutingResult } from "./FixRouting";
|
|
9
18
|
import { ConfidenceSignal } from "../ConfidenceSignal";
|
|
10
19
|
import { ObservabilityAssistantResult } from "../../Chat/ObservabilityAssistant";
|
|
20
|
+
import ToolResultSerializer from "../../Toolbox/Serializer";
|
|
11
21
|
import logger from "../../../Logger";
|
|
12
22
|
import CaptureSpan from "../../../Telemetry/CaptureSpan";
|
|
13
23
|
|
|
@@ -44,7 +54,17 @@ const PREVENTIVE_TRIAGE_FRAMING: string = `IMPORTANT — this is PREVENTIVE TRIA
|
|
|
44
54
|
- Establish the most likely root cause of the finding.
|
|
45
55
|
- Estimate the blast radius: which services/users/operations are affected, and how badly.
|
|
46
56
|
- Recommend exactly ONE next action (the single most useful thing an engineer should do first).
|
|
47
|
-
Cite the evidence for every factual claim. If the telemetry cannot support a conclusion, say so plainly. Do not refer to "the incident" — there is none
|
|
57
|
+
Cite the evidence for every factual claim. If the telemetry cannot support a conclusion, say so plainly. Do not refer to "the incident" — there is none.
|
|
58
|
+
|
|
59
|
+
CLASSIFICATION — you MUST end your analysis with one line in EXACTLY this format, on its own line, with nothing after it:
|
|
60
|
+
Classification: <verdict>
|
|
61
|
+
where <verdict> is exactly one of: code-fault, user-error, expected-denial, infrastructure, unknown.
|
|
62
|
+
- code-fault: a defect in the monitored code that a code change should fix.
|
|
63
|
+
- user-error: an expected consequence of invalid end-user input (bad parameters, malformed values, garbage in a URL). The code rejected it correctly; the input was wrong.
|
|
64
|
+
- expected-denial: an intentional check doing its job — authentication/authorization failures, plan or paywall rejections, rate limits, or security scanners/fuzzers tripping validation on purpose-built probes.
|
|
65
|
+
- infrastructure: an environmental condition (network timeout, connection reset, resource exhaustion, dependency outage) rather than a logic defect.
|
|
66
|
+
- unknown: the evidence does not support a confident verdict.
|
|
67
|
+
This verdict gates automation: ONLY code-fault findings may get an automatic fix pull request, so classify conservatively — when torn between code-fault and anything else, pick the other one.`;
|
|
48
68
|
|
|
49
69
|
export default class InsightTriageRunner {
|
|
50
70
|
/*
|
|
@@ -63,11 +83,13 @@ export default class InsightTriageRunner {
|
|
|
63
83
|
const { aiRunId, projectId, sentinelInsightId, attemptCount } = data;
|
|
64
84
|
|
|
65
85
|
let contextSummary: string;
|
|
86
|
+
let insight: AIInsight | null = null;
|
|
66
87
|
try {
|
|
67
|
-
|
|
88
|
+
insight = await AIInsightService.findOneById({
|
|
68
89
|
id: sentinelInsightId,
|
|
69
90
|
select: {
|
|
70
91
|
_id: true,
|
|
92
|
+
projectId: true,
|
|
71
93
|
insightType: true,
|
|
72
94
|
severity: true,
|
|
73
95
|
title: true,
|
|
@@ -75,6 +97,7 @@ export default class InsightTriageRunner {
|
|
|
75
97
|
evidence: true,
|
|
76
98
|
serviceName: true,
|
|
77
99
|
traceId: true,
|
|
100
|
+
telemetryExceptionId: true,
|
|
78
101
|
metricName: true,
|
|
79
102
|
firstSeenAt: true,
|
|
80
103
|
lastSeenAt: true,
|
|
@@ -118,30 +141,217 @@ export default class InsightTriageRunner {
|
|
|
118
141
|
confidence: ConfidenceSignal;
|
|
119
142
|
result: ObservabilityAssistantResult;
|
|
120
143
|
}): Promise<void> => {
|
|
144
|
+
const classification: ExceptionAIClassification =
|
|
145
|
+
this.parseClassification(postData.analysisMarkdown);
|
|
146
|
+
|
|
121
147
|
/*
|
|
122
|
-
* The quiet inbox —
|
|
123
|
-
* no workspace notification, no owner/on-call
|
|
124
|
-
* and no instrumentation-task trigger. Insights
|
|
125
|
-
* never enter the notification pipeline
|
|
126
|
-
* the summary waits on the Insights page
|
|
148
|
+
* The quiet inbox — the summary and verdict land on the insight
|
|
149
|
+
* row: no feed items, no workspace notification, no owner/on-call
|
|
150
|
+
* ping, no metrics, and no instrumentation-task trigger. Insights
|
|
151
|
+
* never page and never enter the notification pipeline
|
|
152
|
+
* (Preventive-lane rule); the summary waits on the Insights page
|
|
153
|
+
* until a human looks.
|
|
127
154
|
*/
|
|
128
155
|
await AIInsightService.updateOneById({
|
|
129
156
|
id: sentinelInsightId,
|
|
130
157
|
data: {
|
|
131
158
|
triageSummaryMarkdown: postData.analysisMarkdown,
|
|
132
159
|
triageCompletedAt: OneUptimeDate.getCurrentDate(),
|
|
160
|
+
classification: classification,
|
|
133
161
|
},
|
|
134
162
|
props: { isRoot: true },
|
|
135
163
|
});
|
|
136
164
|
|
|
137
165
|
logger.debug(
|
|
138
|
-
`AI insights: triage summary posted for insight ${sentinelInsightId.toString()} (run ${aiRunId.toString()}, confident=${postData.confidence.confident} via ${postData.confidence.source}).`,
|
|
166
|
+
`AI insights: triage summary posted for insight ${sentinelInsightId.toString()} (run ${aiRunId.toString()}, classification=${classification}, confident=${postData.confidence.confident} via ${postData.confidence.source}).`,
|
|
139
167
|
);
|
|
168
|
+
|
|
169
|
+
/*
|
|
170
|
+
* Verdict-driven follow-through. Best-effort by design: the
|
|
171
|
+
* triage summary is already posted, and a failure here must not
|
|
172
|
+
* fail the triage run itself.
|
|
173
|
+
*/
|
|
174
|
+
try {
|
|
175
|
+
await this.actOnClassification({
|
|
176
|
+
insight: insight!,
|
|
177
|
+
classification: classification,
|
|
178
|
+
});
|
|
179
|
+
} catch (error) {
|
|
180
|
+
logger.error(
|
|
181
|
+
`AI insights: post-triage action failed for insight ${sentinelInsightId.toString()} (verdict ${classification}): ${error}`,
|
|
182
|
+
);
|
|
183
|
+
}
|
|
140
184
|
},
|
|
141
185
|
},
|
|
142
186
|
});
|
|
143
187
|
}
|
|
144
188
|
|
|
189
|
+
/*
|
|
190
|
+
* Parse the mandatory trailing "Classification: <verdict>" line out of
|
|
191
|
+
* the triage analysis. The LAST match wins (the model may discuss the
|
|
192
|
+
* taxonomy earlier in its reasoning). Anything unparseable is Unknown —
|
|
193
|
+
* and Unknown never routes a fix, so a malformed answer fails closed.
|
|
194
|
+
*/
|
|
195
|
+
public static parseClassification(
|
|
196
|
+
analysisMarkdown: string,
|
|
197
|
+
): ExceptionAIClassification {
|
|
198
|
+
const matches: Array<RegExpMatchArray> = Array.from(
|
|
199
|
+
(analysisMarkdown || "").matchAll(
|
|
200
|
+
/^\s*\**\s*classification\s*\**\s*[:=]\s*\**\s*(code-fault|user-error|expected-denial|infrastructure|unknown)\b/gim,
|
|
201
|
+
),
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
const last: RegExpMatchArray | undefined = matches[matches.length - 1];
|
|
205
|
+
|
|
206
|
+
if (!last || !last[1]) {
|
|
207
|
+
return ExceptionAIClassification.Unknown;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return last[1].toLowerCase() as ExceptionAIClassification;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/*
|
|
214
|
+
* Act on the triage verdict:
|
|
215
|
+
*
|
|
216
|
+
* 1. Stamp the verdict onto the exception group row (aiClassification)
|
|
217
|
+
* so the exceptions list can filter on it and the fix lane can skip
|
|
218
|
+
* known-non-defects without re-triaging.
|
|
219
|
+
* 2. code-fault → route the automatic fix (InsightFixRouting owns every
|
|
220
|
+
* gate: enableAi, enableInsightFixTasks, budget, readiness, dedupe)
|
|
221
|
+
* and flip the insight to FixOpened when a run was queued.
|
|
222
|
+
* 3. expected-denial → optionally auto-archive the exception group when
|
|
223
|
+
* the project opted in (autoArchiveNonActionableExceptions). Only
|
|
224
|
+
* expected denials are archived — user errors and infrastructure
|
|
225
|
+
* conditions stay visible for a human to judge.
|
|
226
|
+
*/
|
|
227
|
+
private static async actOnClassification(data: {
|
|
228
|
+
insight: AIInsight;
|
|
229
|
+
classification: ExceptionAIClassification;
|
|
230
|
+
}): Promise<void> {
|
|
231
|
+
const { insight, classification } = data;
|
|
232
|
+
|
|
233
|
+
if (!insight.id || !insight.projectId) {
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/*
|
|
238
|
+
* The verdict stamp on the exception group is metadata and always
|
|
239
|
+
* lands; the AUTOMATION below (fix PR, auto-archive) additionally
|
|
240
|
+
* requires that no human closed the insight while the triage run sat
|
|
241
|
+
* in the queue or executed. The `insight` object is a snapshot from
|
|
242
|
+
* claim time — minutes to half an hour old — so the status MUST be
|
|
243
|
+
* re-read here: an insight a human Dismissed or Resolved in the
|
|
244
|
+
* meantime gets no PR and keeps its terminal status.
|
|
245
|
+
*/
|
|
246
|
+
if (insight.telemetryExceptionId) {
|
|
247
|
+
await TelemetryExceptionService.updateOneById({
|
|
248
|
+
id: insight.telemetryExceptionId,
|
|
249
|
+
data: {
|
|
250
|
+
aiClassification: classification,
|
|
251
|
+
},
|
|
252
|
+
props: { isRoot: true },
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const currentInsight: AIInsight | null = await AIInsightService.findOneById(
|
|
257
|
+
{
|
|
258
|
+
id: insight.id,
|
|
259
|
+
select: {
|
|
260
|
+
_id: true,
|
|
261
|
+
status: true,
|
|
262
|
+
},
|
|
263
|
+
props: { isRoot: true },
|
|
264
|
+
},
|
|
265
|
+
);
|
|
266
|
+
|
|
267
|
+
if (
|
|
268
|
+
!currentInsight ||
|
|
269
|
+
(currentInsight.status &&
|
|
270
|
+
AIInsightStatusHelper.isTerminalStatus(currentInsight.status))
|
|
271
|
+
) {
|
|
272
|
+
logger.debug(
|
|
273
|
+
`AI insights: skipping post-triage automation for insight ${insight.id.toString()} — insight is ${currentInsight?.status || "deleted"}.`,
|
|
274
|
+
);
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
const project: Project | null = await ProjectService.findOneById({
|
|
279
|
+
id: insight.projectId,
|
|
280
|
+
select: {
|
|
281
|
+
_id: true,
|
|
282
|
+
enableAi: true,
|
|
283
|
+
enableInsightFixTasks: true,
|
|
284
|
+
autoArchiveNonActionableExceptions: true,
|
|
285
|
+
},
|
|
286
|
+
props: { isRoot: true },
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
if (!project) {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/*
|
|
294
|
+
* TraceLatencyRegression insights are fix-routed DETERMINISTICALLY at
|
|
295
|
+
* scan time (their span-tree evidence needs no LLM verdict — see
|
|
296
|
+
* InsightScanner.routeNewInsight); routing them again here would
|
|
297
|
+
* duplicate the run/PR. The verdict gate applies to exception
|
|
298
|
+
* insights only.
|
|
299
|
+
*/
|
|
300
|
+
const isVerdictGatedInsight: boolean =
|
|
301
|
+
insight.insightType !== AIInsightType.TraceLatencyRegression;
|
|
302
|
+
|
|
303
|
+
if (
|
|
304
|
+
classification === ExceptionAIClassification.CodeFault &&
|
|
305
|
+
isVerdictGatedInsight
|
|
306
|
+
) {
|
|
307
|
+
const fixResult: InsightFixRoutingResult =
|
|
308
|
+
await InsightFixRouting.routeInsightFix({
|
|
309
|
+
insight: insight,
|
|
310
|
+
project: project,
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
if (fixResult.fixAiRunId) {
|
|
314
|
+
/*
|
|
315
|
+
* Conditional flip: only a still-ActionRequired insight moves to
|
|
316
|
+
* FixOpened, so a dismissal landing between the status re-read
|
|
317
|
+
* above and this write can never be clobbered.
|
|
318
|
+
*/
|
|
319
|
+
await AIInsightService.updateOneBy({
|
|
320
|
+
query: {
|
|
321
|
+
_id: insight.id.toString(),
|
|
322
|
+
status: AIInsightStatus.ActionRequired,
|
|
323
|
+
},
|
|
324
|
+
data: {
|
|
325
|
+
status: AIInsightStatus.FixOpened,
|
|
326
|
+
fixAiRunId: fixResult.fixAiRunId,
|
|
327
|
+
},
|
|
328
|
+
props: { isRoot: true },
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if (
|
|
336
|
+
classification === ExceptionAIClassification.ExpectedDenial &&
|
|
337
|
+
project.autoArchiveNonActionableExceptions === true &&
|
|
338
|
+
insight.telemetryExceptionId
|
|
339
|
+
) {
|
|
340
|
+
await TelemetryExceptionService.updateOneById({
|
|
341
|
+
id: insight.telemetryExceptionId,
|
|
342
|
+
data: {
|
|
343
|
+
isArchived: true,
|
|
344
|
+
markedAsArchivedAt: OneUptimeDate.getCurrentDate(),
|
|
345
|
+
},
|
|
346
|
+
props: { isRoot: true },
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
logger.debug(
|
|
350
|
+
`AI insights: auto-archived exception ${insight.telemetryExceptionId.toString()} (expected-denial verdict, project opted in).`,
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
145
355
|
// Build the compact insight record that seeds the triage.
|
|
146
356
|
private static buildInsightSummary(insight: AIInsight): string {
|
|
147
357
|
const lines: Array<string> = [];
|
|
@@ -191,7 +401,13 @@ export default class InsightTriageRunner {
|
|
|
191
401
|
lines.push(renderedEvidence);
|
|
192
402
|
}
|
|
193
403
|
|
|
194
|
-
|
|
404
|
+
/*
|
|
405
|
+
* Detector detailMarkdown and evidence quote raw exception messages —
|
|
406
|
+
* sweep secrets/PII before the text reaches the LLM provider. (The
|
|
407
|
+
* chat/investigation toolbox path already redacts its tool results;
|
|
408
|
+
* this closes the same gap for the detector-supplied context.)
|
|
409
|
+
*/
|
|
410
|
+
return ToolResultSerializer.redact(lines.join("\n")).text;
|
|
195
411
|
}
|
|
196
412
|
|
|
197
413
|
/*
|
|
@@ -116,6 +116,45 @@ export default class SubjectCodeFixRun {
|
|
|
116
116
|
);
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
+
/*
|
|
120
|
+
* Per-(service, recipe) dedupe for the service-scoped instrumentation
|
|
121
|
+
* recipes (ImproveLogging / ImproveTracing): at most one non-terminal
|
|
122
|
+
* run of a recipe per (project, telemetryServiceId). Same in-memory
|
|
123
|
+
* taskContext matching as the per-trace guard above — the id lives in
|
|
124
|
+
* JSON the Query layer cannot filter on, and the candidate set is
|
|
125
|
+
* bounded and rare.
|
|
126
|
+
*/
|
|
127
|
+
@CaptureSpan()
|
|
128
|
+
public static async findNonTerminalRunForTelemetryService(data: {
|
|
129
|
+
projectId: ObjectID;
|
|
130
|
+
taskType: CodeFixTaskType;
|
|
131
|
+
telemetryServiceId: string;
|
|
132
|
+
}): Promise<AIRun | null> {
|
|
133
|
+
const activeRuns: Array<AIRun> = await AIRunService.findBy({
|
|
134
|
+
query: {
|
|
135
|
+
projectId: data.projectId,
|
|
136
|
+
runType: AIRunType.CodeFix,
|
|
137
|
+
codeFixTaskType: data.taskType,
|
|
138
|
+
status: QueryHelper.notIn([
|
|
139
|
+
AIRunStatus.Completed,
|
|
140
|
+
AIRunStatus.Error,
|
|
141
|
+
AIRunStatus.Cancelled,
|
|
142
|
+
AIRunStatus.Stale,
|
|
143
|
+
]),
|
|
144
|
+
},
|
|
145
|
+
select: { _id: true, taskContext: true },
|
|
146
|
+
limit: LIMIT_PER_PROJECT,
|
|
147
|
+
skip: 0,
|
|
148
|
+
props: { isRoot: true },
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
return (
|
|
152
|
+
activeRuns.find((run: AIRun): boolean => {
|
|
153
|
+
return run.taskContext?.telemetryServiceId === data.telemetryServiceId;
|
|
154
|
+
}) || null
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
|
|
119
158
|
/*
|
|
120
159
|
* Record the durable intent as a Queued CodeFix AIRun that the external
|
|
121
160
|
* agent worker claims via /ai-agent-task/get-pending-task. `userId` is
|