@junando/core 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +65 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +435 -112
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -261,6 +261,16 @@ var Fingerprint = class Fingerprint {
|
|
|
261
261
|
}
|
|
262
262
|
};
|
|
263
263
|
//#endregion
|
|
264
|
+
//#region src/domain/ports/index.ts
|
|
265
|
+
/**
|
|
266
|
+
* Terminal outcome of a single notification send.
|
|
267
|
+
* Feeds the `notify` section of the wide event.
|
|
268
|
+
*/
|
|
269
|
+
const NotifyOutcome = {
|
|
270
|
+
Success: "success",
|
|
271
|
+
Failure: "failure"
|
|
272
|
+
};
|
|
273
|
+
//#endregion
|
|
264
274
|
//#region src/domain/services/clustering.service.ts
|
|
265
275
|
var ClusteringService = class {
|
|
266
276
|
/**
|
|
@@ -330,93 +340,6 @@ function normalizePayload(payload) {
|
|
|
330
340
|
}));
|
|
331
341
|
}
|
|
332
342
|
//#endregion
|
|
333
|
-
//#region src/application/use-cases/process-incident.use-case.ts
|
|
334
|
-
var ProcessIncidentUseCase = class {
|
|
335
|
-
deps;
|
|
336
|
-
clustering;
|
|
337
|
-
constructor(deps) {
|
|
338
|
-
this.deps = deps;
|
|
339
|
-
this.clustering = deps.clustering ?? new ClusteringService();
|
|
340
|
-
}
|
|
341
|
-
async execute(alerts, correlationId) {
|
|
342
|
-
const { dedup, traces, llm, notifier, logger, dedupTtlSeconds, ruleEngine } = this.deps;
|
|
343
|
-
const log = logger.child({
|
|
344
|
-
correlationId,
|
|
345
|
-
useCase: "ProcessIncident"
|
|
346
|
-
});
|
|
347
|
-
log.info({ alertCount: alerts.length }, "Processing alert batch");
|
|
348
|
-
const clusters = this.clustering.buildClusters(alerts);
|
|
349
|
-
log.info({ clusterCount: clusters.length }, "Clusters built");
|
|
350
|
-
this.deps.onClustersBuilt?.(clusters.length);
|
|
351
|
-
for (const cluster of clusters) {
|
|
352
|
-
const log2 = log.child({
|
|
353
|
-
fingerprint: cluster.fingerprint,
|
|
354
|
-
service: cluster.serviceName
|
|
355
|
-
});
|
|
356
|
-
if (!await dedup.isNew(cluster.fingerprint, dedupTtlSeconds)) {
|
|
357
|
-
log2.debug("Duplicate cluster — skipping");
|
|
358
|
-
dedupDuplicate.inc({ source: "alertmanager" });
|
|
359
|
-
continue;
|
|
360
|
-
}
|
|
361
|
-
dedupNew.inc({ source: "alertmanager" });
|
|
362
|
-
let preLlmRouteChannels = [];
|
|
363
|
-
let preLlmEscalateChannels = [];
|
|
364
|
-
if (ruleEngine) {
|
|
365
|
-
const preResult = ruleEngine.evaluatePreLlm(cluster);
|
|
366
|
-
if (preResult.suppressed) {
|
|
367
|
-
log2.info({ matchedRuleId: preResult.matchedRuleId }, "Cluster suppressed by rule engine");
|
|
368
|
-
if (preResult.matchedRuleId) suppressedClusters.inc({ rule_id: preResult.matchedRuleId });
|
|
369
|
-
continue;
|
|
370
|
-
}
|
|
371
|
-
for (const action of preResult.actions) {
|
|
372
|
-
if (action.type === "route" && "channel" in action) preLlmRouteChannels.push(action.channel);
|
|
373
|
-
if (action.type === "escalate" && "channel" in action) preLlmEscalateChannels.push(action.channel);
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
const allSpans = (await Promise.all(cluster.representativeTraceIds.map((id) => traces.findByTraceId(id).catch((err) => {
|
|
377
|
-
log2.warn({
|
|
378
|
-
err,
|
|
379
|
-
traceId: id
|
|
380
|
-
}, "Trace fetch failed — continuing without it");
|
|
381
|
-
return [];
|
|
382
|
-
})))).flat();
|
|
383
|
-
log2.info({ spanCount: allSpans.length }, "Traces extracted");
|
|
384
|
-
let analysis = null;
|
|
385
|
-
try {
|
|
386
|
-
analysis = await llm.analyze(cluster, allSpans);
|
|
387
|
-
log2.info({ urgency: analysis.urgency_level }, "LLM analysis complete");
|
|
388
|
-
} catch (err) {
|
|
389
|
-
log2.warn({ err }, "LLM inference failed — notifying without diagnosis");
|
|
390
|
-
}
|
|
391
|
-
let postLlmEscalateChannels = [];
|
|
392
|
-
if (ruleEngine && analysis) {
|
|
393
|
-
const postResult = ruleEngine.evaluatePostLlm(cluster, analysis);
|
|
394
|
-
for (const action of postResult.actions) {
|
|
395
|
-
if (action.type === "escalate" && "channel" in action) postLlmEscalateChannels.push(action.channel);
|
|
396
|
-
if (action.type === "tag" && "key" in action) log2.info({
|
|
397
|
-
tagKey: action.key,
|
|
398
|
-
tagValue: action.value
|
|
399
|
-
}, "Tag attached to cluster");
|
|
400
|
-
}
|
|
401
|
-
if (postResult.tags && Object.keys(postResult.tags).length > 0) cluster.labels = {
|
|
402
|
-
...cluster.labels,
|
|
403
|
-
...postResult.tags
|
|
404
|
-
};
|
|
405
|
-
}
|
|
406
|
-
try {
|
|
407
|
-
const primaryChannel = preLlmRouteChannels[0];
|
|
408
|
-
const escalateChannels = [...preLlmEscalateChannels, ...postLlmEscalateChannels];
|
|
409
|
-
await notifier.send(cluster, analysis, primaryChannel);
|
|
410
|
-
for (const channel of escalateChannels) await notifier.send(cluster, analysis, channel);
|
|
411
|
-
log2.info("Notification sent");
|
|
412
|
-
} catch (err) {
|
|
413
|
-
log2.error({ err }, "Notification failed");
|
|
414
|
-
throw err;
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
};
|
|
419
|
-
//#endregion
|
|
420
343
|
//#region src/shared/logger/loki-transport.ts
|
|
421
344
|
/**
|
|
422
345
|
* Maximum number of buffered log entries before the oldest is dropped.
|
|
@@ -502,6 +425,174 @@ async function flushLoki() {
|
|
|
502
425
|
}
|
|
503
426
|
}
|
|
504
427
|
//#endregion
|
|
428
|
+
//#region src/shared/logger/wide-event-builder.ts
|
|
429
|
+
/** Maximum serialized event size: 256 KB. Events beyond this are truncated. */
|
|
430
|
+
const MAX_EVENT_BYTES = 256 * 1024;
|
|
431
|
+
/**
|
|
432
|
+
* Per-string cap applied when an event exceeds MAX_EVENT_BYTES.
|
|
433
|
+
* ~256 strings × 1 KB would still fit; real events have far fewer fields.
|
|
434
|
+
*/
|
|
435
|
+
const OVERSIZED_STRING_CAP = 1024;
|
|
436
|
+
function serializedBytes(value) {
|
|
437
|
+
return Buffer.byteLength(JSON.stringify(value), "utf8");
|
|
438
|
+
}
|
|
439
|
+
function shrinkStrings(value, cap) {
|
|
440
|
+
if (typeof value === "string") return value.length > cap ? value.slice(0, cap) : value;
|
|
441
|
+
if (Array.isArray(value)) return value.map((item) => shrinkStrings(item, cap));
|
|
442
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, shrinkStrings(item, cap)]));
|
|
443
|
+
return value;
|
|
444
|
+
}
|
|
445
|
+
var WideEventBuilder = class {
|
|
446
|
+
requestId;
|
|
447
|
+
component;
|
|
448
|
+
fields = {};
|
|
449
|
+
constructor(requestId, component) {
|
|
450
|
+
this.requestId = requestId;
|
|
451
|
+
this.component = component;
|
|
452
|
+
}
|
|
453
|
+
set(key, value) {
|
|
454
|
+
this.fields = {
|
|
455
|
+
...this.fields,
|
|
456
|
+
[key]: value
|
|
457
|
+
};
|
|
458
|
+
return this;
|
|
459
|
+
}
|
|
460
|
+
merge(obj) {
|
|
461
|
+
const { requestId: _r, component: _c, timestamp: _t, _truncated: _x, ...rest } = obj;
|
|
462
|
+
this.fields = {
|
|
463
|
+
...this.fields,
|
|
464
|
+
...rest
|
|
465
|
+
};
|
|
466
|
+
return this;
|
|
467
|
+
}
|
|
468
|
+
flush() {
|
|
469
|
+
const event = {
|
|
470
|
+
requestId: this.requestId,
|
|
471
|
+
component: this.component,
|
|
472
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
473
|
+
...this.fields
|
|
474
|
+
};
|
|
475
|
+
return this.enforceSizeLimit(event);
|
|
476
|
+
}
|
|
477
|
+
enforceSizeLimit(event) {
|
|
478
|
+
if (serializedBytes(event) <= MAX_EVENT_BYTES) return event;
|
|
479
|
+
return {
|
|
480
|
+
...shrinkStrings(event, OVERSIZED_STRING_CAP),
|
|
481
|
+
_truncated: true
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
};
|
|
485
|
+
/** Probability of sampling a normal (non-error, non-slow) event. */
|
|
486
|
+
const NORMAL_SAMPLE_RATE = .05;
|
|
487
|
+
/**
|
|
488
|
+
* Decides whether a wide event should be emitted.
|
|
489
|
+
*
|
|
490
|
+
* Pure function over the event — the only non-determinism is Math.random()
|
|
491
|
+
* for the normal path, which tests can stub.
|
|
492
|
+
*/
|
|
493
|
+
function shouldSample(event) {
|
|
494
|
+
if (event.error != null) return true;
|
|
495
|
+
if (event.durationMs !== void 0 && event.durationMs > 1e4) return true;
|
|
496
|
+
return Math.random() < NORMAL_SAMPLE_RATE;
|
|
497
|
+
}
|
|
498
|
+
//#endregion
|
|
499
|
+
//#region src/shared/logger/redaction.ts
|
|
500
|
+
/** Replacement value for any field outside the whitelist. */
|
|
501
|
+
const REDACTED = "[REDACTED]";
|
|
502
|
+
/** Whitelisted strings longer than this are cut and suffixed. */
|
|
503
|
+
const MAX_STRING_CHARS = 1e3;
|
|
504
|
+
/** Suffix appended to truncated strings so truncation is visible in queries. */
|
|
505
|
+
const TRUNCATION_SUFFIX = "...[truncated]";
|
|
506
|
+
const ERROR_KEY = "error";
|
|
507
|
+
const MESSAGE_KEY = "message";
|
|
508
|
+
const NAME_KEY = "name";
|
|
509
|
+
const STACK_KEY = "stack";
|
|
510
|
+
const DEVELOPMENT = "development";
|
|
511
|
+
/**
|
|
512
|
+
* Top-level fields allowed to pass through. `cluster`, `dedup`, `rule`,
|
|
513
|
+
* `llm` and `notify` are schema-known subtrees whose nested values are safe.
|
|
514
|
+
*/
|
|
515
|
+
const SAFE_FIELDS = /* @__PURE__ */ new Set([
|
|
516
|
+
"requestId",
|
|
517
|
+
"correlationId",
|
|
518
|
+
"timestamp",
|
|
519
|
+
"component",
|
|
520
|
+
"version",
|
|
521
|
+
"outcome",
|
|
522
|
+
"cluster",
|
|
523
|
+
"dedup",
|
|
524
|
+
"rule",
|
|
525
|
+
"llm",
|
|
526
|
+
"notify",
|
|
527
|
+
"durationMs",
|
|
528
|
+
ERROR_KEY
|
|
529
|
+
]);
|
|
530
|
+
function truncateString(value) {
|
|
531
|
+
return value.length > 1e3 ? value.slice(0, MAX_STRING_CHARS) + TRUNCATION_SUFFIX : value;
|
|
532
|
+
}
|
|
533
|
+
function redactValue(value) {
|
|
534
|
+
if (typeof value === "string") return truncateString(value);
|
|
535
|
+
if (Array.isArray(value)) return value.map(redactValue);
|
|
536
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, redactValue(item)]));
|
|
537
|
+
return value;
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* The error section keeps only message and name; the stack is kept solely in
|
|
541
|
+
* development, where it cannot reach production log stores.
|
|
542
|
+
*/
|
|
543
|
+
function redactError(error) {
|
|
544
|
+
const safe = {};
|
|
545
|
+
if (typeof error[MESSAGE_KEY] === "string") safe[MESSAGE_KEY] = truncateString(error[MESSAGE_KEY]);
|
|
546
|
+
if (typeof error[NAME_KEY] === "string") safe[NAME_KEY] = truncateString(error[NAME_KEY]);
|
|
547
|
+
if (process.env["NODE_ENV"] === DEVELOPMENT && typeof error[STACK_KEY] === "string") safe[STACK_KEY] = truncateString(error[STACK_KEY]);
|
|
548
|
+
return safe;
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Deep-redacts an object against the wide-event whitelist.
|
|
552
|
+
*
|
|
553
|
+
* Returns a new object — the input is never mutated. Whitelisted values keep
|
|
554
|
+
* their structure with over-long strings truncated; everything else becomes
|
|
555
|
+
* [REDACTED].
|
|
556
|
+
*/
|
|
557
|
+
function redact(obj) {
|
|
558
|
+
return Object.fromEntries(Object.entries(obj).map(([key, value]) => {
|
|
559
|
+
if (!SAFE_FIELDS.has(key)) return [key, REDACTED];
|
|
560
|
+
if (key === ERROR_KEY && value !== null && typeof value === "object" && !Array.isArray(value)) return [key, redactError(value)];
|
|
561
|
+
return [key, redactValue(value)];
|
|
562
|
+
}));
|
|
563
|
+
}
|
|
564
|
+
//#endregion
|
|
565
|
+
//#region src/shared/logger/enums.ts
|
|
566
|
+
/**
|
|
567
|
+
* Pipeline component taxonomy.
|
|
568
|
+
*
|
|
569
|
+
* `component` (not `service`) distinguishes pipeline stages in wide events.
|
|
570
|
+
* `service` stays constant ("junando"); `component` tells you WHERE the event
|
|
571
|
+
* was emitted from.
|
|
572
|
+
*/
|
|
573
|
+
const Component = {
|
|
574
|
+
Webhook: "webhook",
|
|
575
|
+
Worker: "worker",
|
|
576
|
+
UseCase: "useCase",
|
|
577
|
+
Llm: "llm",
|
|
578
|
+
Notifier: "notifier",
|
|
579
|
+
Dedup: "dedup",
|
|
580
|
+
Traces: "traces",
|
|
581
|
+
Ingest: "ingest"
|
|
582
|
+
};
|
|
583
|
+
/**
|
|
584
|
+
* Terminal outcomes for a wide event across all entry points.
|
|
585
|
+
*/
|
|
586
|
+
const Outcome = {
|
|
587
|
+
Success: "success",
|
|
588
|
+
Suppressed: "suppressed",
|
|
589
|
+
Degraded: "degraded",
|
|
590
|
+
Error: "error",
|
|
591
|
+
Accepted: "accepted",
|
|
592
|
+
Empty: "empty",
|
|
593
|
+
ParseError: "parse_error"
|
|
594
|
+
};
|
|
595
|
+
//#endregion
|
|
505
596
|
//#region src/shared/logger/index.ts
|
|
506
597
|
let _root = buildLogger({});
|
|
507
598
|
function buildLogger(opts) {
|
|
@@ -558,6 +649,154 @@ function reinitLogger(opts) {
|
|
|
558
649
|
_root = buildLogger(opts ?? {});
|
|
559
650
|
}
|
|
560
651
|
//#endregion
|
|
652
|
+
//#region src/application/use-cases/process-incident.use-case.ts
|
|
653
|
+
/** Source label for dedup counter metrics. */
|
|
654
|
+
const DEDUP_METRIC_SOURCE = "alertmanager";
|
|
655
|
+
function toErrorSection(err) {
|
|
656
|
+
if (err instanceof Error) return {
|
|
657
|
+
message: err.message,
|
|
658
|
+
name: err.name,
|
|
659
|
+
...err.stack !== void 0 && { stack: err.stack }
|
|
660
|
+
};
|
|
661
|
+
return { message: String(err) };
|
|
662
|
+
}
|
|
663
|
+
/**
|
|
664
|
+
* Terminal outcome for a processed cluster. Early returns — no switch/case.
|
|
665
|
+
* Notify failure is fatal (the batch is retried via SQS); LLM failure is
|
|
666
|
+
* degraded (notification still went out without a diagnosis).
|
|
667
|
+
*/
|
|
668
|
+
function resolveOutcome({ llmError, notifyError }) {
|
|
669
|
+
if (notifyError != null) return Outcome.Error;
|
|
670
|
+
if (llmError != null) return Outcome.Degraded;
|
|
671
|
+
return Outcome.Success;
|
|
672
|
+
}
|
|
673
|
+
var ProcessIncidentUseCase = class {
|
|
674
|
+
deps;
|
|
675
|
+
clustering;
|
|
676
|
+
wideEventsEnabled;
|
|
677
|
+
constructor(deps) {
|
|
678
|
+
this.deps = deps;
|
|
679
|
+
this.clustering = deps.clustering ?? new ClusteringService();
|
|
680
|
+
this.wideEventsEnabled = process.env["WIDE_EVENTS_ENABLED"] !== "false";
|
|
681
|
+
}
|
|
682
|
+
async execute(alerts, correlationId) {
|
|
683
|
+
const { dedup, traces, llm, notifier, dedupTtlSeconds, ruleEngine } = this.deps;
|
|
684
|
+
const clusters = this.clustering.buildClusters(alerts);
|
|
685
|
+
this.deps.onClustersBuilt?.(clusters.length);
|
|
686
|
+
for (const cluster of clusters) {
|
|
687
|
+
const clusterStartMs = Date.now();
|
|
688
|
+
const builder = new WideEventBuilder(`${correlationId}:${cluster.fingerprint}`, Component.UseCase).set("correlationId", correlationId).set("cluster", {
|
|
689
|
+
fingerprint: cluster.fingerprint,
|
|
690
|
+
serviceName: cluster.serviceName,
|
|
691
|
+
alertCount: cluster.alertCount,
|
|
692
|
+
spanCount: 0
|
|
693
|
+
});
|
|
694
|
+
const dedupResult = await dedup.isNew(cluster.fingerprint, dedupTtlSeconds);
|
|
695
|
+
builder.set("dedup", {
|
|
696
|
+
isNew: dedupResult.isNew,
|
|
697
|
+
ttlSeconds: dedupResult.ttlSeconds,
|
|
698
|
+
...dedupResult.error !== void 0 && { error: dedupResult.error }
|
|
699
|
+
});
|
|
700
|
+
if (!dedupResult.isNew) {
|
|
701
|
+
dedupDuplicate.inc({ source: DEDUP_METRIC_SOURCE });
|
|
702
|
+
continue;
|
|
703
|
+
}
|
|
704
|
+
dedupNew.inc({ source: DEDUP_METRIC_SOURCE });
|
|
705
|
+
let preLlmRouteChannels = [];
|
|
706
|
+
let preLlmEscalateChannels = [];
|
|
707
|
+
if (ruleEngine) {
|
|
708
|
+
const preResult = ruleEngine.evaluatePreLlm(cluster);
|
|
709
|
+
builder.set("rule", {
|
|
710
|
+
matched: preResult.matchedRuleId != null,
|
|
711
|
+
suppressed: preResult.suppressed,
|
|
712
|
+
...preResult.matchedRuleId !== void 0 && { matchedRuleId: preResult.matchedRuleId }
|
|
713
|
+
});
|
|
714
|
+
if (preResult.suppressed) {
|
|
715
|
+
if (preResult.matchedRuleId) suppressedClusters.inc({ rule_id: preResult.matchedRuleId });
|
|
716
|
+
this.emit(builder, Outcome.Suppressed, clusterStartMs);
|
|
717
|
+
continue;
|
|
718
|
+
}
|
|
719
|
+
for (const action of preResult.actions) {
|
|
720
|
+
if (action.type === "route" && "channel" in action) preLlmRouteChannels.push(action.channel);
|
|
721
|
+
if (action.type === "escalate" && "channel" in action) preLlmEscalateChannels.push(action.channel);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
let traceErrors = 0;
|
|
725
|
+
const allSpans = (await Promise.all(cluster.representativeTraceIds.map((id) => traces.findByTraceId(id).catch(() => {
|
|
726
|
+
traceErrors++;
|
|
727
|
+
return [];
|
|
728
|
+
})))).flat();
|
|
729
|
+
builder.set("cluster", {
|
|
730
|
+
fingerprint: cluster.fingerprint,
|
|
731
|
+
serviceName: cluster.serviceName,
|
|
732
|
+
alertCount: cluster.alertCount,
|
|
733
|
+
spanCount: allSpans.length,
|
|
734
|
+
...traceErrors > 0 && { traceErrors }
|
|
735
|
+
});
|
|
736
|
+
let analysis = null;
|
|
737
|
+
let llmError = null;
|
|
738
|
+
try {
|
|
739
|
+
const llmResult = await llm.analyze(cluster, allSpans);
|
|
740
|
+
analysis = llmResult.analysis;
|
|
741
|
+
builder.set("llm", {
|
|
742
|
+
provider: llmResult.provider,
|
|
743
|
+
model: llmResult.model,
|
|
744
|
+
latencyMs: llmResult.latencyMs,
|
|
745
|
+
urgency: llmResult.analysis.urgency_level,
|
|
746
|
+
tokens: llmResult.promptTokens + llmResult.completionTokens
|
|
747
|
+
});
|
|
748
|
+
} catch (err) {
|
|
749
|
+
llmError = err;
|
|
750
|
+
}
|
|
751
|
+
let postLlmEscalateChannels = [];
|
|
752
|
+
if (ruleEngine && analysis) {
|
|
753
|
+
const postResult = ruleEngine.evaluatePostLlm(cluster, analysis);
|
|
754
|
+
for (const action of postResult.actions) if (action.type === "escalate" && "channel" in action) postLlmEscalateChannels.push(action.channel);
|
|
755
|
+
if (postResult.tags && Object.keys(postResult.tags).length > 0) cluster.labels = {
|
|
756
|
+
...cluster.labels,
|
|
757
|
+
...postResult.tags
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
const escalateChannels = [...preLlmEscalateChannels, ...postLlmEscalateChannels];
|
|
761
|
+
const notifyStartMs = Date.now();
|
|
762
|
+
try {
|
|
763
|
+
const primaryChannel = preLlmRouteChannels[0];
|
|
764
|
+
const results = [await notifier.send(cluster, analysis, primaryChannel)];
|
|
765
|
+
for (const channel of escalateChannels) results.push(await notifier.send(cluster, analysis, channel));
|
|
766
|
+
builder.set("notify", {
|
|
767
|
+
channels: results.flatMap((r) => r.channels),
|
|
768
|
+
outcome: NotifyOutcome.Success,
|
|
769
|
+
latencyMs: Date.now() - notifyStartMs
|
|
770
|
+
});
|
|
771
|
+
} catch (err) {
|
|
772
|
+
builder.set("notify", {
|
|
773
|
+
channels: [...preLlmRouteChannels, ...escalateChannels],
|
|
774
|
+
outcome: NotifyOutcome.Failure,
|
|
775
|
+
latencyMs: Date.now() - notifyStartMs
|
|
776
|
+
});
|
|
777
|
+
builder.set("error", toErrorSection(err));
|
|
778
|
+
this.emit(builder, Outcome.Error, clusterStartMs);
|
|
779
|
+
throw err;
|
|
780
|
+
}
|
|
781
|
+
if (llmError != null) builder.set("error", toErrorSection(llmError));
|
|
782
|
+
this.emit(builder, resolveOutcome({
|
|
783
|
+
llmError,
|
|
784
|
+
notifyError: null
|
|
785
|
+
}), clusterStartMs);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
/**
|
|
789
|
+
* Flushes the builder into a final event, applies tail sampling, redacts
|
|
790
|
+
* PII, and emits the single canonical log line for the cluster.
|
|
791
|
+
*/
|
|
792
|
+
emit(builder, outcome, startMs) {
|
|
793
|
+
if (!this.wideEventsEnabled) return;
|
|
794
|
+
const event = builder.set("outcome", outcome).set("durationMs", Date.now() - startMs).flush();
|
|
795
|
+
if (!shouldSample(event)) return;
|
|
796
|
+
this.deps.logger.info(redact(event));
|
|
797
|
+
}
|
|
798
|
+
};
|
|
799
|
+
//#endregion
|
|
561
800
|
//#region src/infrastructure/dedup/redis-dedup.adapter.ts
|
|
562
801
|
const logger$3 = createLogger();
|
|
563
802
|
var RedisDeduplicationStore = class {
|
|
@@ -568,14 +807,22 @@ var RedisDeduplicationStore = class {
|
|
|
568
807
|
}
|
|
569
808
|
async isNew(fingerprint, ttlSeconds) {
|
|
570
809
|
try {
|
|
571
|
-
return
|
|
810
|
+
return {
|
|
811
|
+
isNew: await this.redis.set(`${this.keyPrefix}${fingerprint}`, "1", "EX", ttlSeconds, "NX") === "OK",
|
|
812
|
+
ttlSeconds
|
|
813
|
+
};
|
|
572
814
|
} catch (err) {
|
|
815
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
573
816
|
logger$3.warn({
|
|
574
817
|
err,
|
|
575
818
|
fingerprint
|
|
576
819
|
}, "Redis dedup check failed, failing open");
|
|
577
820
|
dedupRedisFailoverTotal.inc();
|
|
578
|
-
return
|
|
821
|
+
return {
|
|
822
|
+
isNew: true,
|
|
823
|
+
ttlSeconds,
|
|
824
|
+
error: message
|
|
825
|
+
};
|
|
579
826
|
}
|
|
580
827
|
}
|
|
581
828
|
async reset(fingerprint) {
|
|
@@ -587,9 +834,15 @@ var InMemoryDeduplicationStore = class {
|
|
|
587
834
|
async isNew(fingerprint, ttlSeconds) {
|
|
588
835
|
const expiry = this.store.get(fingerprint);
|
|
589
836
|
const now = Date.now();
|
|
590
|
-
if (expiry !== void 0 && expiry > now) return
|
|
837
|
+
if (expiry !== void 0 && expiry > now) return {
|
|
838
|
+
isNew: false,
|
|
839
|
+
ttlSeconds
|
|
840
|
+
};
|
|
591
841
|
this.store.set(fingerprint, now + ttlSeconds * 1e3);
|
|
592
|
-
return
|
|
842
|
+
return {
|
|
843
|
+
isNew: true,
|
|
844
|
+
ttlSeconds
|
|
845
|
+
};
|
|
593
846
|
}
|
|
594
847
|
async reset(fingerprint) {
|
|
595
848
|
this.store.delete(fingerprint);
|
|
@@ -632,6 +885,8 @@ var InMemoryIndexer = class {
|
|
|
632
885
|
//#endregion
|
|
633
886
|
//#region src/infrastructure/llm/llm.adapter.ts
|
|
634
887
|
const logger$2 = createLogger();
|
|
888
|
+
/** Provider name reported by MockLLMProvider results. */
|
|
889
|
+
const MOCK_PROVIDER_NAME = "mock";
|
|
635
890
|
/**
|
|
636
891
|
* Schema for OpenRouter API response validation.
|
|
637
892
|
* Ensures type safety at the external boundary.
|
|
@@ -738,6 +993,15 @@ var GeminiProvider = class {
|
|
|
738
993
|
this.breaker = new Breaker.default(this.analyzeRaw.bind(this), BREAKER_OPTIONS);
|
|
739
994
|
}
|
|
740
995
|
async analyze(cluster, traces) {
|
|
996
|
+
const startMs = Date.now();
|
|
997
|
+
return {
|
|
998
|
+
...await this.analyzeWithBreaker(cluster, traces),
|
|
999
|
+
provider: "gemini",
|
|
1000
|
+
model: this.model,
|
|
1001
|
+
latencyMs: Date.now() - startMs
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
1004
|
+
async analyzeWithBreaker(cluster, traces) {
|
|
741
1005
|
try {
|
|
742
1006
|
return await this.breaker.fire(cluster, traces);
|
|
743
1007
|
} catch {
|
|
@@ -746,10 +1010,16 @@ var GeminiProvider = class {
|
|
|
746
1010
|
}
|
|
747
1011
|
async analyzeRaw(cluster, traces) {
|
|
748
1012
|
const { GoogleGenerativeAI } = await import("@google/generative-ai");
|
|
749
|
-
|
|
1013
|
+
const result = await new GoogleGenerativeAI(this.apiKey).getGenerativeModel({
|
|
750
1014
|
model: this.model,
|
|
751
1015
|
systemInstruction: SYSTEM_PROMPT
|
|
752
|
-
}).generateContent(buildUserPrompt(cluster, traces))
|
|
1016
|
+
}).generateContent(buildUserPrompt(cluster, traces));
|
|
1017
|
+
const usage = result.response.usageMetadata;
|
|
1018
|
+
return {
|
|
1019
|
+
analysis: parseAnalysis(result.response.text()),
|
|
1020
|
+
promptTokens: usage?.promptTokenCount ?? 0,
|
|
1021
|
+
completionTokens: usage?.candidatesTokenCount ?? 0
|
|
1022
|
+
};
|
|
753
1023
|
}
|
|
754
1024
|
};
|
|
755
1025
|
/**
|
|
@@ -764,16 +1034,25 @@ var ClaudeProvider = class {
|
|
|
764
1034
|
this.model = model;
|
|
765
1035
|
}
|
|
766
1036
|
async analyze(cluster, traces) {
|
|
1037
|
+
const startMs = Date.now();
|
|
767
1038
|
const Anthropic = (await import("@anthropic-ai/sdk")).default;
|
|
768
|
-
|
|
1039
|
+
const message = await new Anthropic({ apiKey: this.apiKey }).messages.create({
|
|
769
1040
|
model: this.model,
|
|
770
|
-
max_tokens:
|
|
1041
|
+
max_tokens: LLM_MAX_TOKENS,
|
|
771
1042
|
system: SYSTEM_PROMPT,
|
|
772
1043
|
messages: [{
|
|
773
1044
|
role: "user",
|
|
774
1045
|
content: buildUserPrompt(cluster, traces)
|
|
775
1046
|
}]
|
|
776
|
-
})
|
|
1047
|
+
});
|
|
1048
|
+
return {
|
|
1049
|
+
analysis: parseAnalysis(message.content.find((b) => b.type === "text")?.text ?? ""),
|
|
1050
|
+
provider: "claude",
|
|
1051
|
+
model: this.model,
|
|
1052
|
+
latencyMs: Date.now() - startMs,
|
|
1053
|
+
promptTokens: message.usage?.input_tokens ?? 0,
|
|
1054
|
+
completionTokens: message.usage?.output_tokens ?? 0
|
|
1055
|
+
};
|
|
777
1056
|
}
|
|
778
1057
|
};
|
|
779
1058
|
/**
|
|
@@ -785,11 +1064,18 @@ var MockLLMProvider = class {
|
|
|
785
1064
|
async analyze(cluster, _traces) {
|
|
786
1065
|
this.callLog.push({ cluster });
|
|
787
1066
|
return {
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
1067
|
+
analysis: {
|
|
1068
|
+
probable_cause: `Mock: ${cluster.alertType} on ${cluster.serviceName}`,
|
|
1069
|
+
impacted_services: [cluster.serviceName],
|
|
1070
|
+
recommended_steps: ["Check the logs", "Verify the deployment"],
|
|
1071
|
+
urgency_level: "high",
|
|
1072
|
+
requires_rollback: false
|
|
1073
|
+
},
|
|
1074
|
+
provider: MOCK_PROVIDER_NAME,
|
|
1075
|
+
model: MOCK_PROVIDER_NAME,
|
|
1076
|
+
latencyMs: 0,
|
|
1077
|
+
promptTokens: 0,
|
|
1078
|
+
completionTokens: 0
|
|
793
1079
|
};
|
|
794
1080
|
}
|
|
795
1081
|
};
|
|
@@ -803,11 +1089,13 @@ var OpenRouterProvider = class {
|
|
|
803
1089
|
model;
|
|
804
1090
|
fallbackModels;
|
|
805
1091
|
fallbackTimeoutMs;
|
|
806
|
-
|
|
1092
|
+
providerName;
|
|
1093
|
+
constructor(apiKey, model = LLM_MODELS.OpenRouter, fallbackModels = [], fallbackTimeoutMs = LLM_FALLBACK_DEFAULTS.TimeoutMs, providerName = "openrouter") {
|
|
807
1094
|
this.apiKey = apiKey;
|
|
808
1095
|
this.model = model;
|
|
809
1096
|
this.fallbackModels = fallbackModels.filter((m) => m !== model);
|
|
810
1097
|
this.fallbackTimeoutMs = fallbackTimeoutMs;
|
|
1098
|
+
this.providerName = providerName;
|
|
811
1099
|
}
|
|
812
1100
|
async analyze(cluster, traces, correlationId) {
|
|
813
1101
|
const prompt = buildUserPrompt(cluster, traces);
|
|
@@ -862,7 +1150,7 @@ var OpenRouterProvider = class {
|
|
|
862
1150
|
if (res.status === 429) {
|
|
863
1151
|
if (this.fallbackModels.length > 0) {
|
|
864
1152
|
const deadlineMs = Date.now() + this.fallbackTimeoutMs;
|
|
865
|
-
return this.analyzeFallback(prompt, correlationId, deadlineMs, this.model);
|
|
1153
|
+
return this.analyzeFallback(prompt, correlationId, deadlineMs, this.model, startMs);
|
|
866
1154
|
}
|
|
867
1155
|
llmInferenceTotal.inc({ status: "rate_limited" });
|
|
868
1156
|
throw new Error(`OpenRouter API failed: ${res.status}`);
|
|
@@ -876,8 +1164,9 @@ var OpenRouterProvider = class {
|
|
|
876
1164
|
correlationId
|
|
877
1165
|
}, "llm:validation:failed");
|
|
878
1166
|
const analysis = parseAnalysis(parsed.success ? parsed.data.choices?.[0]?.message?.content ?? "" : "", correlationId);
|
|
879
|
-
|
|
880
|
-
|
|
1167
|
+
const usage = parsed.success ? parsed.data.usage : void 0;
|
|
1168
|
+
if (usage) {
|
|
1169
|
+
const { prompt_tokens, completion_tokens, total_tokens } = usage;
|
|
881
1170
|
logger$2.info({
|
|
882
1171
|
model: this.model,
|
|
883
1172
|
usage: {
|
|
@@ -891,11 +1180,18 @@ var OpenRouterProvider = class {
|
|
|
891
1180
|
}
|
|
892
1181
|
llmInferenceTotal.inc({ status: "success" });
|
|
893
1182
|
llmInferenceDuration.observe({ model: this.model }, latencyMs / 1e3);
|
|
894
|
-
return
|
|
1183
|
+
return {
|
|
1184
|
+
analysis,
|
|
1185
|
+
provider: this.providerName,
|
|
1186
|
+
model: this.model,
|
|
1187
|
+
latencyMs,
|
|
1188
|
+
promptTokens: usage?.prompt_tokens ?? 0,
|
|
1189
|
+
completionTokens: usage?.completion_tokens ?? 0
|
|
1190
|
+
};
|
|
895
1191
|
}
|
|
896
1192
|
throw new Error("OpenRouter API failed after retry");
|
|
897
1193
|
}
|
|
898
|
-
async analyzeFallback(prompt, correlationId, deadlineMs, fromModel) {
|
|
1194
|
+
async analyzeFallback(prompt, correlationId, deadlineMs, fromModel, startMs) {
|
|
899
1195
|
for (const toModel of this.fallbackModels) {
|
|
900
1196
|
if (Date.now() >= deadlineMs) throw new Error("OpenRouter fallback chain timed out");
|
|
901
1197
|
logger$2.info({
|
|
@@ -932,7 +1228,16 @@ var OpenRouterProvider = class {
|
|
|
932
1228
|
throw new Error(`OpenRouter API failed: ${res.status}`);
|
|
933
1229
|
}
|
|
934
1230
|
const parsed = OpenRouterResponseSchema.safeParse(raw);
|
|
935
|
-
|
|
1231
|
+
const text = parsed.success ? parsed.data.choices?.[0]?.message?.content ?? "" : "";
|
|
1232
|
+
const usage = parsed.success ? parsed.data.usage : void 0;
|
|
1233
|
+
return {
|
|
1234
|
+
analysis: parseAnalysis(text, correlationId),
|
|
1235
|
+
provider: this.providerName,
|
|
1236
|
+
model: toModel,
|
|
1237
|
+
latencyMs: Date.now() - startMs,
|
|
1238
|
+
promptTokens: usage?.prompt_tokens ?? 0,
|
|
1239
|
+
completionTokens: usage?.completion_tokens ?? 0
|
|
1240
|
+
};
|
|
936
1241
|
}
|
|
937
1242
|
throw new Error("OpenRouter API exhausted all models");
|
|
938
1243
|
}
|
|
@@ -944,8 +1249,8 @@ var OpenRouterProvider = class {
|
|
|
944
1249
|
const LLM_PROVIDER_REGISTRY = /* @__PURE__ */ new Map([
|
|
945
1250
|
["gemini", (apiKey, model) => new GeminiProvider(apiKey, model)],
|
|
946
1251
|
["claude", (apiKey, model) => new ClaudeProvider(apiKey, model)],
|
|
947
|
-
["openrouter", (apiKey, model, options) => new OpenRouterProvider(apiKey, model, options?.fallbackModels, options?.fallbackTimeoutMs)],
|
|
948
|
-
["qwen", (apiKey, model, options) => new OpenRouterProvider(apiKey, model, options?.fallbackModels, options?.fallbackTimeoutMs)]
|
|
1252
|
+
["openrouter", (apiKey, model, options) => new OpenRouterProvider(apiKey, model, options?.fallbackModels, options?.fallbackTimeoutMs, "openrouter")],
|
|
1253
|
+
["qwen", (apiKey, model, options) => new OpenRouterProvider(apiKey, model, options?.fallbackModels, options?.fallbackTimeoutMs, "qwen")]
|
|
949
1254
|
]);
|
|
950
1255
|
function createLLMProvider(provider, apiKey, model, options) {
|
|
951
1256
|
const factory = LLM_PROVIDER_REGISTRY.get(provider);
|
|
@@ -1017,6 +1322,7 @@ var SlackNotifier = class {
|
|
|
1017
1322
|
}
|
|
1018
1323
|
async send(cluster, analysis, _channel) {
|
|
1019
1324
|
const payload = analysis ? this.buildAnalysisMessage(cluster, analysis) : this.buildFallbackMessage(cluster);
|
|
1325
|
+
const startMs = Date.now();
|
|
1020
1326
|
try {
|
|
1021
1327
|
const res = await fetch(SLACK_API_URL, {
|
|
1022
1328
|
method: "POST",
|
|
@@ -1037,6 +1343,11 @@ var SlackNotifier = class {
|
|
|
1037
1343
|
channel: "slack",
|
|
1038
1344
|
outcome: "success"
|
|
1039
1345
|
});
|
|
1346
|
+
return {
|
|
1347
|
+
outcome: NotifyOutcome.Success,
|
|
1348
|
+
latencyMs: Date.now() - startMs,
|
|
1349
|
+
channels: [this.channel]
|
|
1350
|
+
};
|
|
1040
1351
|
} catch (err) {
|
|
1041
1352
|
notificationsTotal.inc({
|
|
1042
1353
|
channel: "slack",
|
|
@@ -1155,6 +1466,7 @@ var SlackNotifier = class {
|
|
|
1155
1466
|
var ConsoleNotifier = class {
|
|
1156
1467
|
sent = [];
|
|
1157
1468
|
async send(cluster, analysis, _channel) {
|
|
1469
|
+
const startMs = Date.now();
|
|
1158
1470
|
try {
|
|
1159
1471
|
this.sent.push({
|
|
1160
1472
|
cluster,
|
|
@@ -1171,6 +1483,11 @@ var ConsoleNotifier = class {
|
|
|
1171
1483
|
channel: "unknown",
|
|
1172
1484
|
outcome: "success"
|
|
1173
1485
|
});
|
|
1486
|
+
return {
|
|
1487
|
+
outcome: NotifyOutcome.Success,
|
|
1488
|
+
latencyMs: Date.now() - startMs,
|
|
1489
|
+
channels: ["console"]
|
|
1490
|
+
};
|
|
1174
1491
|
} catch (err) {
|
|
1175
1492
|
notificationsTotal.inc({
|
|
1176
1493
|
channel: "unknown",
|
|
@@ -1321,6 +1638,7 @@ var TeamsNotifier = class {
|
|
|
1321
1638
|
const payload = buildAdaptiveCardPayload(analysis ? buildAnalysisCard(cluster, analysis) : buildFallbackCard(cluster));
|
|
1322
1639
|
const controller = new AbortController();
|
|
1323
1640
|
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
1641
|
+
const startMs = Date.now();
|
|
1324
1642
|
try {
|
|
1325
1643
|
const res = await fetch(this.webhookUrl, {
|
|
1326
1644
|
method: "POST",
|
|
@@ -1333,6 +1651,11 @@ var TeamsNotifier = class {
|
|
|
1333
1651
|
channel: "teams",
|
|
1334
1652
|
outcome: "success"
|
|
1335
1653
|
});
|
|
1654
|
+
return {
|
|
1655
|
+
outcome: NotifyOutcome.Success,
|
|
1656
|
+
latencyMs: Date.now() - startMs,
|
|
1657
|
+
channels: ["teams"]
|
|
1658
|
+
};
|
|
1336
1659
|
} catch (err) {
|
|
1337
1660
|
if (err instanceof TeamsNotifierError) {
|
|
1338
1661
|
notificationsTotal.inc({
|
|
@@ -1391,7 +1714,7 @@ var RoutingNotifier = class {
|
|
|
1391
1714
|
* Backward-compatible with existing call sites that don't use rule actions.
|
|
1392
1715
|
*/
|
|
1393
1716
|
async send(cluster, analysis) {
|
|
1394
|
-
|
|
1717
|
+
return this.defaultNotifier.send(cluster, analysis);
|
|
1395
1718
|
}
|
|
1396
1719
|
/**
|
|
1397
1720
|
* Dispatch notifications based on rule engine actions.
|
|
@@ -1963,6 +2286,6 @@ async function loadConfig() {
|
|
|
1963
2286
|
return result.data;
|
|
1964
2287
|
}
|
|
1965
2288
|
//#endregion
|
|
1966
|
-
export { ALERT_TYPE_LABELS, AlertClusterSchema, AlertStatusSchema, AlertType, AlertmanagerPayloadSchema, CIRCUIT_BREAKER, ChannelRegistry, ClaudeProvider, ClusteringService, ConsoleNotifier, DEDUP_TTL_MS_MULTIPLIER, DEV_SERVER_PORT, FactoryRegistry, Fingerprint, GeminiProvider, HOUR_MS, HTTP_TIMEOUT_MS, InMemoryAlertQueue, InMemoryDeduplicationStore, InMemoryIndexer, IncidentSchema, LLMAnalysisSchema, LLMProviderType, LLM_FALLBACK_DEFAULTS, LLM_MAX_TOKENS, LLM_MODELS, LokiTraceRepository, MockLLMProvider, MockTraceRepository, NormalizedAlertSchema, OpenSearchIndexer, PAYLOAD_DEFAULTS, ProcessIncidentUseCase, RATE_LIMITER, REDIS_KEY_PREFIX, RedisDeduplicationStore, RoutingNotifier, RuleActionSchema, RuleActionType, RuleConditionSchema, RuleConfigurationSchema, RuleEngine, RuleEvaluationPhase, RuleSchema, RuleSectionSchema, SLACK_API_URL, SQSAlertQueue, SeverityLevel, SlackNotifier, TEAMS_WEBHOOK_TIMEOUT_MS, TeamsNotifier, TeamsNotifierError, URGENCY_EMOJI, UrgencyLevelSchema, WEBHOOK_DEFAULTS, compileCondition, createLLMProvider, createLogger, createNotifier, dispatchActions, flushLoki, loadConfig, metrics_exports as metrics, normalizePayload, parseRuleConfig, reinitLogger, startSqsLagPoller };
|
|
2289
|
+
export { ALERT_TYPE_LABELS, AlertClusterSchema, AlertStatusSchema, AlertType, AlertmanagerPayloadSchema, CIRCUIT_BREAKER, ChannelRegistry, ClaudeProvider, ClusteringService, ConsoleNotifier, DEDUP_TTL_MS_MULTIPLIER, DEV_SERVER_PORT, FactoryRegistry, Fingerprint, GeminiProvider, HOUR_MS, HTTP_TIMEOUT_MS, InMemoryAlertQueue, InMemoryDeduplicationStore, InMemoryIndexer, IncidentSchema, LLMAnalysisSchema, LLMProviderType, LLM_FALLBACK_DEFAULTS, LLM_MAX_TOKENS, LLM_MODELS, LokiTraceRepository, MockLLMProvider, MockTraceRepository, NormalizedAlertSchema, NotifyOutcome, OpenSearchIndexer, PAYLOAD_DEFAULTS, ProcessIncidentUseCase, RATE_LIMITER, REDIS_KEY_PREFIX, RedisDeduplicationStore, RoutingNotifier, RuleActionSchema, RuleActionType, RuleConditionSchema, RuleConfigurationSchema, RuleEngine, RuleEvaluationPhase, RuleSchema, RuleSectionSchema, SLACK_API_URL, SQSAlertQueue, SeverityLevel, SlackNotifier, TEAMS_WEBHOOK_TIMEOUT_MS, TeamsNotifier, TeamsNotifierError, URGENCY_EMOJI, UrgencyLevelSchema, WEBHOOK_DEFAULTS, compileCondition, createLLMProvider, createLogger, createNotifier, dispatchActions, flushLoki, loadConfig, metrics_exports as metrics, normalizePayload, parseRuleConfig, reinitLogger, startSqsLagPoller };
|
|
1967
2290
|
|
|
1968
2291
|
//# sourceMappingURL=index.js.map
|