@obsunified/collector 2.3.0 → 2.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/auth.d.ts +2 -1
- package/dist/auth.js +1 -1
- package/dist/{chunk-2FALTDCE.js → chunk-VSOZJO6Y.js} +12 -1
- package/dist/chunk-VSOZJO6Y.js.map +1 -0
- package/dist/index.d.ts +271 -62
- package/dist/index.js +1077 -137
- package/dist/index.js.map +1 -1
- package/dist/{ingest-auth-BmP5-f96.d.ts → ingest-auth-BTowieER.d.ts} +66 -2
- package/package.json +3 -3
- package/src/framework/collector.ts +22 -0
- package/src/framework/env.ts +9 -1
- package/src/framework/ingest-auth-warning.test.ts +71 -0
- package/src/index.ts +15 -0
- package/src/lib/alert-email.ts +104 -0
- package/src/lib/alerts-store.test.ts +147 -2
- package/src/lib/alerts-store.ts +438 -63
- package/src/lib/metric-alerts.test.ts +769 -0
- package/src/lib/metric-alerts.ts +640 -0
- package/src/lib/metrics-store.ts +9 -2
- package/src/lib/projects-store.test.ts +44 -1
- package/src/lib/projects-store.ts +21 -0
- package/src/lib/sql-db-postgres.integration.test.ts +247 -0
- package/src/lib/store/trace-queries.ts +1 -1
- package/src/lib/webhook-url.test.ts +49 -0
- package/src/lib/webhook-url.ts +72 -14
- package/src/migrations/054_alert_rules_pending_secs.sql +11 -0
- package/src/migrations/055_alert_state_pending_since.sql +12 -0
- package/src/migrations/056_metric_point_series_received.sql +9 -0
- package/src/migrations/057_alert_evaluations_outcome.sql +11 -0
- package/src/migrations/058_alert_state_pending_seen_at.sql +12 -0
- package/src/migrations/059_alert_evaluations_error.sql +11 -0
- package/src/migrations-postgres/054_alert_rules_pending_secs.sql +7 -0
- package/src/migrations-postgres/055_alert_state_pending_since.sql +6 -0
- package/src/migrations-postgres/056_metric_point_series_received.sql +7 -0
- package/src/migrations-postgres/057_alert_evaluations_outcome.sql +6 -0
- package/src/migrations-postgres/058_alert_state_pending_seen_at.sql +6 -0
- package/src/migrations-postgres/059_alert_evaluations_error.sql +6 -0
- package/src/migrations-postgres/README.md +13 -0
- package/src/otlp/decode/logs.ts +30 -2
- package/src/otlp/decode/metrics.ts +67 -2
- package/src/otlp/decode/traces.ts +57 -4
- package/src/otlp/decode/values.ts +64 -10
- package/src/otlp/decode.test.ts +305 -0
- package/src/otlp/decode.ts +9 -3
- package/src/plugins/alerts-evaluator-pending.test.ts +322 -0
- package/src/plugins/alerts-evaluator-retention.test.ts +2 -0
- package/src/plugins/alerts-evaluator.test.ts +10 -2
- package/src/plugins/alerts-evaluator.ts +239 -40
- package/src/plugins/alerts-routes.ts +17 -2
- package/src/plugins/autosre-routes.ts +9 -0
- package/src/plugins/logs-receiver.ts +8 -8
- package/src/plugins/metrics-receiver.ts +10 -9
- package/src/plugins/otlp-receiver.ts +10 -26
- package/src/plugins/profile-routes.test.ts +22 -0
- package/src/plugins/profile-routes.ts +11 -2
- package/src/plugins/replay-query-routes.test.ts +63 -1
- package/src/plugins/replay-query-routes.ts +40 -9
- package/dist/chunk-2FALTDCE.js.map +0 -1
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
AlertChannel,
|
|
3
|
+
AlertEmailChannel,
|
|
4
|
+
AlertEvaluationOutcome,
|
|
5
|
+
AlertQueryMetrics,
|
|
3
6
|
AlertRule,
|
|
4
7
|
AlertState,
|
|
5
8
|
AlertWebhookChannel,
|
|
@@ -10,7 +13,16 @@ import {
|
|
|
10
13
|
consoleLogger,
|
|
11
14
|
type Logger,
|
|
12
15
|
} from "../framework/logger";
|
|
13
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
type AlertEmailTransport,
|
|
18
|
+
type AlertNotificationPayload,
|
|
19
|
+
formatAlertEmail,
|
|
20
|
+
} from "../lib/alert-email";
|
|
21
|
+
import {
|
|
22
|
+
type AlertRuleEvaluation,
|
|
23
|
+
AlertsStore,
|
|
24
|
+
compareValue,
|
|
25
|
+
} from "../lib/alerts-store";
|
|
14
26
|
import {
|
|
15
27
|
openOrDeduplicateIncident,
|
|
16
28
|
resolveServiceIncidentTarget,
|
|
@@ -29,6 +41,8 @@ const DEFAULT_RULE_TIMEOUT_MS = 10_000;
|
|
|
29
41
|
interface AlertEvaluatorStore {
|
|
30
42
|
listEnabledRules(): Promise<AlertRule[]>;
|
|
31
43
|
evaluateRule(rule: AlertRule): Promise<number>;
|
|
44
|
+
/** Preferred when present: also says whether a metrics rule had no data. */
|
|
45
|
+
evaluateRuleWithStatus?(rule: AlertRule): Promise<AlertRuleEvaluation>;
|
|
32
46
|
getState(
|
|
33
47
|
ruleId: string,
|
|
34
48
|
projectId: string,
|
|
@@ -37,7 +51,18 @@ interface AlertEvaluatorStore {
|
|
|
37
51
|
last_state_change: string;
|
|
38
52
|
/** 0 while the transition's notification is owed; absent pre-049 (= delivered). */
|
|
39
53
|
notified?: number;
|
|
54
|
+
/** Start of the current pending period (migration 055); null/absent when not pending. */
|
|
55
|
+
pending_since?: string | null;
|
|
56
|
+
/** Last evaluation that saw the pending condition hold (migration 058). */
|
|
57
|
+
pending_seen_at?: string | null;
|
|
40
58
|
} | null>;
|
|
59
|
+
/** Record (`{ since, seenAt }`) or clear (null) the rule's pending period. */
|
|
60
|
+
setPending(
|
|
61
|
+
ruleId: string,
|
|
62
|
+
projectId: string,
|
|
63
|
+
pending: { since: string; seenAt: string } | null,
|
|
64
|
+
ruleCreatedAt: string,
|
|
65
|
+
): Promise<void>;
|
|
41
66
|
/** Flip the exact transition `(state, lastStateChange)` to delivered. */
|
|
42
67
|
markNotified(
|
|
43
68
|
ruleId: string,
|
|
@@ -61,34 +86,19 @@ interface AlertEvaluatorStore {
|
|
|
61
86
|
value: number,
|
|
62
87
|
state: AlertState,
|
|
63
88
|
notified: boolean,
|
|
89
|
+
outcome?: AlertEvaluationOutcome,
|
|
90
|
+
/** With outcome `error`: why (the store sanitizes it). */
|
|
91
|
+
errorMessage?: string,
|
|
64
92
|
): Promise<void>;
|
|
65
93
|
}
|
|
66
94
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
threshold: number;
|
|
75
|
-
comparison: string;
|
|
76
|
-
state: AlertState;
|
|
77
|
-
evaluatedAt: string;
|
|
78
|
-
projectId: string;
|
|
79
|
-
/**
|
|
80
|
-
* RFC 0002 Stage 6 — populated when the rule is bound to an Analysis
|
|
81
|
-
* (`rule.analysisId` set). The narrative is the gated, LLM-generated
|
|
82
|
-
* sentence the dashboard panel was last showing — what's actually
|
|
83
|
-
* happening, in plain English. Webhooks consuming this can put the
|
|
84
|
-
* narrative in the Slack/PagerDuty body directly.
|
|
85
|
-
*/
|
|
86
|
-
analysis?: {
|
|
87
|
-
id: string;
|
|
88
|
-
narrative: string | null;
|
|
89
|
-
status: string | null;
|
|
90
|
-
};
|
|
91
|
-
}
|
|
95
|
+
/**
|
|
96
|
+
* Default `maxEvaluationGapMs`: two of the default 5-minute evaluation
|
|
97
|
+
* ticks. Hosts with another cadence pass their own (twice the interval).
|
|
98
|
+
*/
|
|
99
|
+
export const DEFAULT_MAX_EVALUATION_GAP_MS = 2 * 5 * 60_000;
|
|
100
|
+
|
|
101
|
+
type WebhookPayload = AlertNotificationPayload;
|
|
92
102
|
|
|
93
103
|
export interface WebhookDeliveryOptions {
|
|
94
104
|
/** Permit targets on loopback / private / link-local hosts. */
|
|
@@ -99,6 +109,51 @@ export interface WebhookDeliveryOptions {
|
|
|
99
109
|
* reach private networks from `fetch` and leave it unset.
|
|
100
110
|
*/
|
|
101
111
|
hostLookup?: HostLookup;
|
|
112
|
+
/**
|
|
113
|
+
* SMTP transport for email channels (`env.ALERT_EMAIL`). Unset on hosts
|
|
114
|
+
* that cannot send mail: an email channel then fails delivery (logged,
|
|
115
|
+
* retried next tick) instead of being skipped.
|
|
116
|
+
*/
|
|
117
|
+
email?: AlertEmailTransport;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async function fireEmail(
|
|
121
|
+
channel: AlertEmailChannel,
|
|
122
|
+
payload: WebhookPayload,
|
|
123
|
+
logger: Logger,
|
|
124
|
+
tracer?: ChildSpanRunner,
|
|
125
|
+
delivery: WebhookDeliveryOptions = {},
|
|
126
|
+
): Promise<boolean> {
|
|
127
|
+
const transport = delivery.email;
|
|
128
|
+
if (!transport) {
|
|
129
|
+
logger.error(
|
|
130
|
+
"[alerts-evaluator] email channel has no SMTP transport on this collector",
|
|
131
|
+
{ rule_id: payload.rule.id, recipients: channel.to.length },
|
|
132
|
+
);
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
const exec = async (
|
|
136
|
+
span: { setAttribute(k: string, v: unknown): void } | null,
|
|
137
|
+
): Promise<boolean> => {
|
|
138
|
+
try {
|
|
139
|
+
await transport.send({ to: channel.to, ...formatAlertEmail(payload) });
|
|
140
|
+
if (span) span.setAttribute("alert.email.sent", true);
|
|
141
|
+
return true;
|
|
142
|
+
} catch (err) {
|
|
143
|
+
if (span) span.setAttribute("alert.email.sent", false);
|
|
144
|
+
logger.error("[alerts-evaluator] email delivery failed", {
|
|
145
|
+
rule_id: payload.rule.id,
|
|
146
|
+
recipients: channel.to.length,
|
|
147
|
+
error: err instanceof Error ? err.message : String(err),
|
|
148
|
+
});
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
};
|
|
152
|
+
if (!tracer) return exec(null);
|
|
153
|
+
return tracer("email.alert", async (span) => exec(span), {
|
|
154
|
+
"alert.state": payload.state,
|
|
155
|
+
"alert.email.recipients": channel.to.length,
|
|
156
|
+
});
|
|
102
157
|
}
|
|
103
158
|
|
|
104
159
|
async function fireWebhook(
|
|
@@ -191,9 +246,23 @@ async function fireChannels(
|
|
|
191
246
|
): Promise<boolean> {
|
|
192
247
|
if (channels.length === 0) return true;
|
|
193
248
|
const results = await Promise.all(
|
|
194
|
-
channels
|
|
195
|
-
|
|
196
|
-
|
|
249
|
+
channels.map((ch) => {
|
|
250
|
+
if (ch.type === "webhook")
|
|
251
|
+
return fireWebhook(
|
|
252
|
+
ch as AlertWebhookChannel,
|
|
253
|
+
payload,
|
|
254
|
+
logger,
|
|
255
|
+
tracer,
|
|
256
|
+
delivery,
|
|
257
|
+
);
|
|
258
|
+
if (ch.type === "email")
|
|
259
|
+
return fireEmail(ch, payload, logger, tracer, delivery);
|
|
260
|
+
logger.error("[alerts-evaluator] unknown channel type", {
|
|
261
|
+
rule_id: payload.rule.id,
|
|
262
|
+
type: String((ch as { type?: unknown }).type),
|
|
263
|
+
});
|
|
264
|
+
return Promise.resolve(false);
|
|
265
|
+
}),
|
|
197
266
|
);
|
|
198
267
|
return results.every(Boolean);
|
|
199
268
|
}
|
|
@@ -235,6 +304,12 @@ async function mapWithConcurrency<T, R>(
|
|
|
235
304
|
return results;
|
|
236
305
|
}
|
|
237
306
|
|
|
307
|
+
/** Pending-hold settings for one batch (see `evaluateRuleBatch`). */
|
|
308
|
+
interface HoldOptions {
|
|
309
|
+
maxEvaluationGapMs: number;
|
|
310
|
+
holdStartedAt?: string;
|
|
311
|
+
}
|
|
312
|
+
|
|
238
313
|
async function evaluateOneRule(
|
|
239
314
|
store: AlertEvaluatorStore,
|
|
240
315
|
rule: AlertRule,
|
|
@@ -242,20 +317,75 @@ async function evaluateOneRule(
|
|
|
242
317
|
tracer: ChildSpanRunner | undefined,
|
|
243
318
|
ruleTimeoutMs: number,
|
|
244
319
|
delivery: WebhookDeliveryOptions,
|
|
320
|
+
hold: HoldOptions,
|
|
245
321
|
onAlertFiring?: (rule: AlertRule) => Promise<void>,
|
|
246
322
|
): Promise<{ fired: number; resolved: number; redelivered: number }> {
|
|
247
323
|
const none = { fired: 0, resolved: 0, redelivered: 0 };
|
|
324
|
+
let previousForError: AlertState | null = null;
|
|
248
325
|
try {
|
|
249
|
-
const
|
|
250
|
-
|
|
326
|
+
const priorState = await store.getState(rule.id, rule.projectId);
|
|
327
|
+
const previous: AlertState = priorState?.current_state ?? "ok";
|
|
328
|
+
previousForError = previous;
|
|
329
|
+
const raw = await withTimeout<number | AlertRuleEvaluation>(
|
|
330
|
+
store.evaluateRuleWithStatus
|
|
331
|
+
? store.evaluateRuleWithStatus(rule)
|
|
332
|
+
: store.evaluateRule(rule),
|
|
251
333
|
ruleTimeoutMs,
|
|
252
334
|
`rule evaluation timed out after ${ruleTimeoutMs}ms`,
|
|
253
335
|
);
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
const previous: AlertState = priorState?.current_state ?? "ok";
|
|
257
|
-
const next: AlertState = shouldFire ? "firing" : "ok";
|
|
336
|
+
const { value, noData } =
|
|
337
|
+
typeof raw === "number" ? { value: raw, noData: false } : raw;
|
|
258
338
|
const now = new Date().toISOString();
|
|
339
|
+
const nowMs = Date.parse(now);
|
|
340
|
+
const keepOnNoData =
|
|
341
|
+
noData &&
|
|
342
|
+
rule.signal === "metrics" &&
|
|
343
|
+
(rule.query as AlertQueryMetrics).noData === "keep";
|
|
344
|
+
const matches =
|
|
345
|
+
!keepOnNoData && compareValue(value, rule.threshold, rule.comparison);
|
|
346
|
+
|
|
347
|
+
// Pending duration (Prometheus `for`): a rule that is not firing only
|
|
348
|
+
// fires once its condition has held on consecutive evaluations for
|
|
349
|
+
// `pendingSecs`. alert_state keeps when the hold started
|
|
350
|
+
// (pending_since) and when it was last seen (pending_seen_at). The
|
|
351
|
+
// hold is broken by an evaluation that does not match, by a gap
|
|
352
|
+
// between evaluations longer than `maxEvaluationGapMs` (missed ticks,
|
|
353
|
+
// a stopped collector), by a hold last seen before this collector
|
|
354
|
+
// process started (`holdStartedAt`), and by editing the rule.
|
|
355
|
+
// Resolution stays immediate.
|
|
356
|
+
const pendingSecs = rule.pendingSecs ?? 0;
|
|
357
|
+
const pendingSince = priorState?.pending_since ?? null;
|
|
358
|
+
const seenAt = priorState?.pending_seen_at ?? pendingSince;
|
|
359
|
+
const holdBroken =
|
|
360
|
+
pendingSince !== null &&
|
|
361
|
+
(seenAt === null ||
|
|
362
|
+
nowMs - Date.parse(seenAt) > hold.maxEvaluationGapMs ||
|
|
363
|
+
(hold.holdStartedAt !== undefined && seenAt < hold.holdStartedAt));
|
|
364
|
+
let pendingActive = pendingSince !== null;
|
|
365
|
+
let shouldFire = matches;
|
|
366
|
+
if (matches && previous === "ok" && pendingSecs > 0) {
|
|
367
|
+
const since = holdBroken || pendingSince === null ? now : pendingSince;
|
|
368
|
+
await store.setPending(
|
|
369
|
+
rule.id,
|
|
370
|
+
rule.projectId,
|
|
371
|
+
{ since, seenAt: now },
|
|
372
|
+
rule.createdAt,
|
|
373
|
+
);
|
|
374
|
+
pendingActive = true;
|
|
375
|
+
shouldFire = nowMs - Date.parse(since) >= pendingSecs * 1000;
|
|
376
|
+
}
|
|
377
|
+
const next: AlertState = keepOnNoData
|
|
378
|
+
? previous
|
|
379
|
+
: shouldFire
|
|
380
|
+
? "firing"
|
|
381
|
+
: "ok";
|
|
382
|
+
// Cleared once the hold ended or turned into a firing transition.
|
|
383
|
+
const clearPending = async () => {
|
|
384
|
+
if (pendingActive && (!matches || next === "firing")) {
|
|
385
|
+
await store.setPending(rule.id, rule.projectId, null, rule.createdAt);
|
|
386
|
+
}
|
|
387
|
+
};
|
|
388
|
+
const outcome: AlertEvaluationOutcome = keepOnNoData ? "no_data" : "value";
|
|
259
389
|
|
|
260
390
|
// Stage 6: when the rule is bound to an analysis, pull the
|
|
261
391
|
// latest narrative + status so the webhook payload describes
|
|
@@ -301,6 +431,7 @@ async function evaluateOneRule(
|
|
|
301
431
|
// transition; otherwise the same-state branch below retries on the
|
|
302
432
|
// next tick without a second transition.
|
|
303
433
|
await store.transitionState(rule.id, rule.projectId, next, now);
|
|
434
|
+
await clearPending();
|
|
304
435
|
if (next === "firing" && onAlertFiring) {
|
|
305
436
|
await onAlertFiring(rule).catch(() => undefined);
|
|
306
437
|
}
|
|
@@ -311,7 +442,14 @@ async function evaluateOneRule(
|
|
|
311
442
|
if (ok) {
|
|
312
443
|
await store.markNotified(rule.id, rule.projectId, next, now);
|
|
313
444
|
}
|
|
314
|
-
await store.recordEvaluation(
|
|
445
|
+
await store.recordEvaluation(
|
|
446
|
+
rule.id,
|
|
447
|
+
rule.projectId,
|
|
448
|
+
value,
|
|
449
|
+
next,
|
|
450
|
+
ok,
|
|
451
|
+
outcome,
|
|
452
|
+
);
|
|
315
453
|
if (!ok) {
|
|
316
454
|
logger.warn(
|
|
317
455
|
"[alerts-evaluator] state recorded but delivery failed; will retry",
|
|
@@ -323,9 +461,13 @@ async function evaluateOneRule(
|
|
|
323
461
|
: { fired: 0, resolved: 1, redelivered: 0 };
|
|
324
462
|
}
|
|
325
463
|
|
|
464
|
+
await clearPending();
|
|
465
|
+
|
|
326
466
|
// Same state. While the transition into it is still marked undelivered,
|
|
327
467
|
// retry the notification; the state itself is already correct. A row
|
|
328
468
|
// without the flag (schema before migration 049) counts as delivered.
|
|
469
|
+
// This also runs on a no-data tick: the owed notice is about a past
|
|
470
|
+
// transition.
|
|
329
471
|
const owed = priorState !== null && priorState.notified === 0;
|
|
330
472
|
if (owed) {
|
|
331
473
|
const ok = await deliver(previous);
|
|
@@ -337,7 +479,14 @@ async function evaluateOneRule(
|
|
|
337
479
|
priorState.last_state_change,
|
|
338
480
|
);
|
|
339
481
|
}
|
|
340
|
-
await store.recordEvaluation(
|
|
482
|
+
await store.recordEvaluation(
|
|
483
|
+
rule.id,
|
|
484
|
+
rule.projectId,
|
|
485
|
+
value,
|
|
486
|
+
next,
|
|
487
|
+
ok,
|
|
488
|
+
outcome,
|
|
489
|
+
);
|
|
341
490
|
if (!ok) {
|
|
342
491
|
logger.warn("[alerts-evaluator] delivery retry failed; will retry", {
|
|
343
492
|
rule_id: rule.id,
|
|
@@ -350,13 +499,36 @@ async function evaluateOneRule(
|
|
|
350
499
|
}
|
|
351
500
|
|
|
352
501
|
// Same state, already notified — write an evaluation, do not fire.
|
|
353
|
-
await store.recordEvaluation(
|
|
502
|
+
await store.recordEvaluation(
|
|
503
|
+
rule.id,
|
|
504
|
+
rule.projectId,
|
|
505
|
+
value,
|
|
506
|
+
next,
|
|
507
|
+
false,
|
|
508
|
+
outcome,
|
|
509
|
+
);
|
|
354
510
|
} catch (err) {
|
|
511
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
355
512
|
logger.error("[alerts-evaluator] rule evaluation failed", {
|
|
356
513
|
rule_id: rule.id,
|
|
357
514
|
project_id: rule.projectId,
|
|
358
|
-
error:
|
|
515
|
+
error: message,
|
|
359
516
|
});
|
|
517
|
+
// Leave a trace in the rule's history: an `error` evaluation in the
|
|
518
|
+
// unchanged state. Best effort; the database may be what failed.
|
|
519
|
+
if (previousForError !== null) {
|
|
520
|
+
await store
|
|
521
|
+
.recordEvaluation(
|
|
522
|
+
rule.id,
|
|
523
|
+
rule.projectId,
|
|
524
|
+
0,
|
|
525
|
+
previousForError,
|
|
526
|
+
false,
|
|
527
|
+
"error",
|
|
528
|
+
message,
|
|
529
|
+
)
|
|
530
|
+
.catch(() => undefined);
|
|
531
|
+
}
|
|
360
532
|
}
|
|
361
533
|
|
|
362
534
|
return none;
|
|
@@ -372,7 +544,20 @@ export async function evaluateRuleBatch(
|
|
|
372
544
|
ruleTimeoutMs?: number;
|
|
373
545
|
allowPrivateWebhookTargets?: boolean;
|
|
374
546
|
hostLookup?: HostLookup;
|
|
547
|
+
email?: AlertEmailTransport;
|
|
375
548
|
onAlertFiring?: (rule: AlertRule) => Promise<void>;
|
|
549
|
+
/**
|
|
550
|
+
* Longest gap between two evaluations that still continues a pending
|
|
551
|
+
* hold. Default `DEFAULT_MAX_EVALUATION_GAP_MS` (two 5-minute ticks);
|
|
552
|
+
* pass twice the host's evaluation interval.
|
|
553
|
+
*/
|
|
554
|
+
maxEvaluationGapMs?: number;
|
|
555
|
+
/**
|
|
556
|
+
* ISO start of this collector process. A hold last seen before it
|
|
557
|
+
* starts over, as Prometheus resets `for` on restart. Unset on hosts
|
|
558
|
+
* without a long-lived process (Workers), where the gap rule applies.
|
|
559
|
+
*/
|
|
560
|
+
holdStartedAt?: string;
|
|
376
561
|
},
|
|
377
562
|
): Promise<{
|
|
378
563
|
evaluated: number;
|
|
@@ -385,6 +570,7 @@ export async function evaluateRuleBatch(
|
|
|
385
570
|
const delivery: WebhookDeliveryOptions = {
|
|
386
571
|
allowPrivateWebhookTargets: options?.allowPrivateWebhookTargets,
|
|
387
572
|
hostLookup: options?.hostLookup,
|
|
573
|
+
email: options?.email,
|
|
388
574
|
};
|
|
389
575
|
const concurrency = Math.max(
|
|
390
576
|
1,
|
|
@@ -402,6 +588,11 @@ export async function evaluateRuleBatch(
|
|
|
402
588
|
options?.tracer,
|
|
403
589
|
ruleTimeoutMs,
|
|
404
590
|
delivery,
|
|
591
|
+
{
|
|
592
|
+
maxEvaluationGapMs:
|
|
593
|
+
options?.maxEvaluationGapMs ?? DEFAULT_MAX_EVALUATION_GAP_MS,
|
|
594
|
+
holdStartedAt: options?.holdStartedAt,
|
|
595
|
+
},
|
|
405
596
|
options?.onAlertFiring,
|
|
406
597
|
),
|
|
407
598
|
);
|
|
@@ -421,6 +612,8 @@ export async function evaluateAllRules(
|
|
|
421
612
|
concurrency?: number;
|
|
422
613
|
ruleTimeoutMs?: number;
|
|
423
614
|
hostLookup?: HostLookup;
|
|
615
|
+
maxEvaluationGapMs?: number;
|
|
616
|
+
holdStartedAt?: string;
|
|
424
617
|
},
|
|
425
618
|
): Promise<{
|
|
426
619
|
evaluated: number;
|
|
@@ -432,7 +625,10 @@ export async function evaluateAllRules(
|
|
|
432
625
|
const tracer = options?.tracer;
|
|
433
626
|
const allowPrivateWebhookTargets = env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true";
|
|
434
627
|
const db = sqlDbFor(env);
|
|
435
|
-
const store = new AlertsStore(db, {
|
|
628
|
+
const store = new AlertsStore(db, {
|
|
629
|
+
allowPrivateWebhookTargets,
|
|
630
|
+
emailDeliveryAvailable: Boolean(env.ALERT_EMAIL),
|
|
631
|
+
});
|
|
436
632
|
const rules = await store.listEnabledRules();
|
|
437
633
|
return evaluateRuleBatch(store, rules, {
|
|
438
634
|
logger,
|
|
@@ -441,6 +637,9 @@ export async function evaluateAllRules(
|
|
|
441
637
|
ruleTimeoutMs: options?.ruleTimeoutMs,
|
|
442
638
|
allowPrivateWebhookTargets,
|
|
443
639
|
hostLookup: options?.hostLookup,
|
|
640
|
+
email: env.ALERT_EMAIL,
|
|
641
|
+
maxEvaluationGapMs: options?.maxEvaluationGapMs,
|
|
642
|
+
holdStartedAt: options?.holdStartedAt,
|
|
444
643
|
onAlertFiring: async (rule) => {
|
|
445
644
|
// Target the rule's service when it names one (environment taken from
|
|
446
645
|
// that service's latest spans); otherwise fall back to the rule itself.
|
|
@@ -6,8 +6,12 @@ import { alertEvidenceReferences } from "../lib/evidence-references";
|
|
|
6
6
|
import { sqlDbFor } from "../lib/sql-db";
|
|
7
7
|
import { getProjectId } from "./_context";
|
|
8
8
|
|
|
9
|
-
const storeOptions = (env: {
|
|
9
|
+
const storeOptions = (env: {
|
|
10
|
+
ALERT_WEBHOOK_ALLOW_PRIVATE?: string;
|
|
11
|
+
ALERT_EMAIL?: unknown;
|
|
12
|
+
}) => ({
|
|
10
13
|
allowPrivateWebhookTargets: env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
|
|
14
|
+
emailDeliveryAvailable: Boolean(env.ALERT_EMAIL),
|
|
11
15
|
});
|
|
12
16
|
|
|
13
17
|
export const alertsRoutesPlugin: CollectorPlugin = {
|
|
@@ -90,9 +94,20 @@ export const alertsRoutesPlugin: CollectorPlugin = {
|
|
|
90
94
|
const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
|
|
91
95
|
const rule = await store.getRule(id, projectId);
|
|
92
96
|
if (!rule) return c.json({ error: "Rule not found" }, 404);
|
|
93
|
-
|
|
97
|
+
let value: number;
|
|
98
|
+
let noData: boolean;
|
|
99
|
+
try {
|
|
100
|
+
({ value, noData } = await store.evaluateRuleWithStatus(rule));
|
|
101
|
+
} catch (err) {
|
|
102
|
+
// e.g. a metrics query over its series or point cap.
|
|
103
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
104
|
+
return c.json({ error: message }, 422);
|
|
105
|
+
}
|
|
94
106
|
const response: AlertTestResponse = {
|
|
95
107
|
value,
|
|
108
|
+
noData,
|
|
109
|
+
// The bare comparison: a rule with `pendingSecs` additionally
|
|
110
|
+
// needs the condition to hold that long before it fires.
|
|
96
111
|
wouldFire: compareValue(value, rule.threshold, rule.comparison),
|
|
97
112
|
comparison: rule.comparison,
|
|
98
113
|
threshold: rule.threshold,
|
|
@@ -104,6 +104,15 @@ const isAgentAutonomyLevel = (value: unknown): value is AgentAutonomyLevel =>
|
|
|
104
104
|
* Enforces Review Item #4: POST /internal/remediations/:id/approve and /reject
|
|
105
105
|
* require an authenticated human dashboard session (`obs_session` cookie) and
|
|
106
106
|
* reject programmatic agent/MCP bearer tokens (`403 Forbidden`).
|
|
107
|
+
*
|
|
108
|
+
* Trust model: this only checks that an `obs_session` cookie is *present*.
|
|
109
|
+
* With the built-in `createDashboardAuth()` the cookie's HMAC signature and
|
|
110
|
+
* expiry are verified by the `/internal/*` middleware before this runs, so
|
|
111
|
+
* a forged cookie never gets here. Hosts that supply their own
|
|
112
|
+
* `dashboardAuth` middleware own that verification: the route cannot tell
|
|
113
|
+
* which auth is mounted, and re-verifying against DASHBOARD_PASSWORD here
|
|
114
|
+
* would break custom hosts whose sessions are signed differently. See the
|
|
115
|
+
* "Who counts as a human approver" note in apps/docs autosre.mdx.
|
|
107
116
|
*/
|
|
108
117
|
async function requireHumanDashboardSession(c: {
|
|
109
118
|
req: { header: (name: string) => string | undefined };
|
|
@@ -6,7 +6,7 @@ import { sqlDbFor } from "../lib/sql-db";
|
|
|
6
6
|
import { logToTailEvent, publishTail } from "../lib/tail-publisher";
|
|
7
7
|
import {
|
|
8
8
|
type DecodedLogRecord,
|
|
9
|
-
|
|
9
|
+
decodeLogsRequestCapped,
|
|
10
10
|
OtlpDecodeError,
|
|
11
11
|
type ReadBodyResult,
|
|
12
12
|
readOtlpBody,
|
|
@@ -32,9 +32,15 @@ export const logsReceiverPlugin: CollectorPlugin = {
|
|
|
32
32
|
throw err;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
// The record cap is applied during decode, before adaptation, so
|
|
36
|
+
// an oversized batch never gets a second full in-memory copy.
|
|
35
37
|
let decoded: DecodedLogRecord[];
|
|
38
|
+
let rejected = 0;
|
|
36
39
|
try {
|
|
37
|
-
decoded =
|
|
40
|
+
({ records: decoded, rejected } = decodeLogsRequestCapped(
|
|
41
|
+
body,
|
|
42
|
+
MAX_LOGS_PER_REQUEST,
|
|
43
|
+
));
|
|
38
44
|
} catch (err) {
|
|
39
45
|
if (err instanceof OtlpDecodeError) {
|
|
40
46
|
return c.json({ error: err.message }, err.status);
|
|
@@ -42,12 +48,6 @@ export const logsReceiverPlugin: CollectorPlugin = {
|
|
|
42
48
|
throw err;
|
|
43
49
|
}
|
|
44
50
|
|
|
45
|
-
let rejected = 0;
|
|
46
|
-
if (decoded.length > MAX_LOGS_PER_REQUEST) {
|
|
47
|
-
rejected = decoded.length - MAX_LOGS_PER_REQUEST;
|
|
48
|
-
decoded = decoded.slice(0, MAX_LOGS_PER_REQUEST);
|
|
49
|
-
}
|
|
50
|
-
|
|
51
51
|
const store = new LogsStore(sqlDbFor(c.env));
|
|
52
52
|
const retentionHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
|
|
53
53
|
const now = new Date();
|
|
@@ -5,7 +5,8 @@ import type { CollectorPlugin } from "../framework/collector";
|
|
|
5
5
|
import { MetricsStore } from "../lib/metrics-store";
|
|
6
6
|
import { sqlDbFor } from "../lib/sql-db";
|
|
7
7
|
import {
|
|
8
|
-
|
|
8
|
+
type DecodedMetricPoint,
|
|
9
|
+
decodeMetricsRequestCapped,
|
|
9
10
|
OtlpDecodeError,
|
|
10
11
|
type ReadBodyResult,
|
|
11
12
|
readOtlpBody,
|
|
@@ -31,9 +32,15 @@ export const metricsReceiverPlugin: CollectorPlugin = {
|
|
|
31
32
|
throw err;
|
|
32
33
|
}
|
|
33
34
|
|
|
34
|
-
|
|
35
|
+
// The point cap is applied during decode, before adaptation, so
|
|
36
|
+
// an oversized batch never gets a second full in-memory copy.
|
|
37
|
+
let points: DecodedMetricPoint[];
|
|
38
|
+
let rejected = 0;
|
|
35
39
|
try {
|
|
36
|
-
points =
|
|
40
|
+
({ points, rejected } = decodeMetricsRequestCapped(
|
|
41
|
+
body,
|
|
42
|
+
MAX_POINTS_PER_REQUEST,
|
|
43
|
+
));
|
|
37
44
|
} catch (err) {
|
|
38
45
|
if (err instanceof OtlpDecodeError) {
|
|
39
46
|
return c.json({ error: err.message }, err.status);
|
|
@@ -41,12 +48,6 @@ export const metricsReceiverPlugin: CollectorPlugin = {
|
|
|
41
48
|
throw err;
|
|
42
49
|
}
|
|
43
50
|
|
|
44
|
-
let rejected = 0;
|
|
45
|
-
if (points.length > MAX_POINTS_PER_REQUEST) {
|
|
46
|
-
rejected = points.length - MAX_POINTS_PER_REQUEST;
|
|
47
|
-
points = points.slice(0, MAX_POINTS_PER_REQUEST);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
51
|
const retentionHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
|
|
51
52
|
const now = new Date();
|
|
52
53
|
const receivedAt = now.toISOString();
|
|
@@ -5,7 +5,7 @@ import type { CollectorPlugin } from "../framework/collector";
|
|
|
5
5
|
import { toStoredSpans } from "../lib/otlp";
|
|
6
6
|
import { publishTail, spanToTailEvent } from "../lib/tail-publisher";
|
|
7
7
|
import {
|
|
8
|
-
|
|
8
|
+
decodeTraceRequestCapped,
|
|
9
9
|
OtlpDecodeError,
|
|
10
10
|
type ReadBodyResult,
|
|
11
11
|
readOtlpBody,
|
|
@@ -32,38 +32,22 @@ export const otlpReceiverPlugin: CollectorPlugin = {
|
|
|
32
32
|
throw err;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
// Decode and apply the span cap on the raw message before adapting,
|
|
36
|
+
// tracking rejected excess for the partial_success envelope. The
|
|
37
|
+
// cap runs pre-adaptation so an oversized batch never gets a
|
|
38
|
+
// second full in-memory copy.
|
|
39
|
+
let capped: ReturnType<typeof decodeTraceRequestCapped>;
|
|
36
40
|
try {
|
|
37
|
-
|
|
41
|
+
capped = decodeTraceRequestCapped(body, MAX_SPANS_PER_REQUEST);
|
|
38
42
|
} catch (err) {
|
|
39
43
|
if (err instanceof OtlpDecodeError) {
|
|
40
44
|
return c.json({ error: err.message }, err.status);
|
|
41
45
|
}
|
|
42
46
|
throw err;
|
|
43
47
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
// touching the rest of the pipeline.
|
|
48
|
-
let total = 0;
|
|
49
|
-
let rejected = 0;
|
|
50
|
-
const trimmedResourceSpans = [];
|
|
51
|
-
for (const rs of payload.resourceSpans ?? []) {
|
|
52
|
-
const scopeSpans = [];
|
|
53
|
-
for (const ss of rs.scopeSpans ?? []) {
|
|
54
|
-
const spans = [];
|
|
55
|
-
for (const s of ss.spans ?? []) {
|
|
56
|
-
if (total >= MAX_SPANS_PER_REQUEST) {
|
|
57
|
-
rejected++;
|
|
58
|
-
} else {
|
|
59
|
-
spans.push(s);
|
|
60
|
-
total++;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
scopeSpans.push({ ...ss, spans });
|
|
64
|
-
}
|
|
65
|
-
trimmedResourceSpans.push({ ...rs, scopeSpans });
|
|
66
|
-
}
|
|
48
|
+
const total = capped.accepted;
|
|
49
|
+
const rejected = capped.rejected;
|
|
50
|
+
const trimmedResourceSpans = capped.request.resourceSpans;
|
|
67
51
|
|
|
68
52
|
const parsedSpans = toStoredSpans(
|
|
69
53
|
{ resourceSpans: trimmedResourceSpans },
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { safeFilenamePart } from "./profile-routes";
|
|
3
|
+
|
|
4
|
+
describe("safeFilenamePart", () => {
|
|
5
|
+
it("passes real profile ids and hex trace ids through unchanged", () => {
|
|
6
|
+
expect(safeFilenamePart("01JC5Z8Q3MABCDEFGHJKMNPQRS")).toBe(
|
|
7
|
+
"01JC5Z8Q3MABCDEFGHJKMNPQRS",
|
|
8
|
+
);
|
|
9
|
+
expect(
|
|
10
|
+
safeFilenamePart("0af7651916cd43dd8448eb211c80319c".slice(0, 8)),
|
|
11
|
+
).toBe("0af76519");
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it("neutralises header-injection characters in trace_id", () => {
|
|
15
|
+
const out = safeFilenamePart('a"; x=\r\nSet-Cookie: y');
|
|
16
|
+
expect(out).toMatch(/^[A-Za-z0-9_.-]*$/);
|
|
17
|
+
expect(out).not.toContain('"');
|
|
18
|
+
expect(out).not.toContain("\r");
|
|
19
|
+
expect(out).not.toContain("\n");
|
|
20
|
+
expect(out).not.toContain(";");
|
|
21
|
+
});
|
|
22
|
+
});
|
|
@@ -58,6 +58,15 @@ const profileId = (): string => {
|
|
|
58
58
|
return timeStr + randStr;
|
|
59
59
|
};
|
|
60
60
|
|
|
61
|
+
/**
|
|
62
|
+
* Restrict a value interpolated into a Content-Disposition filename to a
|
|
63
|
+
* safe charset (ids and hex trace ids pass through unchanged). Prevents a
|
|
64
|
+
* crafted `trace_id` query value from injecting quotes, `;` parameters or
|
|
65
|
+
* CR/LF into the header.
|
|
66
|
+
*/
|
|
67
|
+
export const safeFilenamePart = (value: string): string =>
|
|
68
|
+
value.replace(/[^A-Za-z0-9_.-]/g, "_");
|
|
69
|
+
|
|
61
70
|
// Parse a non-negative integer from an untrusted source. Returns the
|
|
62
71
|
// fallback when the input is missing, malformed, NaN, negative, or
|
|
63
72
|
// non-finite. parseInt happily returns NaN for "foo" and that NaN was
|
|
@@ -292,7 +301,7 @@ export const profileRoutesPlugin: CollectorPlugin = {
|
|
|
292
301
|
headers: {
|
|
293
302
|
"Content-Type": "application/octet-stream",
|
|
294
303
|
"Content-Encoding": "gzip",
|
|
295
|
-
"Content-Disposition": `attachment; filename="${id}-trace-${filterTraceId.slice(0, 8)}.pprof.gz"`,
|
|
304
|
+
"Content-Disposition": `attachment; filename="${safeFilenamePart(id)}-trace-${safeFilenamePart(filterTraceId.slice(0, 8))}.pprof.gz"`,
|
|
296
305
|
// Surface the size shrink so the UI can show savings.
|
|
297
306
|
"X-Obs-Filter-Original-Bytes": String(raw.byteLength),
|
|
298
307
|
"X-Obs-Filter-Filtered-Bytes": String(filtered.byteLength),
|
|
@@ -313,7 +322,7 @@ export const profileRoutesPlugin: CollectorPlugin = {
|
|
|
313
322
|
return new Response(obj.body, {
|
|
314
323
|
headers: {
|
|
315
324
|
"Content-Type": "application/octet-stream",
|
|
316
|
-
"Content-Disposition": `attachment; filename="${id}.pprof.gz"`,
|
|
325
|
+
"Content-Disposition": `attachment; filename="${safeFilenamePart(id)}.pprof.gz"`,
|
|
317
326
|
},
|
|
318
327
|
});
|
|
319
328
|
}
|