@obsunified/collector 2.3.0 → 2.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/auth.d.ts +2 -1
- package/dist/auth.js +1 -1
- package/dist/{chunk-2FALTDCE.js → chunk-VSOZJO6Y.js} +12 -1
- package/dist/chunk-VSOZJO6Y.js.map +1 -0
- package/dist/index.d.ts +271 -62
- package/dist/index.js +1077 -137
- package/dist/index.js.map +1 -1
- package/dist/{ingest-auth-BmP5-f96.d.ts → ingest-auth-BTowieER.d.ts} +66 -2
- package/package.json +3 -3
- package/src/framework/collector.ts +22 -0
- package/src/framework/env.ts +9 -1
- package/src/framework/ingest-auth-warning.test.ts +71 -0
- package/src/index.ts +15 -0
- package/src/lib/alert-email.ts +104 -0
- package/src/lib/alerts-store.test.ts +147 -2
- package/src/lib/alerts-store.ts +438 -63
- package/src/lib/metric-alerts.test.ts +769 -0
- package/src/lib/metric-alerts.ts +640 -0
- package/src/lib/metrics-store.ts +9 -2
- package/src/lib/projects-store.test.ts +44 -1
- package/src/lib/projects-store.ts +21 -0
- package/src/lib/sql-db-postgres.integration.test.ts +247 -0
- package/src/lib/store/trace-queries.ts +1 -1
- package/src/lib/webhook-url.test.ts +49 -0
- package/src/lib/webhook-url.ts +72 -14
- package/src/migrations/054_alert_rules_pending_secs.sql +11 -0
- package/src/migrations/055_alert_state_pending_since.sql +12 -0
- package/src/migrations/056_metric_point_series_received.sql +9 -0
- package/src/migrations/057_alert_evaluations_outcome.sql +11 -0
- package/src/migrations/058_alert_state_pending_seen_at.sql +12 -0
- package/src/migrations/059_alert_evaluations_error.sql +11 -0
- package/src/migrations-postgres/054_alert_rules_pending_secs.sql +7 -0
- package/src/migrations-postgres/055_alert_state_pending_since.sql +6 -0
- package/src/migrations-postgres/056_metric_point_series_received.sql +7 -0
- package/src/migrations-postgres/057_alert_evaluations_outcome.sql +6 -0
- package/src/migrations-postgres/058_alert_state_pending_seen_at.sql +6 -0
- package/src/migrations-postgres/059_alert_evaluations_error.sql +6 -0
- package/src/migrations-postgres/README.md +13 -0
- package/src/otlp/decode/logs.ts +30 -2
- package/src/otlp/decode/metrics.ts +67 -2
- package/src/otlp/decode/traces.ts +57 -4
- package/src/otlp/decode/values.ts +64 -10
- package/src/otlp/decode.test.ts +305 -0
- package/src/otlp/decode.ts +9 -3
- package/src/plugins/alerts-evaluator-pending.test.ts +322 -0
- package/src/plugins/alerts-evaluator-retention.test.ts +2 -0
- package/src/plugins/alerts-evaluator.test.ts +10 -2
- package/src/plugins/alerts-evaluator.ts +239 -40
- package/src/plugins/alerts-routes.ts +17 -2
- package/src/plugins/autosre-routes.ts +9 -0
- package/src/plugins/logs-receiver.ts +8 -8
- package/src/plugins/metrics-receiver.ts +10 -9
- package/src/plugins/otlp-receiver.ts +10 -26
- package/src/plugins/profile-routes.test.ts +22 -0
- package/src/plugins/profile-routes.ts +11 -2
- package/src/plugins/replay-query-routes.test.ts +63 -1
- package/src/plugins/replay-query-routes.ts +40 -9
- package/dist/chunk-2FALTDCE.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
randomHex,
|
|
10
10
|
sha256Hex,
|
|
11
11
|
sqlDbFor
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-VSOZJO6Y.js";
|
|
13
13
|
import {
|
|
14
14
|
LlmCallError,
|
|
15
15
|
aiEvaluationEvidenceReferences,
|
|
@@ -3575,22 +3575,66 @@ var isPrivateIpv4 = (o) => o[0] === 0 || // "this" network / unspecified
|
|
|
3575
3575
|
o[0] === 10 || o[0] === 127 || o[0] === 100 && o[1] >= 64 && o[1] <= 127 || // CGNAT
|
|
3576
3576
|
o[0] === 169 && o[1] === 254 || // link-local incl. cloud metadata
|
|
3577
3577
|
o[0] === 172 && o[1] >= 16 && o[1] <= 31 || o[0] === 192 && o[1] === 168 || o[0] >= 224;
|
|
3578
|
+
var parseIpv6 = (literal) => {
|
|
3579
|
+
let text = literal.replace(/%.*$/, "");
|
|
3580
|
+
const tail = /:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(text);
|
|
3581
|
+
let tailGroups = [];
|
|
3582
|
+
if (tail) {
|
|
3583
|
+
const v4 = parseIpv4(tail[1]);
|
|
3584
|
+
if (!v4) return null;
|
|
3585
|
+
tailGroups = [v4[0] << 8 | v4[1], v4[2] << 8 | v4[3]];
|
|
3586
|
+
text = text.slice(0, -tail[1].length);
|
|
3587
|
+
if (text.endsWith(":") && !text.endsWith("::")) text = text.slice(0, -1);
|
|
3588
|
+
}
|
|
3589
|
+
const halves = text.split("::");
|
|
3590
|
+
if (halves.length > 2) return null;
|
|
3591
|
+
const toGroups = (part) => {
|
|
3592
|
+
if (part === "") return [];
|
|
3593
|
+
const groups = [];
|
|
3594
|
+
for (const g of part.split(":")) {
|
|
3595
|
+
if (!/^[0-9a-f]{1,4}$/.test(g)) return null;
|
|
3596
|
+
groups.push(Number.parseInt(g, 16));
|
|
3597
|
+
}
|
|
3598
|
+
return groups;
|
|
3599
|
+
};
|
|
3600
|
+
const head = toGroups(halves[0]);
|
|
3601
|
+
const rest = halves.length === 2 ? toGroups(halves[1]) : [];
|
|
3602
|
+
if (!head || !rest) return null;
|
|
3603
|
+
const explicit = head.length + rest.length + tailGroups.length;
|
|
3604
|
+
if (halves.length === 1) {
|
|
3605
|
+
return explicit === 8 ? [...head, ...tailGroups] : null;
|
|
3606
|
+
}
|
|
3607
|
+
if (explicit > 7) return null;
|
|
3608
|
+
return [
|
|
3609
|
+
...head,
|
|
3610
|
+
...new Array(8 - explicit).fill(0),
|
|
3611
|
+
...rest,
|
|
3612
|
+
...tailGroups
|
|
3613
|
+
];
|
|
3614
|
+
};
|
|
3615
|
+
var embeddedIpv4 = (hi, lo) => [
|
|
3616
|
+
hi >> 8,
|
|
3617
|
+
hi & 255,
|
|
3618
|
+
lo >> 8,
|
|
3619
|
+
lo & 255
|
|
3620
|
+
];
|
|
3578
3621
|
var isPrivateIpv6 = (host) => {
|
|
3579
3622
|
const literal = host.replace(/^\[|\]$/g, "").toLowerCase();
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
if (
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
if (
|
|
3623
|
+
const g = parseIpv6(literal);
|
|
3624
|
+
if (!g) return true;
|
|
3625
|
+
const zeroPrefix = (n) => g.slice(0, n).every((x) => x === 0);
|
|
3626
|
+
if (zeroPrefix(6)) {
|
|
3627
|
+
if (g[6] === 0 && (g[7] === 0 || g[7] === 1)) return true;
|
|
3628
|
+
return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
|
|
3629
|
+
}
|
|
3630
|
+
if (zeroPrefix(5) && g[5] === 65535) {
|
|
3631
|
+
return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
|
|
3632
|
+
}
|
|
3633
|
+
if (g[0] === 100 && g[1] === 65435 && g.slice(2, 6).every((x) => x === 0)) {
|
|
3634
|
+
return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
|
|
3635
|
+
}
|
|
3636
|
+
if (g[0] === 8194) return isPrivateIpv4(embeddedIpv4(g[1], g[2]));
|
|
3637
|
+
const first = g[0];
|
|
3594
3638
|
if ((first & 65024) === 64512) return true;
|
|
3595
3639
|
if ((first & 65472) === 65152) return true;
|
|
3596
3640
|
if ((first & 65280) === 65280) return true;
|
|
@@ -5798,7 +5842,11 @@ var MetricsStore = class {
|
|
|
5798
5842
|
async ingestBatch(opts) {
|
|
5799
5843
|
const { projectId, points, receivedAt, expiresAt } = opts;
|
|
5800
5844
|
if (points.length === 0) return;
|
|
5801
|
-
const seriesByIdentity = await this.resolveSeries(
|
|
5845
|
+
const seriesByIdentity = await this.resolveSeries(
|
|
5846
|
+
projectId,
|
|
5847
|
+
points,
|
|
5848
|
+
receivedAt
|
|
5849
|
+
);
|
|
5802
5850
|
const pointStmt = this.db.prepare(`
|
|
5803
5851
|
INSERT INTO metric_point (
|
|
5804
5852
|
id, series_id, project_id, ts_ns, start_ts_ns,
|
|
@@ -5917,7 +5965,7 @@ var MetricsStore = class {
|
|
|
5917
5965
|
* Ensures a `metric_series` row exists for every unique identity in the
|
|
5918
5966
|
* batch. Returns a map of identity → series row for point linking.
|
|
5919
5967
|
*/
|
|
5920
|
-
async resolveSeries(projectId, points) {
|
|
5968
|
+
async resolveSeries(projectId, points, receivedAt) {
|
|
5921
5969
|
const byIdentity = /* @__PURE__ */ new Map();
|
|
5922
5970
|
for (const p of points) {
|
|
5923
5971
|
if (!byIdentity.has(p.identity)) byIdentity.set(p.identity, p);
|
|
@@ -5944,7 +5992,7 @@ var MetricsStore = class {
|
|
|
5944
5992
|
`);
|
|
5945
5993
|
const toInsert = [];
|
|
5946
5994
|
const attemptedIds = /* @__PURE__ */ new Set();
|
|
5947
|
-
const createdAt =
|
|
5995
|
+
const createdAt = receivedAt;
|
|
5948
5996
|
for (const [identity, p] of byIdentity.entries()) {
|
|
5949
5997
|
if (existing.has(identity)) continue;
|
|
5950
5998
|
const id = crypto.randomUUID();
|
|
@@ -7085,7 +7133,7 @@ async function selectTraceCandidates(db, options) {
|
|
|
7085
7133
|
}
|
|
7086
7134
|
if (options.search) {
|
|
7087
7135
|
const term = `%${options.search}%`;
|
|
7088
|
-
whereClause += " AND (span_name LIKE ? OR status_message LIKE ? OR attributes_json LIKE ? OR events_json LIKE ?)";
|
|
7136
|
+
whereClause += " AND (span_name LIKE ? OR status_message LIKE ? OR CAST(attributes_json AS TEXT) LIKE ? OR CAST(events_json AS TEXT) LIKE ?)";
|
|
7089
7137
|
binds.push(term, term, term, term);
|
|
7090
7138
|
}
|
|
7091
7139
|
const having = options.status === "error" ? "HAVING SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) > 0" : options.status === "ok" ? "HAVING SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) = 0" : "";
|
|
@@ -8498,6 +8546,17 @@ var createTelemetryCollectorApp = (pluginsOrConfig) => {
|
|
|
8498
8546
|
app.use("/v1/*", config.auth.middleware);
|
|
8499
8547
|
} else if (config.auth?.ingest) {
|
|
8500
8548
|
app.use("/v1/*", config.auth.ingest);
|
|
8549
|
+
} else if (!config.allowUnauthenticatedIngest) {
|
|
8550
|
+
let warned = false;
|
|
8551
|
+
app.use("/v1/*", async (_c, next) => {
|
|
8552
|
+
if (!warned) {
|
|
8553
|
+
warned = true;
|
|
8554
|
+
runtime.logger.warn(
|
|
8555
|
+
"[collector] WARNING: /v1/* ingest is UNAUTHENTICATED \u2014 this app was built without `auth`, so anyone who can reach it can write telemetry. Pass `auth: { middleware: createIngestAuth() }` (and set INGEST_KEY or create a key in the dashboard), or set `allowUnauthenticatedIngest: true` to acknowledge and silence this warning."
|
|
8556
|
+
);
|
|
8557
|
+
}
|
|
8558
|
+
await next();
|
|
8559
|
+
});
|
|
8501
8560
|
}
|
|
8502
8561
|
if (config.dashboardAuth) {
|
|
8503
8562
|
const { middleware } = config.dashboardAuth;
|
|
@@ -12872,8 +12931,426 @@ var aiSpanPayloadsProcessorPlugin = {
|
|
|
12872
12931
|
}
|
|
12873
12932
|
};
|
|
12874
12933
|
|
|
12934
|
+
// src/lib/alert-email.ts
|
|
12935
|
+
var MAX_EMAIL_RECIPIENTS = 20;
|
|
12936
|
+
var EMAIL_RE = /^[^\s@<>,;:"()[\]\\]+@[^\s@<>,;:"()[\]\\]+\.[^\s@<>,;:"()[\]\\]+$/;
|
|
12937
|
+
var isValidEmailAddress = (addr) => addr.length <= 254 && EMAIL_RE.test(addr);
|
|
12938
|
+
var oneLine = (s) => s.replace(/[\r\n]+/g, " ").trim();
|
|
12939
|
+
function formatAlertEmail(payload) {
|
|
12940
|
+
const label = payload.state === "firing" ? "FIRING" : "RESOLVED";
|
|
12941
|
+
const name = oneLine(payload.rule.name);
|
|
12942
|
+
const lines = [
|
|
12943
|
+
payload.state === "firing" ? `Alert rule "${name}" is firing.` : `Alert rule "${name}" has resolved.`,
|
|
12944
|
+
"",
|
|
12945
|
+
`Rule: ${name} (${payload.rule.id})`,
|
|
12946
|
+
`Signal: ${payload.rule.signal}`,
|
|
12947
|
+
`State: ${payload.state}`,
|
|
12948
|
+
`Value: ${payload.value}`,
|
|
12949
|
+
`Condition: value ${payload.comparison} ${payload.threshold}`,
|
|
12950
|
+
`Threshold: ${payload.threshold}`,
|
|
12951
|
+
`Comparison: ${payload.comparison}`,
|
|
12952
|
+
`Evaluated: ${payload.evaluatedAt}`,
|
|
12953
|
+
`Project: ${payload.projectId}`
|
|
12954
|
+
];
|
|
12955
|
+
if (payload.analysis) {
|
|
12956
|
+
lines.push(
|
|
12957
|
+
`Analysis: ${payload.analysis.id}${payload.analysis.status ? ` (${payload.analysis.status})` : ""}`
|
|
12958
|
+
);
|
|
12959
|
+
if (payload.analysis.narrative) {
|
|
12960
|
+
lines.push("", payload.analysis.narrative);
|
|
12961
|
+
}
|
|
12962
|
+
}
|
|
12963
|
+
lines.push("", "Sent by obs-unified.");
|
|
12964
|
+
return {
|
|
12965
|
+
subject: `[obs-unified] ${label}: ${name}`,
|
|
12966
|
+
text: `${lines.join("\n")}
|
|
12967
|
+
`
|
|
12968
|
+
};
|
|
12969
|
+
}
|
|
12970
|
+
|
|
12875
12971
|
// src/lib/alerts-store.ts
|
|
12876
12972
|
import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT4 } from "@obsunified/types";
|
|
12973
|
+
|
|
12974
|
+
// src/lib/metric-alerts.ts
|
|
12975
|
+
var METRIC_AGGREGATIONS = [
|
|
12976
|
+
"last",
|
|
12977
|
+
"min",
|
|
12978
|
+
"max",
|
|
12979
|
+
"avg",
|
|
12980
|
+
"sum",
|
|
12981
|
+
"increase",
|
|
12982
|
+
"rate",
|
|
12983
|
+
"age_seconds"
|
|
12984
|
+
];
|
|
12985
|
+
var METRIC_FIELDS = [
|
|
12986
|
+
"value",
|
|
12987
|
+
"count",
|
|
12988
|
+
"sum"
|
|
12989
|
+
];
|
|
12990
|
+
var TEMPORALITY_DELTA = 1;
|
|
12991
|
+
var MAX_METRIC_NAMES = 20;
|
|
12992
|
+
var MAX_ATTRIBUTE_KEYS = 32;
|
|
12993
|
+
var MAX_MATCHED_SERIES = 1e3;
|
|
12994
|
+
var MAX_LOADED_POINTS = 5e4;
|
|
12995
|
+
var MAX_METRIC_WINDOW_MINS = 1440;
|
|
12996
|
+
var DEFAULT_STALE_SECS = 300;
|
|
12997
|
+
var MAX_STALE_SECS = 86400;
|
|
12998
|
+
var METRIC_NO_DATA_POLICIES = ["zero", "keep"];
|
|
12999
|
+
var ID_CHUNK = 50;
|
|
13000
|
+
var SELECTOR_KEYS = [
|
|
13001
|
+
"metric",
|
|
13002
|
+
"attributes",
|
|
13003
|
+
"resourceAttributes",
|
|
13004
|
+
"serviceName",
|
|
13005
|
+
"field"
|
|
13006
|
+
];
|
|
13007
|
+
var QUERY_KEYS = [
|
|
13008
|
+
...SELECTOR_KEYS,
|
|
13009
|
+
"aggregation",
|
|
13010
|
+
"denominator",
|
|
13011
|
+
"minDenominator",
|
|
13012
|
+
"staleSecs",
|
|
13013
|
+
"noData"
|
|
13014
|
+
];
|
|
13015
|
+
var DENOMINATOR_KEYS = [...SELECTOR_KEYS, "aggregation"];
|
|
13016
|
+
var isPlainObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
13017
|
+
var validateAttributes = (value, path) => {
|
|
13018
|
+
if (value === void 0) return;
|
|
13019
|
+
if (!isPlainObject(value))
|
|
13020
|
+
throw new Error(`${path} must be an object of attribute: value pairs`);
|
|
13021
|
+
const entries = Object.entries(value);
|
|
13022
|
+
if (entries.length > MAX_ATTRIBUTE_KEYS)
|
|
13023
|
+
throw new Error(`${path} may list at most ${MAX_ATTRIBUTE_KEYS} keys`);
|
|
13024
|
+
for (const [key, v] of entries) {
|
|
13025
|
+
if (!key.trim()) throw new Error(`${path} has an empty attribute name`);
|
|
13026
|
+
const ok = typeof v === "string" || typeof v === "boolean" || typeof v === "number" && Number.isFinite(v);
|
|
13027
|
+
if (!ok)
|
|
13028
|
+
throw new Error(
|
|
13029
|
+
`${path}.${key} must be a string, finite number or boolean`
|
|
13030
|
+
);
|
|
13031
|
+
}
|
|
13032
|
+
};
|
|
13033
|
+
var validateSelector = (q, path, allowedKeys) => {
|
|
13034
|
+
for (const key of Object.keys(q)) {
|
|
13035
|
+
if (!allowedKeys.includes(key))
|
|
13036
|
+
throw new Error(
|
|
13037
|
+
`${path}.${key} is not a metrics query field (allowed: ${allowedKeys.join(", ")})`
|
|
13038
|
+
);
|
|
13039
|
+
}
|
|
13040
|
+
const metric = q.metric;
|
|
13041
|
+
if (typeof metric === "string") {
|
|
13042
|
+
if (!metric.trim()) throw new Error(`${path}.metric must not be empty`);
|
|
13043
|
+
} else if (Array.isArray(metric)) {
|
|
13044
|
+
if (metric.length === 0)
|
|
13045
|
+
throw new Error(`${path}.metric must name at least one metric`);
|
|
13046
|
+
if (metric.length > MAX_METRIC_NAMES)
|
|
13047
|
+
throw new Error(
|
|
13048
|
+
`${path}.metric may name at most ${MAX_METRIC_NAMES} metrics`
|
|
13049
|
+
);
|
|
13050
|
+
for (const m of metric) {
|
|
13051
|
+
if (typeof m !== "string" || !m.trim())
|
|
13052
|
+
throw new Error(`${path}.metric entries must be non-empty strings`);
|
|
13053
|
+
}
|
|
13054
|
+
} else {
|
|
13055
|
+
throw new Error(
|
|
13056
|
+
`${path}.metric is required: a metric name or a list of names`
|
|
13057
|
+
);
|
|
13058
|
+
}
|
|
13059
|
+
validateAttributes(q.attributes, `${path}.attributes`);
|
|
13060
|
+
validateAttributes(q.resourceAttributes, `${path}.resourceAttributes`);
|
|
13061
|
+
if (q.serviceName !== void 0 && (typeof q.serviceName !== "string" || !q.serviceName.trim()))
|
|
13062
|
+
throw new Error(`${path}.serviceName must be a non-empty string`);
|
|
13063
|
+
if (q.field !== void 0 && !METRIC_FIELDS.includes(q.field))
|
|
13064
|
+
throw new Error(
|
|
13065
|
+
`${path}.field must be one of: ${METRIC_FIELDS.join(", ")}`
|
|
13066
|
+
);
|
|
13067
|
+
};
|
|
13068
|
+
function validateMetricQuery(query) {
|
|
13069
|
+
if (!isPlainObject(query))
|
|
13070
|
+
throw new Error("query must be an object for the metrics signal");
|
|
13071
|
+
validateSelector(query, "query", QUERY_KEYS);
|
|
13072
|
+
const aggregation = query.aggregation;
|
|
13073
|
+
if (!METRIC_AGGREGATIONS.includes(aggregation))
|
|
13074
|
+
throw new Error(
|
|
13075
|
+
`query.aggregation must be one of: ${METRIC_AGGREGATIONS.join(", ")}`
|
|
13076
|
+
);
|
|
13077
|
+
if (query.staleSecs !== void 0 && (!Number.isInteger(query.staleSecs) || query.staleSecs < 1 || query.staleSecs > MAX_STALE_SECS))
|
|
13078
|
+
throw new Error(
|
|
13079
|
+
`query.staleSecs must be an integer from 1 to ${MAX_STALE_SECS}`
|
|
13080
|
+
);
|
|
13081
|
+
if (query.noData !== void 0 && !METRIC_NO_DATA_POLICIES.includes(query.noData))
|
|
13082
|
+
throw new Error(
|
|
13083
|
+
`query.noData must be one of: ${METRIC_NO_DATA_POLICIES.join(", ")}`
|
|
13084
|
+
);
|
|
13085
|
+
if (query.denominator === void 0) {
|
|
13086
|
+
if (query.minDenominator !== void 0)
|
|
13087
|
+
throw new Error("query.minDenominator requires query.denominator");
|
|
13088
|
+
return;
|
|
13089
|
+
}
|
|
13090
|
+
if (aggregation === "age_seconds")
|
|
13091
|
+
throw new Error("query.denominator cannot be used with age_seconds");
|
|
13092
|
+
if (!isPlainObject(query.denominator))
|
|
13093
|
+
throw new Error("query.denominator must be an object");
|
|
13094
|
+
validateSelector(query.denominator, "query.denominator", DENOMINATOR_KEYS);
|
|
13095
|
+
const denAgg = query.denominator.aggregation;
|
|
13096
|
+
if (denAgg !== void 0) {
|
|
13097
|
+
if (denAgg === "age_seconds" || !METRIC_AGGREGATIONS.includes(denAgg))
|
|
13098
|
+
throw new Error(
|
|
13099
|
+
`query.denominator.aggregation must be one of: ${METRIC_AGGREGATIONS.filter((a) => a !== "age_seconds").join(", ")}`
|
|
13100
|
+
);
|
|
13101
|
+
}
|
|
13102
|
+
if (query.minDenominator !== void 0 && (typeof query.minDenominator !== "number" || !Number.isFinite(query.minDenominator) || query.minDenominator < 0))
|
|
13103
|
+
throw new Error("query.minDenominator must be a finite number >= 0");
|
|
13104
|
+
}
|
|
13105
|
+
var toBigInt = (ns) => {
|
|
13106
|
+
if (!ns) return null;
|
|
13107
|
+
try {
|
|
13108
|
+
return BigInt(ns);
|
|
13109
|
+
} catch {
|
|
13110
|
+
return null;
|
|
13111
|
+
}
|
|
13112
|
+
};
|
|
13113
|
+
var readField = (p, field) => {
|
|
13114
|
+
const raw = field === "count" ? p.count : field === "sum" ? p.sum : p.value;
|
|
13115
|
+
const n = typeof raw === "string" ? Number(raw) : raw;
|
|
13116
|
+
return typeof n === "number" && Number.isFinite(n) ? n : null;
|
|
13117
|
+
};
|
|
13118
|
+
var sortPoints = (points) => [...points].sort((a, b) => {
|
|
13119
|
+
const ta = toBigInt(a.tsNs) ?? 0n;
|
|
13120
|
+
const tb = toBigInt(b.tsNs) ?? 0n;
|
|
13121
|
+
if (ta !== tb) return ta < tb ? -1 : 1;
|
|
13122
|
+
return a.receivedAt < b.receivedAt ? -1 : a.receivedAt > b.receivedAt ? 1 : 0;
|
|
13123
|
+
});
|
|
13124
|
+
var cumulativeIncrease = (points, windowStartNs, bornInWindow) => {
|
|
13125
|
+
if (points.length === 0) return 0;
|
|
13126
|
+
const first = points[0];
|
|
13127
|
+
const countsFirst = first.start !== null ? first.start >= windowStartNs : bornInWindow;
|
|
13128
|
+
let total = countsFirst ? first.v : 0;
|
|
13129
|
+
for (let i = 1; i < points.length; i++) {
|
|
13130
|
+
const prev = points[i - 1];
|
|
13131
|
+
const cur = points[i];
|
|
13132
|
+
const restarted = cur.start !== null && prev.start !== null && cur.start !== prev.start;
|
|
13133
|
+
if (restarted || cur.v < prev.v) total += cur.v;
|
|
13134
|
+
else total += cur.v - prev.v;
|
|
13135
|
+
}
|
|
13136
|
+
return total;
|
|
13137
|
+
};
|
|
13138
|
+
function reduceSeries(points, opts) {
|
|
13139
|
+
const usable = [];
|
|
13140
|
+
for (const p of sortPoints(points)) {
|
|
13141
|
+
const v = readField(p, opts.field);
|
|
13142
|
+
if (v !== null) usable.push({ v, start: toBigInt(p.startTsNs) });
|
|
13143
|
+
}
|
|
13144
|
+
if (usable.length === 0) return null;
|
|
13145
|
+
const values = usable.map((u) => u.v);
|
|
13146
|
+
switch (opts.aggregation) {
|
|
13147
|
+
case "last":
|
|
13148
|
+
return values[values.length - 1];
|
|
13149
|
+
case "min":
|
|
13150
|
+
return values.reduce((a, b) => b < a ? b : a);
|
|
13151
|
+
case "max":
|
|
13152
|
+
return values.reduce((a, b) => b > a ? b : a);
|
|
13153
|
+
case "sum":
|
|
13154
|
+
return values.reduce((a, b) => a + b, 0);
|
|
13155
|
+
case "avg":
|
|
13156
|
+
return values.reduce((a, b) => a + b, 0) / values.length;
|
|
13157
|
+
case "increase":
|
|
13158
|
+
case "rate": {
|
|
13159
|
+
const increase = opts.temporality === TEMPORALITY_DELTA ? values.reduce((a, b) => a + b, 0) : cumulativeIncrease(
|
|
13160
|
+
usable,
|
|
13161
|
+
opts.windowStartNs,
|
|
13162
|
+
opts.bornInWindow ?? false
|
|
13163
|
+
);
|
|
13164
|
+
return opts.aggregation === "rate" ? increase / opts.windowSecs : increase;
|
|
13165
|
+
}
|
|
13166
|
+
}
|
|
13167
|
+
}
|
|
13168
|
+
var metricRatio = (numerator, denominator, minDenominator) => denominator > 0 && denominator >= minDenominator ? numerator / denominator : 0;
|
|
13169
|
+
var parseAttrs2 = (raw) => {
|
|
13170
|
+
if (raw === null || raw === void 0) return {};
|
|
13171
|
+
if (typeof raw === "string") {
|
|
13172
|
+
try {
|
|
13173
|
+
const parsed = JSON.parse(raw);
|
|
13174
|
+
return isPlainObject(parsed) ? parsed : {};
|
|
13175
|
+
} catch {
|
|
13176
|
+
return {};
|
|
13177
|
+
}
|
|
13178
|
+
}
|
|
13179
|
+
return isPlainObject(raw) ? raw : {};
|
|
13180
|
+
};
|
|
13181
|
+
var attrsMatch = (stored, wanted) => {
|
|
13182
|
+
if (!wanted) return true;
|
|
13183
|
+
for (const [key, expected] of Object.entries(wanted)) {
|
|
13184
|
+
const actual = stored[key];
|
|
13185
|
+
if (actual === void 0 || actual === null) return false;
|
|
13186
|
+
if (typeof actual === "object") return false;
|
|
13187
|
+
if (String(actual) !== String(expected)) return false;
|
|
13188
|
+
}
|
|
13189
|
+
return true;
|
|
13190
|
+
};
|
|
13191
|
+
var metricNames = (selector) => [
|
|
13192
|
+
...new Set(
|
|
13193
|
+
(Array.isArray(selector.metric) ? selector.metric : [selector.metric]).map(
|
|
13194
|
+
(m) => m.trim()
|
|
13195
|
+
)
|
|
13196
|
+
)
|
|
13197
|
+
];
|
|
13198
|
+
var chunks = (items, size) => {
|
|
13199
|
+
const out = [];
|
|
13200
|
+
for (let i = 0; i < items.length; i += size)
|
|
13201
|
+
out.push(items.slice(i, i + size));
|
|
13202
|
+
return out;
|
|
13203
|
+
};
|
|
13204
|
+
async function matchingSeries(db, projectId, selector) {
|
|
13205
|
+
const names = metricNames(selector);
|
|
13206
|
+
let sql = `SELECT id, temporality, created_at, attributes_json, resource_attrs_json
|
|
13207
|
+
FROM metric_series
|
|
13208
|
+
WHERE project_id = ? AND name IN (${names.map(() => "?").join(", ")})`;
|
|
13209
|
+
const binds = [projectId, ...names];
|
|
13210
|
+
if (selector.serviceName) {
|
|
13211
|
+
sql += ` AND service_name = ?`;
|
|
13212
|
+
binds.push(selector.serviceName);
|
|
13213
|
+
}
|
|
13214
|
+
const rs = await db.prepare(sql).bind(...binds).all();
|
|
13215
|
+
const matched = (rs.results ?? []).filter(
|
|
13216
|
+
(row) => attrsMatch(parseAttrs2(row.attributes_json), selector.attributes) && attrsMatch(
|
|
13217
|
+
parseAttrs2(row.resource_attrs_json),
|
|
13218
|
+
selector.resourceAttributes
|
|
13219
|
+
)
|
|
13220
|
+
);
|
|
13221
|
+
if (matched.length > MAX_MATCHED_SERIES)
|
|
13222
|
+
throw new Error(
|
|
13223
|
+
`metrics query matches ${matched.length} series (limit ${MAX_MATCHED_SERIES}); narrow it with attributes or serviceName`
|
|
13224
|
+
);
|
|
13225
|
+
return matched.map((row) => ({
|
|
13226
|
+
id: row.id,
|
|
13227
|
+
createdAt: row.created_at,
|
|
13228
|
+
temporality: row.temporality === null || row.temporality === void 0 ? null : Number(row.temporality)
|
|
13229
|
+
}));
|
|
13230
|
+
}
|
|
13231
|
+
var STALENESS_APPLIES = /* @__PURE__ */ new Set([
|
|
13232
|
+
"last",
|
|
13233
|
+
"min",
|
|
13234
|
+
"max",
|
|
13235
|
+
"avg",
|
|
13236
|
+
"sum"
|
|
13237
|
+
]);
|
|
13238
|
+
async function evaluateSelector(db, selector, aggregation, ctx) {
|
|
13239
|
+
const series = await matchingSeries(db, ctx.projectId, selector);
|
|
13240
|
+
if (series.length === 0) return { value: 0, contributing: 0 };
|
|
13241
|
+
const windowStartMs = ctx.now.getTime() - ctx.windowMins * 6e4;
|
|
13242
|
+
const cutoff = new Date(windowStartMs).toISOString();
|
|
13243
|
+
const bySeries = /* @__PURE__ */ new Map();
|
|
13244
|
+
let loaded = 0;
|
|
13245
|
+
for (const chunk of chunks(series, ID_CHUNK)) {
|
|
13246
|
+
const cap = ctx.maxLoadedPoints ?? MAX_LOADED_POINTS;
|
|
13247
|
+
const budget = cap - loaded;
|
|
13248
|
+
const rs = await db.prepare(
|
|
13249
|
+
`SELECT series_id, ts_ns, start_ts_ns, value, count, sum, received_at
|
|
13250
|
+
FROM metric_point
|
|
13251
|
+
WHERE project_id = ? AND series_id IN (${chunk.map(() => "?").join(", ")})
|
|
13252
|
+
AND received_at >= ?
|
|
13253
|
+
LIMIT ?`
|
|
13254
|
+
).bind(ctx.projectId, ...chunk.map((s) => s.id), cutoff, budget + 1).all();
|
|
13255
|
+
const rows = rs.results ?? [];
|
|
13256
|
+
loaded += rows.length;
|
|
13257
|
+
if (loaded > cap)
|
|
13258
|
+
throw new Error(
|
|
13259
|
+
`metrics query loads more than ${cap} points in its window; shorten windowMins or narrow the query`
|
|
13260
|
+
);
|
|
13261
|
+
for (const row of rows) {
|
|
13262
|
+
const list = bySeries.get(row.series_id) ?? [];
|
|
13263
|
+
list.push({
|
|
13264
|
+
tsNs: String(row.ts_ns),
|
|
13265
|
+
startTsNs: row.start_ts_ns === null ? null : String(row.start_ts_ns),
|
|
13266
|
+
value: row.value,
|
|
13267
|
+
count: row.count,
|
|
13268
|
+
sum: row.sum,
|
|
13269
|
+
receivedAt: row.received_at
|
|
13270
|
+
});
|
|
13271
|
+
bySeries.set(row.series_id, list);
|
|
13272
|
+
}
|
|
13273
|
+
}
|
|
13274
|
+
const field = selector.field ?? "value";
|
|
13275
|
+
const windowStartNs = BigInt(windowStartMs) * 1000000n;
|
|
13276
|
+
const freshCutoff = new Date(
|
|
13277
|
+
ctx.now.getTime() - (ctx.staleSecs ?? DEFAULT_STALE_SECS) * 1e3
|
|
13278
|
+
).toISOString();
|
|
13279
|
+
let total = 0;
|
|
13280
|
+
let contributing = 0;
|
|
13281
|
+
for (const s of series) {
|
|
13282
|
+
const points = bySeries.get(s.id) ?? [];
|
|
13283
|
+
if (STALENESS_APPLIES.has(aggregation)) {
|
|
13284
|
+
const newest = points.reduce(
|
|
13285
|
+
(max, p) => max === null || p.receivedAt > max ? p.receivedAt : max,
|
|
13286
|
+
null
|
|
13287
|
+
);
|
|
13288
|
+
if (newest === null || newest < freshCutoff) continue;
|
|
13289
|
+
}
|
|
13290
|
+
const reduced = reduceSeries(points, {
|
|
13291
|
+
aggregation,
|
|
13292
|
+
field,
|
|
13293
|
+
temporality: s.temporality,
|
|
13294
|
+
windowStartNs,
|
|
13295
|
+
windowSecs: ctx.windowMins * 60,
|
|
13296
|
+
bornInWindow: s.createdAt >= cutoff
|
|
13297
|
+
});
|
|
13298
|
+
if (reduced !== null) {
|
|
13299
|
+
total += reduced;
|
|
13300
|
+
contributing += 1;
|
|
13301
|
+
}
|
|
13302
|
+
}
|
|
13303
|
+
return { value: total, contributing };
|
|
13304
|
+
}
|
|
13305
|
+
async function ageSeconds(db, selector, ctx) {
|
|
13306
|
+
const series = await matchingSeries(db, ctx.projectId, selector);
|
|
13307
|
+
let newest = null;
|
|
13308
|
+
for (const chunk of chunks(series, ID_CHUNK)) {
|
|
13309
|
+
const row = await db.prepare(
|
|
13310
|
+
`SELECT MAX(received_at) AS newest FROM metric_point
|
|
13311
|
+
WHERE project_id = ? AND series_id IN (${chunk.map(() => "?").join(", ")})`
|
|
13312
|
+
).bind(ctx.projectId, ...chunk.map((s) => s.id)).first();
|
|
13313
|
+
if (row?.newest && (newest === null || row.newest > newest))
|
|
13314
|
+
newest = row.newest;
|
|
13315
|
+
}
|
|
13316
|
+
const reference = Date.parse(newest ?? ctx.ruleCreatedAt);
|
|
13317
|
+
if (!Number.isFinite(reference)) return 0;
|
|
13318
|
+
return Math.max(0, (ctx.now.getTime() - reference) / 1e3);
|
|
13319
|
+
}
|
|
13320
|
+
async function evaluateMetricQuery(db, query, ctx) {
|
|
13321
|
+
if (query.aggregation === "age_seconds")
|
|
13322
|
+
return { value: await ageSeconds(db, query, ctx), noData: false };
|
|
13323
|
+
const withStale = { ...ctx, staleSecs: query.staleSecs ?? ctx.staleSecs };
|
|
13324
|
+
const numerator = await evaluateSelector(
|
|
13325
|
+
db,
|
|
13326
|
+
query,
|
|
13327
|
+
query.aggregation,
|
|
13328
|
+
withStale
|
|
13329
|
+
);
|
|
13330
|
+
if (!query.denominator)
|
|
13331
|
+
return {
|
|
13332
|
+
value: numerator.value,
|
|
13333
|
+
noData: numerator.contributing === 0
|
|
13334
|
+
};
|
|
13335
|
+
const denominator = await evaluateSelector(
|
|
13336
|
+
db,
|
|
13337
|
+
query.denominator,
|
|
13338
|
+
query.denominator.aggregation ?? query.aggregation,
|
|
13339
|
+
withStale
|
|
13340
|
+
);
|
|
13341
|
+
return {
|
|
13342
|
+
value: metricRatio(
|
|
13343
|
+
numerator.value,
|
|
13344
|
+
denominator.value,
|
|
13345
|
+
query.minDenominator ?? 0
|
|
13346
|
+
),
|
|
13347
|
+
noData: numerator.contributing === 0 && denominator.contributing === 0
|
|
13348
|
+
};
|
|
13349
|
+
}
|
|
13350
|
+
|
|
13351
|
+
// src/lib/alerts-store.ts
|
|
13352
|
+
var MAX_PENDING_SECS = 86400;
|
|
13353
|
+
var EMAIL_UNAVAILABLE_MESSAGE = "email channels need SMTP configured on the collector (ALERT_SMTP_HOST and ALERT_SMTP_FROM on the Node collector); the Cloudflare Worker collector cannot send email";
|
|
12877
13354
|
var parseJsonField = (value) => {
|
|
12878
13355
|
if (typeof value === "string") return JSON.parse(value);
|
|
12879
13356
|
return value;
|
|
@@ -12892,6 +13369,8 @@ var rowToRule = (row, state) => ({
|
|
|
12892
13369
|
createdAt: row.created_at,
|
|
12893
13370
|
updatedAt: row.updated_at,
|
|
12894
13371
|
analysisId: row.analysis_id ?? null,
|
|
13372
|
+
pendingSecs: Number(row.pending_secs ?? 0),
|
|
13373
|
+
pendingSince: state?.pending_since ?? null,
|
|
12895
13374
|
currentState: state?.current_state,
|
|
12896
13375
|
lastStateChange: state?.last_state_change ?? null
|
|
12897
13376
|
});
|
|
@@ -12902,11 +13381,18 @@ var rowToEvaluation = (row) => ({
|
|
|
12902
13381
|
evaluatedAt: row.evaluated_at,
|
|
12903
13382
|
value: row.value,
|
|
12904
13383
|
state: row.state,
|
|
12905
|
-
notified: row.notified === 1
|
|
13384
|
+
notified: row.notified === 1,
|
|
13385
|
+
outcome: row.outcome ?? "value",
|
|
13386
|
+
error: row.error_message ?? null
|
|
12906
13387
|
});
|
|
13388
|
+
var MAX_ERROR_MESSAGE = 500;
|
|
13389
|
+
var sanitizeEvaluationError = (message) => {
|
|
13390
|
+
const cleaned = message.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s"'<>]+/gi, "<url>").replace(/\s+/g, " ").trim();
|
|
13391
|
+
return cleaned.length > MAX_ERROR_MESSAGE ? `${cleaned.slice(0, MAX_ERROR_MESSAGE - 1)}\u2026` : cleaned;
|
|
13392
|
+
};
|
|
12907
13393
|
var windowCutoffIso = (windowMins) => new Date(Date.now() - windowMins * 60 * 1e3).toISOString();
|
|
12908
|
-
var SIGNALS2 = ["spans", "logs", "usage", "ai"];
|
|
12909
|
-
var COMPARISONS = [">", ">=", "<", "<="];
|
|
13394
|
+
var SIGNALS2 = ["spans", "logs", "usage", "ai", "metrics"];
|
|
13395
|
+
var COMPARISONS = [">", ">=", "<", "<=", "==", "!="];
|
|
12910
13396
|
function compareValue(value, threshold, comparison) {
|
|
12911
13397
|
switch (comparison) {
|
|
12912
13398
|
case ">":
|
|
@@ -12917,6 +13403,10 @@ function compareValue(value, threshold, comparison) {
|
|
|
12917
13403
|
return value < threshold;
|
|
12918
13404
|
case "<=":
|
|
12919
13405
|
return value <= threshold;
|
|
13406
|
+
case "==":
|
|
13407
|
+
return value === threshold;
|
|
13408
|
+
case "!=":
|
|
13409
|
+
return value !== threshold;
|
|
12920
13410
|
}
|
|
12921
13411
|
}
|
|
12922
13412
|
var AlertsStore = class {
|
|
@@ -12934,11 +13424,46 @@ var AlertsStore = class {
|
|
|
12934
13424
|
throw new Error("windowMins must be >= 1");
|
|
12935
13425
|
if (!COMPARISONS.includes(input.comparison))
|
|
12936
13426
|
throw new Error(`comparison must be one of: ${COMPARISONS.join(", ")}`);
|
|
13427
|
+
if (input.pendingSecs !== void 0 && (!Number.isInteger(input.pendingSecs) || input.pendingSecs < 0 || input.pendingSecs > MAX_PENDING_SECS))
|
|
13428
|
+
throw new Error(
|
|
13429
|
+
`pendingSecs must be an integer from 0 to ${MAX_PENDING_SECS}`
|
|
13430
|
+
);
|
|
13431
|
+
if (input.signal === "metrics") {
|
|
13432
|
+
validateMetricQuery(input.query);
|
|
13433
|
+
if (input.windowMins > MAX_METRIC_WINDOW_MINS)
|
|
13434
|
+
throw new Error(
|
|
13435
|
+
`windowMins must be <= ${MAX_METRIC_WINDOW_MINS} for the metrics signal`
|
|
13436
|
+
);
|
|
13437
|
+
}
|
|
12937
13438
|
if (!Array.isArray(input.channels) || input.channels.length === 0)
|
|
12938
13439
|
throw new Error("at least one channel is required");
|
|
12939
13440
|
for (const ch of input.channels) {
|
|
13441
|
+
if (ch.type === "email") {
|
|
13442
|
+
if (!Array.isArray(ch.to) || ch.to.length === 0)
|
|
13443
|
+
throw new Error("email channel.to must list at least one address");
|
|
13444
|
+
if (ch.to.length > MAX_EMAIL_RECIPIENTS)
|
|
13445
|
+
throw new Error(
|
|
13446
|
+
`email channel.to may list at most ${MAX_EMAIL_RECIPIENTS} addresses`
|
|
13447
|
+
);
|
|
13448
|
+
for (const addr of ch.to) {
|
|
13449
|
+
if (typeof addr !== "string" || !isValidEmailAddress(addr))
|
|
13450
|
+
throw new Error(
|
|
13451
|
+
`email channel.to has an invalid address: ${JSON.stringify(addr)}`
|
|
13452
|
+
);
|
|
13453
|
+
}
|
|
13454
|
+
const extra = Object.keys(ch).filter((k) => k !== "type" && k !== "to");
|
|
13455
|
+
if (extra.length > 0)
|
|
13456
|
+
throw new Error(
|
|
13457
|
+
`email channel accepts only "type" and "to" (SMTP settings come from the collector environment); got: ${extra.join(", ")}`
|
|
13458
|
+
);
|
|
13459
|
+
if (!this.options.emailDeliveryAvailable)
|
|
13460
|
+
throw new Error(EMAIL_UNAVAILABLE_MESSAGE);
|
|
13461
|
+
continue;
|
|
13462
|
+
}
|
|
12940
13463
|
if (ch.type !== "webhook")
|
|
12941
|
-
throw new Error(
|
|
13464
|
+
throw new Error(
|
|
13465
|
+
`unknown channel type: ${ch.type}`
|
|
13466
|
+
);
|
|
12942
13467
|
if (!ch.url || typeof ch.url !== "string")
|
|
12943
13468
|
throw new Error("channel.url must be http(s)://");
|
|
12944
13469
|
try {
|
|
@@ -12961,7 +13486,55 @@ var AlertsStore = class {
|
|
|
12961
13486
|
const stateByRule = new Map(
|
|
12962
13487
|
(stateRs.results ?? []).map((s) => [s.rule_id, s])
|
|
12963
13488
|
);
|
|
12964
|
-
|
|
13489
|
+
const lastByRule = await this.lastEvaluations(projectId);
|
|
13490
|
+
return rows.map((r) => {
|
|
13491
|
+
const state = stateByRule.get(r.id);
|
|
13492
|
+
const last = lastByRule.get(r.id);
|
|
13493
|
+
return {
|
|
13494
|
+
...rowToRule(r, state),
|
|
13495
|
+
notificationDelivered: state ? state.notified === void 0 || state.notified === null ? true : Number(state.notified) === 1 : null,
|
|
13496
|
+
lastEvaluation: last ? {
|
|
13497
|
+
evaluatedAt: last.evaluated_at,
|
|
13498
|
+
outcome: last.outcome ?? "value",
|
|
13499
|
+
value: Number(last.value),
|
|
13500
|
+
error: (last.outcome ?? "value") === "error" ? last.error_message ?? null : null
|
|
13501
|
+
} : null
|
|
13502
|
+
};
|
|
13503
|
+
});
|
|
13504
|
+
}
|
|
13505
|
+
/**
|
|
13506
|
+
* The newest evaluation of every rule in the project, one query. Falls
|
|
13507
|
+
* back to the pre-057/059 column set on older databases.
|
|
13508
|
+
*/
|
|
13509
|
+
async lastEvaluations(projectId) {
|
|
13510
|
+
const run = (columns) => this.db.prepare(
|
|
13511
|
+
`SELECT e.rule_id, e.evaluated_at, e.value${columns}
|
|
13512
|
+
FROM alert_evaluations e
|
|
13513
|
+
WHERE e.project_id = ?
|
|
13514
|
+
AND e.evaluated_at = (
|
|
13515
|
+
SELECT MAX(e2.evaluated_at) FROM alert_evaluations e2
|
|
13516
|
+
WHERE e2.rule_id = e.rule_id
|
|
13517
|
+
)`
|
|
13518
|
+
).bind(projectId).all();
|
|
13519
|
+
let rs;
|
|
13520
|
+
try {
|
|
13521
|
+
rs = await run(", e.outcome, e.error_message");
|
|
13522
|
+
} catch (err) {
|
|
13523
|
+
if (!isMissingColumn(err, "error_message") && !isMissingColumn(err, "outcome"))
|
|
13524
|
+
throw err;
|
|
13525
|
+
try {
|
|
13526
|
+
rs = await run(", e.outcome");
|
|
13527
|
+
} catch (inner) {
|
|
13528
|
+
if (!isMissingColumn(inner, "outcome")) throw inner;
|
|
13529
|
+
rs = await run("");
|
|
13530
|
+
}
|
|
13531
|
+
}
|
|
13532
|
+
const out = /* @__PURE__ */ new Map();
|
|
13533
|
+
for (const row of rs.results ?? []) {
|
|
13534
|
+
const seen = out.get(row.rule_id);
|
|
13535
|
+
if (!seen || row.outcome === "error") out.set(row.rule_id, row);
|
|
13536
|
+
}
|
|
13537
|
+
return out;
|
|
12965
13538
|
}
|
|
12966
13539
|
async getRule(id, projectId) {
|
|
12967
13540
|
const row = await this.db.prepare(`SELECT * FROM alert_rules WHERE id = ? AND project_id = ?`).bind(id, projectId).first();
|
|
@@ -12973,10 +13546,8 @@ var AlertsStore = class {
|
|
|
12973
13546
|
this.validateInput(input);
|
|
12974
13547
|
const id = randomHex(16);
|
|
12975
13548
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12976
|
-
|
|
12977
|
-
|
|
12978
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
12979
|
-
).bind(
|
|
13549
|
+
const pendingSecs = input.pendingSecs ?? 0;
|
|
13550
|
+
const values = [
|
|
12980
13551
|
id,
|
|
12981
13552
|
projectId,
|
|
12982
13553
|
input.name.trim(),
|
|
@@ -12990,7 +13561,18 @@ var AlertsStore = class {
|
|
|
12990
13561
|
now,
|
|
12991
13562
|
now,
|
|
12992
13563
|
input.analysisId ?? null
|
|
12993
|
-
|
|
13564
|
+
];
|
|
13565
|
+
await this.withPendingColumnFallback(
|
|
13566
|
+
pendingSecs,
|
|
13567
|
+
() => this.db.prepare(
|
|
13568
|
+
`INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at, analysis_id, pending_secs)
|
|
13569
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
13570
|
+
).bind(...values, pendingSecs).run(),
|
|
13571
|
+
() => this.db.prepare(
|
|
13572
|
+
`INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at, analysis_id)
|
|
13573
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
13574
|
+
).bind(...values).run()
|
|
13575
|
+
);
|
|
12994
13576
|
const created = await this.getRule(id, projectId);
|
|
12995
13577
|
if (!created) throw new Error("Failed to create rule");
|
|
12996
13578
|
return created;
|
|
@@ -13007,16 +13589,13 @@ var AlertsStore = class {
|
|
|
13007
13589
|
comparison: patch.comparison ?? existing.comparison,
|
|
13008
13590
|
channels: patch.channels ?? existing.channels,
|
|
13009
13591
|
enabled: patch.enabled ?? existing.enabled,
|
|
13010
|
-
analysisId: "analysisId" in patch ? patch.analysisId : existing.analysisId
|
|
13592
|
+
analysisId: "analysisId" in patch ? patch.analysisId : existing.analysisId,
|
|
13593
|
+
pendingSecs: patch.pendingSecs ?? existing.pendingSecs
|
|
13011
13594
|
};
|
|
13012
13595
|
this.validateInput(merged);
|
|
13013
13596
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13014
|
-
|
|
13015
|
-
|
|
13016
|
-
name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
|
|
13017
|
-
comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?
|
|
13018
|
-
WHERE id = ? AND project_id = ?`
|
|
13019
|
-
).bind(
|
|
13597
|
+
const pendingSecs = merged.pendingSecs ?? 0;
|
|
13598
|
+
const values = [
|
|
13020
13599
|
merged.name.trim(),
|
|
13021
13600
|
merged.signal,
|
|
13022
13601
|
JSON.stringify(merged.query),
|
|
@@ -13026,12 +13605,72 @@ var AlertsStore = class {
|
|
|
13026
13605
|
JSON.stringify(merged.channels),
|
|
13027
13606
|
merged.enabled === false ? 0 : 1,
|
|
13028
13607
|
now,
|
|
13029
|
-
merged.analysisId ?? null
|
|
13030
|
-
|
|
13031
|
-
|
|
13032
|
-
|
|
13608
|
+
merged.analysisId ?? null
|
|
13609
|
+
];
|
|
13610
|
+
await this.withPendingColumnFallback(
|
|
13611
|
+
pendingSecs,
|
|
13612
|
+
() => this.db.prepare(
|
|
13613
|
+
`UPDATE alert_rules SET
|
|
13614
|
+
name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
|
|
13615
|
+
comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?,
|
|
13616
|
+
pending_secs = ?
|
|
13617
|
+
WHERE id = ? AND project_id = ?`
|
|
13618
|
+
).bind(...values, pendingSecs, id, projectId).run(),
|
|
13619
|
+
() => this.db.prepare(
|
|
13620
|
+
`UPDATE alert_rules SET
|
|
13621
|
+
name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
|
|
13622
|
+
comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?
|
|
13623
|
+
WHERE id = ? AND project_id = ?`
|
|
13624
|
+
).bind(...values, id, projectId).run()
|
|
13625
|
+
);
|
|
13626
|
+
const evaluationChanged = merged.signal !== existing.signal || JSON.stringify(merged.query) !== JSON.stringify(existing.query) || merged.threshold !== existing.threshold || merged.windowMins !== existing.windowMins || merged.comparison !== existing.comparison || merged.enabled !== false !== existing.enabled || (merged.analysisId ?? null) !== (existing.analysisId ?? null) || pendingSecs !== (existing.pendingSecs ?? 0);
|
|
13627
|
+
if (evaluationChanged) await this.resetPending(id, projectId);
|
|
13033
13628
|
return this.getRule(id, projectId);
|
|
13034
13629
|
}
|
|
13630
|
+
/**
|
|
13631
|
+
* Clears a rule's pending period. Tolerates databases that have not
|
|
13632
|
+
* applied 055/058 (there is nothing to clear on them).
|
|
13633
|
+
*/
|
|
13634
|
+
async resetPending(ruleId, projectId) {
|
|
13635
|
+
try {
|
|
13636
|
+
await this.db.prepare(
|
|
13637
|
+
`UPDATE alert_state SET pending_since = NULL, pending_seen_at = NULL
|
|
13638
|
+
WHERE rule_id = ? AND project_id = ?`
|
|
13639
|
+
).bind(ruleId, projectId).run();
|
|
13640
|
+
} catch (err) {
|
|
13641
|
+
if (!isMissingColumn(err, "pending_seen_at")) {
|
|
13642
|
+
if (isMissingColumn(err, "pending_since")) return;
|
|
13643
|
+
throw err;
|
|
13644
|
+
}
|
|
13645
|
+
try {
|
|
13646
|
+
await this.db.prepare(
|
|
13647
|
+
`UPDATE alert_state SET pending_since = NULL
|
|
13648
|
+
WHERE rule_id = ? AND project_id = ?`
|
|
13649
|
+
).bind(ruleId, projectId).run();
|
|
13650
|
+
} catch (inner) {
|
|
13651
|
+
if (!isMissingColumn(inner, "pending_since")) throw inner;
|
|
13652
|
+
}
|
|
13653
|
+
}
|
|
13654
|
+
}
|
|
13655
|
+
/**
|
|
13656
|
+
* Runs `current`; on a database that has not applied migration 054 (no
|
|
13657
|
+
* `alert_rules.pending_secs`) a rule without a pending duration is
|
|
13658
|
+
* written with the legacy statement so rolling deploys keep working. A
|
|
13659
|
+
* rule that needs the column is refused with a clear message instead of
|
|
13660
|
+
* silently losing its pending duration.
|
|
13661
|
+
*/
|
|
13662
|
+
async withPendingColumnFallback(pendingSecs, current, legacy) {
|
|
13663
|
+
try {
|
|
13664
|
+
return await current();
|
|
13665
|
+
} catch (err) {
|
|
13666
|
+
if (!isMissingColumn(err, "pending_secs")) throw err;
|
|
13667
|
+
if (pendingSecs > 0)
|
|
13668
|
+
throw new Error(
|
|
13669
|
+
"pendingSecs needs database migration 054 (alert_rules.pending_secs); apply migrations and retry"
|
|
13670
|
+
);
|
|
13671
|
+
return legacy();
|
|
13672
|
+
}
|
|
13673
|
+
}
|
|
13035
13674
|
async deleteRule(id, projectId) {
|
|
13036
13675
|
const owned = await this.db.prepare(
|
|
13037
13676
|
`SELECT 1 AS ok FROM alert_rules WHERE id = ? AND project_id = ?`
|
|
@@ -13102,11 +13741,32 @@ var AlertsStore = class {
|
|
|
13102
13741
|
throw err;
|
|
13103
13742
|
}
|
|
13104
13743
|
}
|
|
13105
|
-
|
|
13744
|
+
/**
|
|
13745
|
+
* Record (`{ since, seenAt }`) or clear (`null`) the pending period of a
|
|
13746
|
+
* rule that has a pending duration (migrations 055, 058): when the
|
|
13747
|
+
* condition started holding, and the last evaluation that saw it hold.
|
|
13748
|
+
* A rule without a state row gets one in state `ok` since its creation,
|
|
13749
|
+
* already delivered (nothing is owed for a rule that never changed
|
|
13750
|
+
* state).
|
|
13751
|
+
*/
|
|
13752
|
+
async setPending(ruleId, projectId, pending, ruleCreatedAt) {
|
|
13106
13753
|
await this.db.prepare(
|
|
13107
|
-
`INSERT INTO
|
|
13108
|
-
VALUES (?, ?,
|
|
13754
|
+
`INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change, notified, pending_since, pending_seen_at)
|
|
13755
|
+
VALUES (?, ?, 'ok', ?, 1, ?, ?)
|
|
13756
|
+
ON CONFLICT(rule_id) DO UPDATE SET
|
|
13757
|
+
pending_since = excluded.pending_since,
|
|
13758
|
+
pending_seen_at = excluded.pending_seen_at`
|
|
13109
13759
|
).bind(
|
|
13760
|
+
ruleId,
|
|
13761
|
+
projectId,
|
|
13762
|
+
ruleCreatedAt,
|
|
13763
|
+
pending?.since ?? null,
|
|
13764
|
+
pending?.seenAt ?? null
|
|
13765
|
+
).run();
|
|
13766
|
+
}
|
|
13767
|
+
async recordEvaluation(ruleId, projectId, value, state, notified, outcome = "value", errorMessage) {
|
|
13768
|
+
const error = outcome === "error" && errorMessage ? sanitizeEvaluationError(errorMessage) : null;
|
|
13769
|
+
const values = [
|
|
13110
13770
|
randomHex(16),
|
|
13111
13771
|
ruleId,
|
|
13112
13772
|
projectId,
|
|
@@ -13114,15 +13774,43 @@ var AlertsStore = class {
|
|
|
13114
13774
|
value,
|
|
13115
13775
|
state,
|
|
13116
13776
|
notified ? 1 : 0
|
|
13117
|
-
|
|
13777
|
+
];
|
|
13778
|
+
try {
|
|
13779
|
+
await this.db.prepare(
|
|
13780
|
+
`INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified, outcome, error_message)
|
|
13781
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
13782
|
+
).bind(...values, outcome, error).run();
|
|
13783
|
+
return;
|
|
13784
|
+
} catch (err) {
|
|
13785
|
+
if (!isMissingColumn(err, "error_message")) {
|
|
13786
|
+
if (!isMissingColumn(err, "outcome")) throw err;
|
|
13787
|
+
} else {
|
|
13788
|
+
try {
|
|
13789
|
+
await this.db.prepare(
|
|
13790
|
+
`INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified, outcome)
|
|
13791
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
13792
|
+
).bind(...values, outcome).run();
|
|
13793
|
+
return;
|
|
13794
|
+
} catch (inner) {
|
|
13795
|
+
if (!isMissingColumn(inner, "outcome")) throw inner;
|
|
13796
|
+
}
|
|
13797
|
+
}
|
|
13798
|
+
await this.db.prepare(
|
|
13799
|
+
`INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified)
|
|
13800
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
13801
|
+
).bind(...values).run();
|
|
13802
|
+
}
|
|
13118
13803
|
}
|
|
13119
13804
|
async listEvaluations(params) {
|
|
13120
13805
|
const cutoff = new Date(
|
|
13121
13806
|
Date.now() - params.hours * 60 * 60 * 1e3
|
|
13122
13807
|
).toISOString();
|
|
13123
|
-
const
|
|
13808
|
+
const limit = Math.min(params.limit ?? 500, 1e3);
|
|
13809
|
+
const rs = params.projectId ? await this.db.prepare(
|
|
13810
|
+
`SELECT * FROM alert_evaluations WHERE rule_id = ? AND project_id = ? AND evaluated_at >= ? ORDER BY evaluated_at DESC LIMIT ?`
|
|
13811
|
+
).bind(params.ruleId, params.projectId, cutoff, limit).all() : await this.db.prepare(
|
|
13124
13812
|
`SELECT * FROM alert_evaluations WHERE rule_id = ? AND evaluated_at >= ? ORDER BY evaluated_at DESC LIMIT ?`
|
|
13125
|
-
).bind(params.ruleId, cutoff,
|
|
13813
|
+
).bind(params.ruleId, cutoff, limit).all();
|
|
13126
13814
|
const rows = rs.results ?? [];
|
|
13127
13815
|
const projectId = params.projectId ?? rows[0]?.project_id;
|
|
13128
13816
|
const rule = projectId ? await this.getRule(params.ruleId, projectId) : null;
|
|
@@ -13136,7 +13824,21 @@ var AlertsStore = class {
|
|
|
13136
13824
|
return response;
|
|
13137
13825
|
}
|
|
13138
13826
|
// ── Evaluators (count-over-window per signal) ──
|
|
13139
|
-
async evaluateRule(rule) {
|
|
13827
|
+
async evaluateRule(rule, now = /* @__PURE__ */ new Date()) {
|
|
13828
|
+
return (await this.evaluateRuleWithStatus(rule, now)).value;
|
|
13829
|
+
}
|
|
13830
|
+
/** Like `evaluateRule`, plus whether a metrics rule found no fresh data. */
|
|
13831
|
+
async evaluateRuleWithStatus(rule, now = /* @__PURE__ */ new Date()) {
|
|
13832
|
+
if (rule.signal === "metrics" && !rule.analysisId)
|
|
13833
|
+
return evaluateMetricQuery(this.db, rule.query, {
|
|
13834
|
+
projectId: rule.projectId,
|
|
13835
|
+
windowMins: rule.windowMins,
|
|
13836
|
+
ruleCreatedAt: rule.createdAt,
|
|
13837
|
+
now
|
|
13838
|
+
});
|
|
13839
|
+
return { value: await this.evaluateCountRule(rule), noData: false };
|
|
13840
|
+
}
|
|
13841
|
+
async evaluateCountRule(rule) {
|
|
13140
13842
|
if (rule.analysisId) {
|
|
13141
13843
|
const row = await this.db.prepare(
|
|
13142
13844
|
`SELECT primary_value FROM analysis_results
|
|
@@ -13154,6 +13856,12 @@ var AlertsStore = class {
|
|
|
13154
13856
|
return this.evaluateUsageRule(rule);
|
|
13155
13857
|
case "ai":
|
|
13156
13858
|
return this.evaluateAIRule(rule);
|
|
13859
|
+
case "metrics":
|
|
13860
|
+
throw new Error(
|
|
13861
|
+
"metrics rules evaluate through evaluateRuleWithStatus"
|
|
13862
|
+
);
|
|
13863
|
+
default:
|
|
13864
|
+
throw new Error(`unknown alert signal: ${String(rule.signal)}`);
|
|
13157
13865
|
}
|
|
13158
13866
|
}
|
|
13159
13867
|
/**
|
|
@@ -13239,14 +13947,46 @@ var AlertsStore = class {
|
|
|
13239
13947
|
return row?.c ?? 0;
|
|
13240
13948
|
}
|
|
13241
13949
|
};
|
|
13242
|
-
var isMissingNotifiedColumn = (err) =>
|
|
13950
|
+
var isMissingNotifiedColumn = (err) => isMissingColumn(err, "notified");
|
|
13951
|
+
var isMissingColumn = (err, column) => {
|
|
13243
13952
|
const message = err instanceof Error ? err.message : String(err);
|
|
13244
|
-
return
|
|
13953
|
+
return message.toLowerCase().includes(column) && /no such column|has no column|does not exist/i.test(message);
|
|
13245
13954
|
};
|
|
13246
13955
|
|
|
13247
13956
|
// src/plugins/alerts-evaluator.ts
|
|
13248
13957
|
var DEFAULT_RULE_CONCURRENCY = 5;
|
|
13249
13958
|
var DEFAULT_RULE_TIMEOUT_MS = 1e4;
|
|
13959
|
+
var DEFAULT_MAX_EVALUATION_GAP_MS = 2 * 5 * 6e4;
|
|
13960
|
+
async function fireEmail(channel, payload, logger, tracer, delivery = {}) {
|
|
13961
|
+
const transport = delivery.email;
|
|
13962
|
+
if (!transport) {
|
|
13963
|
+
logger.error(
|
|
13964
|
+
"[alerts-evaluator] email channel has no SMTP transport on this collector",
|
|
13965
|
+
{ rule_id: payload.rule.id, recipients: channel.to.length }
|
|
13966
|
+
);
|
|
13967
|
+
return false;
|
|
13968
|
+
}
|
|
13969
|
+
const exec = async (span) => {
|
|
13970
|
+
try {
|
|
13971
|
+
await transport.send({ to: channel.to, ...formatAlertEmail(payload) });
|
|
13972
|
+
if (span) span.setAttribute("alert.email.sent", true);
|
|
13973
|
+
return true;
|
|
13974
|
+
} catch (err) {
|
|
13975
|
+
if (span) span.setAttribute("alert.email.sent", false);
|
|
13976
|
+
logger.error("[alerts-evaluator] email delivery failed", {
|
|
13977
|
+
rule_id: payload.rule.id,
|
|
13978
|
+
recipients: channel.to.length,
|
|
13979
|
+
error: err instanceof Error ? err.message : String(err)
|
|
13980
|
+
});
|
|
13981
|
+
return false;
|
|
13982
|
+
}
|
|
13983
|
+
};
|
|
13984
|
+
if (!tracer) return exec(null);
|
|
13985
|
+
return tracer("email.alert", async (span) => exec(span), {
|
|
13986
|
+
"alert.state": payload.state,
|
|
13987
|
+
"alert.email.recipients": channel.to.length
|
|
13988
|
+
});
|
|
13989
|
+
}
|
|
13250
13990
|
async function fireWebhook(channel, payload, logger, tracer, delivery = {}) {
|
|
13251
13991
|
const target = redactWebhookUrl(channel.url);
|
|
13252
13992
|
try {
|
|
@@ -13318,7 +14058,23 @@ async function fireWebhook(channel, payload, logger, tracer, delivery = {}) {
|
|
|
13318
14058
|
async function fireChannels(channels, payload, logger, tracer, delivery = {}) {
|
|
13319
14059
|
if (channels.length === 0) return true;
|
|
13320
14060
|
const results = await Promise.all(
|
|
13321
|
-
channels.
|
|
14061
|
+
channels.map((ch) => {
|
|
14062
|
+
if (ch.type === "webhook")
|
|
14063
|
+
return fireWebhook(
|
|
14064
|
+
ch,
|
|
14065
|
+
payload,
|
|
14066
|
+
logger,
|
|
14067
|
+
tracer,
|
|
14068
|
+
delivery
|
|
14069
|
+
);
|
|
14070
|
+
if (ch.type === "email")
|
|
14071
|
+
return fireEmail(ch, payload, logger, tracer, delivery);
|
|
14072
|
+
logger.error("[alerts-evaluator] unknown channel type", {
|
|
14073
|
+
rule_id: payload.rule.id,
|
|
14074
|
+
type: String(ch.type)
|
|
14075
|
+
});
|
|
14076
|
+
return Promise.resolve(false);
|
|
14077
|
+
})
|
|
13322
14078
|
);
|
|
13323
14079
|
return results.every(Boolean);
|
|
13324
14080
|
}
|
|
@@ -13349,19 +14105,47 @@ async function mapWithConcurrency(items, concurrency, mapper) {
|
|
|
13349
14105
|
);
|
|
13350
14106
|
return results;
|
|
13351
14107
|
}
|
|
13352
|
-
async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery, onAlertFiring) {
|
|
14108
|
+
async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery, hold, onAlertFiring) {
|
|
13353
14109
|
const none = { fired: 0, resolved: 0, redelivered: 0 };
|
|
14110
|
+
let previousForError = null;
|
|
13354
14111
|
try {
|
|
13355
|
-
const
|
|
13356
|
-
|
|
14112
|
+
const priorState = await store.getState(rule.id, rule.projectId);
|
|
14113
|
+
const previous = priorState?.current_state ?? "ok";
|
|
14114
|
+
previousForError = previous;
|
|
14115
|
+
const raw = await withTimeout(
|
|
14116
|
+
store.evaluateRuleWithStatus ? store.evaluateRuleWithStatus(rule) : store.evaluateRule(rule),
|
|
13357
14117
|
ruleTimeoutMs,
|
|
13358
14118
|
`rule evaluation timed out after ${ruleTimeoutMs}ms`
|
|
13359
14119
|
);
|
|
13360
|
-
const
|
|
13361
|
-
const priorState = await store.getState(rule.id, rule.projectId);
|
|
13362
|
-
const previous = priorState?.current_state ?? "ok";
|
|
13363
|
-
const next = shouldFire ? "firing" : "ok";
|
|
14120
|
+
const { value, noData } = typeof raw === "number" ? { value: raw, noData: false } : raw;
|
|
13364
14121
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14122
|
+
const nowMs = Date.parse(now);
|
|
14123
|
+
const keepOnNoData = noData && rule.signal === "metrics" && rule.query.noData === "keep";
|
|
14124
|
+
const matches = !keepOnNoData && compareValue(value, rule.threshold, rule.comparison);
|
|
14125
|
+
const pendingSecs = rule.pendingSecs ?? 0;
|
|
14126
|
+
const pendingSince = priorState?.pending_since ?? null;
|
|
14127
|
+
const seenAt = priorState?.pending_seen_at ?? pendingSince;
|
|
14128
|
+
const holdBroken = pendingSince !== null && (seenAt === null || nowMs - Date.parse(seenAt) > hold.maxEvaluationGapMs || hold.holdStartedAt !== void 0 && seenAt < hold.holdStartedAt);
|
|
14129
|
+
let pendingActive = pendingSince !== null;
|
|
14130
|
+
let shouldFire = matches;
|
|
14131
|
+
if (matches && previous === "ok" && pendingSecs > 0) {
|
|
14132
|
+
const since = holdBroken || pendingSince === null ? now : pendingSince;
|
|
14133
|
+
await store.setPending(
|
|
14134
|
+
rule.id,
|
|
14135
|
+
rule.projectId,
|
|
14136
|
+
{ since, seenAt: now },
|
|
14137
|
+
rule.createdAt
|
|
14138
|
+
);
|
|
14139
|
+
pendingActive = true;
|
|
14140
|
+
shouldFire = nowMs - Date.parse(since) >= pendingSecs * 1e3;
|
|
14141
|
+
}
|
|
14142
|
+
const next = keepOnNoData ? previous : shouldFire ? "firing" : "ok";
|
|
14143
|
+
const clearPending = async () => {
|
|
14144
|
+
if (pendingActive && (!matches || next === "firing")) {
|
|
14145
|
+
await store.setPending(rule.id, rule.projectId, null, rule.createdAt);
|
|
14146
|
+
}
|
|
14147
|
+
};
|
|
14148
|
+
const outcome = keepOnNoData ? "no_data" : "value";
|
|
13365
14149
|
const analysisAttachment = rule.analysisId ? await store.getAnalysisNarrative(rule.projectId, rule.analysisId).then(
|
|
13366
14150
|
(n) => n ? {
|
|
13367
14151
|
id: rule.analysisId,
|
|
@@ -13387,6 +14171,7 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
|
|
|
13387
14171
|
);
|
|
13388
14172
|
if (previous !== next) {
|
|
13389
14173
|
await store.transitionState(rule.id, rule.projectId, next, now);
|
|
14174
|
+
await clearPending();
|
|
13390
14175
|
if (next === "firing" && onAlertFiring) {
|
|
13391
14176
|
await onAlertFiring(rule).catch(() => void 0);
|
|
13392
14177
|
}
|
|
@@ -13394,7 +14179,14 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
|
|
|
13394
14179
|
if (ok) {
|
|
13395
14180
|
await store.markNotified(rule.id, rule.projectId, next, now);
|
|
13396
14181
|
}
|
|
13397
|
-
await store.recordEvaluation(
|
|
14182
|
+
await store.recordEvaluation(
|
|
14183
|
+
rule.id,
|
|
14184
|
+
rule.projectId,
|
|
14185
|
+
value,
|
|
14186
|
+
next,
|
|
14187
|
+
ok,
|
|
14188
|
+
outcome
|
|
14189
|
+
);
|
|
13398
14190
|
if (!ok) {
|
|
13399
14191
|
logger.warn(
|
|
13400
14192
|
"[alerts-evaluator] state recorded but delivery failed; will retry",
|
|
@@ -13403,6 +14195,7 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
|
|
|
13403
14195
|
}
|
|
13404
14196
|
return next === "firing" ? { fired: 1, resolved: 0, redelivered: 0 } : { fired: 0, resolved: 1, redelivered: 0 };
|
|
13405
14197
|
}
|
|
14198
|
+
await clearPending();
|
|
13406
14199
|
const owed = priorState !== null && priorState.notified === 0;
|
|
13407
14200
|
if (owed) {
|
|
13408
14201
|
const ok = await deliver(previous);
|
|
@@ -13414,7 +14207,14 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
|
|
|
13414
14207
|
priorState.last_state_change
|
|
13415
14208
|
);
|
|
13416
14209
|
}
|
|
13417
|
-
await store.recordEvaluation(
|
|
14210
|
+
await store.recordEvaluation(
|
|
14211
|
+
rule.id,
|
|
14212
|
+
rule.projectId,
|
|
14213
|
+
value,
|
|
14214
|
+
next,
|
|
14215
|
+
ok,
|
|
14216
|
+
outcome
|
|
14217
|
+
);
|
|
13418
14218
|
if (!ok) {
|
|
13419
14219
|
logger.warn("[alerts-evaluator] delivery retry failed; will retry", {
|
|
13420
14220
|
rule_id: rule.id,
|
|
@@ -13425,13 +14225,32 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
|
|
|
13425
14225
|
}
|
|
13426
14226
|
return { fired: 0, resolved: 0, redelivered: 1 };
|
|
13427
14227
|
}
|
|
13428
|
-
await store.recordEvaluation(
|
|
14228
|
+
await store.recordEvaluation(
|
|
14229
|
+
rule.id,
|
|
14230
|
+
rule.projectId,
|
|
14231
|
+
value,
|
|
14232
|
+
next,
|
|
14233
|
+
false,
|
|
14234
|
+
outcome
|
|
14235
|
+
);
|
|
13429
14236
|
} catch (err) {
|
|
14237
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13430
14238
|
logger.error("[alerts-evaluator] rule evaluation failed", {
|
|
13431
14239
|
rule_id: rule.id,
|
|
13432
14240
|
project_id: rule.projectId,
|
|
13433
|
-
error:
|
|
14241
|
+
error: message
|
|
13434
14242
|
});
|
|
14243
|
+
if (previousForError !== null) {
|
|
14244
|
+
await store.recordEvaluation(
|
|
14245
|
+
rule.id,
|
|
14246
|
+
rule.projectId,
|
|
14247
|
+
0,
|
|
14248
|
+
previousForError,
|
|
14249
|
+
false,
|
|
14250
|
+
"error",
|
|
14251
|
+
message
|
|
14252
|
+
).catch(() => void 0);
|
|
14253
|
+
}
|
|
13435
14254
|
}
|
|
13436
14255
|
return none;
|
|
13437
14256
|
}
|
|
@@ -13439,7 +14258,8 @@ async function evaluateRuleBatch(store, rules, options) {
|
|
|
13439
14258
|
const logger = options?.logger ?? consoleLogger;
|
|
13440
14259
|
const delivery = {
|
|
13441
14260
|
allowPrivateWebhookTargets: options?.allowPrivateWebhookTargets,
|
|
13442
|
-
hostLookup: options?.hostLookup
|
|
14261
|
+
hostLookup: options?.hostLookup,
|
|
14262
|
+
email: options?.email
|
|
13443
14263
|
};
|
|
13444
14264
|
const concurrency = Math.max(
|
|
13445
14265
|
1,
|
|
@@ -13459,6 +14279,10 @@ async function evaluateRuleBatch(store, rules, options) {
|
|
|
13459
14279
|
options?.tracer,
|
|
13460
14280
|
ruleTimeoutMs,
|
|
13461
14281
|
delivery,
|
|
14282
|
+
{
|
|
14283
|
+
maxEvaluationGapMs: options?.maxEvaluationGapMs ?? DEFAULT_MAX_EVALUATION_GAP_MS,
|
|
14284
|
+
holdStartedAt: options?.holdStartedAt
|
|
14285
|
+
},
|
|
13462
14286
|
options?.onAlertFiring
|
|
13463
14287
|
)
|
|
13464
14288
|
);
|
|
@@ -13474,7 +14298,10 @@ async function evaluateAllRules(env, options) {
|
|
|
13474
14298
|
const tracer = options?.tracer;
|
|
13475
14299
|
const allowPrivateWebhookTargets = env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true";
|
|
13476
14300
|
const db = sqlDbFor(env);
|
|
13477
|
-
const store = new AlertsStore(db, {
|
|
14301
|
+
const store = new AlertsStore(db, {
|
|
14302
|
+
allowPrivateWebhookTargets,
|
|
14303
|
+
emailDeliveryAvailable: Boolean(env.ALERT_EMAIL)
|
|
14304
|
+
});
|
|
13478
14305
|
const rules = await store.listEnabledRules();
|
|
13479
14306
|
return evaluateRuleBatch(store, rules, {
|
|
13480
14307
|
logger,
|
|
@@ -13483,6 +14310,9 @@ async function evaluateAllRules(env, options) {
|
|
|
13483
14310
|
ruleTimeoutMs: options?.ruleTimeoutMs,
|
|
13484
14311
|
allowPrivateWebhookTargets,
|
|
13485
14312
|
hostLookup: options?.hostLookup,
|
|
14313
|
+
email: env.ALERT_EMAIL,
|
|
14314
|
+
maxEvaluationGapMs: options?.maxEvaluationGapMs,
|
|
14315
|
+
holdStartedAt: options?.holdStartedAt,
|
|
13486
14316
|
onAlertFiring: async (rule) => {
|
|
13487
14317
|
const target = await resolveServiceIncidentTarget(
|
|
13488
14318
|
db,
|
|
@@ -13511,7 +14341,8 @@ function createAlertEvaluatorHandler() {
|
|
|
13511
14341
|
// src/plugins/alerts-routes.ts
|
|
13512
14342
|
import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT5 } from "@obsunified/types";
|
|
13513
14343
|
var storeOptions = (env) => ({
|
|
13514
|
-
allowPrivateWebhookTargets: env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true"
|
|
14344
|
+
allowPrivateWebhookTargets: env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
|
|
14345
|
+
emailDeliveryAvailable: Boolean(env.ALERT_EMAIL)
|
|
13515
14346
|
});
|
|
13516
14347
|
var alertsRoutesPlugin = {
|
|
13517
14348
|
name: "alerts-routes",
|
|
@@ -13588,9 +14419,19 @@ var alertsRoutesPlugin = {
|
|
|
13588
14419
|
const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
|
|
13589
14420
|
const rule = await store.getRule(id, projectId);
|
|
13590
14421
|
if (!rule) return c.json({ error: "Rule not found" }, 404);
|
|
13591
|
-
|
|
14422
|
+
let value;
|
|
14423
|
+
let noData;
|
|
14424
|
+
try {
|
|
14425
|
+
({ value, noData } = await store.evaluateRuleWithStatus(rule));
|
|
14426
|
+
} catch (err) {
|
|
14427
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14428
|
+
return c.json({ error: message }, 422);
|
|
14429
|
+
}
|
|
13592
14430
|
const response = {
|
|
13593
14431
|
value,
|
|
14432
|
+
noData,
|
|
14433
|
+
// The bare comparison: a rule with `pendingSecs` additionally
|
|
14434
|
+
// needs the condition to hold that long before it fires.
|
|
13594
14435
|
wouldFire: compareValue(value, rule.threshold, rule.comparison),
|
|
13595
14436
|
comparison: rule.comparison,
|
|
13596
14437
|
threshold: rule.threshold,
|
|
@@ -16923,6 +17764,7 @@ var filterPprofByTraceId = async (profile, traceIdFilter) => {
|
|
|
16923
17764
|
var DEFAULT_CHUNK_LIMIT = 100;
|
|
16924
17765
|
var MAX_CHUNK_LIMIT = 500;
|
|
16925
17766
|
var REPLAY_CHUNK_FETCH_CONCURRENCY = 8;
|
|
17767
|
+
var REPLAY_RESPONSE_BYTE_BUDGET = 64 * 1024 * 1024;
|
|
16926
17768
|
async function mapWithConcurrency2(items, concurrency, mapper) {
|
|
16927
17769
|
const results = new Array(items.length);
|
|
16928
17770
|
let nextIndex = 0;
|
|
@@ -16937,13 +17779,30 @@ async function mapWithConcurrency2(items, concurrency, mapper) {
|
|
|
16937
17779
|
);
|
|
16938
17780
|
return results;
|
|
16939
17781
|
}
|
|
17782
|
+
function selectReplayChunkPage(objects, offset, limit, byteBudget = REPLAY_RESPONSE_BYTE_BUDGET) {
|
|
17783
|
+
const window = objects.slice(offset, offset + limit);
|
|
17784
|
+
let bytes = 0;
|
|
17785
|
+
let count = 0;
|
|
17786
|
+
for (const obj of window) {
|
|
17787
|
+
const size = typeof obj.size === "number" && obj.size > 0 ? obj.size : 0;
|
|
17788
|
+
if (count > 0 && bytes + size > byteBudget) break;
|
|
17789
|
+
bytes += size;
|
|
17790
|
+
count++;
|
|
17791
|
+
}
|
|
17792
|
+
const selected = window.slice(0, count);
|
|
17793
|
+
const end = offset + count;
|
|
17794
|
+
return {
|
|
17795
|
+
selected,
|
|
17796
|
+
nextChunkOffset: end < objects.length ? end : null
|
|
17797
|
+
};
|
|
17798
|
+
}
|
|
16940
17799
|
async function fetchReplayChunks(bucket, objects, concurrency = REPLAY_CHUNK_FETCH_CONCURRENCY) {
|
|
16941
|
-
const
|
|
17800
|
+
const chunks2 = await mapWithConcurrency2(objects, concurrency, async (obj) => {
|
|
16942
17801
|
const objectData = await bucket.get(obj.key);
|
|
16943
17802
|
if (!objectData) return [];
|
|
16944
17803
|
return objectData.json();
|
|
16945
17804
|
});
|
|
16946
|
-
return
|
|
17805
|
+
return chunks2.flat();
|
|
16947
17806
|
}
|
|
16948
17807
|
var replayQueryRoutesPlugin = {
|
|
16949
17808
|
name: "replay-query-routes",
|
|
@@ -17003,15 +17862,11 @@ var replayQueryRoutesPlugin = {
|
|
|
17003
17862
|
) || DEFAULT_CHUNK_LIMIT
|
|
17004
17863
|
)
|
|
17005
17864
|
);
|
|
17006
|
-
const selectedObjects = objects
|
|
17007
|
-
chunkOffset,
|
|
17008
|
-
chunkOffset + chunkLimit
|
|
17009
|
-
);
|
|
17865
|
+
const { selected: selectedObjects, nextChunkOffset } = selectReplayChunkPage(objects, chunkOffset, chunkLimit);
|
|
17010
17866
|
const events = await fetchReplayChunks(
|
|
17011
17867
|
c.env.REPLAYS_BUCKET,
|
|
17012
17868
|
selectedObjects
|
|
17013
17869
|
);
|
|
17014
|
-
const nextChunkOffset = chunkOffset + chunkLimit < objects.length ? chunkOffset + chunkLimit : null;
|
|
17015
17870
|
return c.json({
|
|
17016
17871
|
metadata,
|
|
17017
17872
|
events,
|
|
@@ -19870,7 +20725,7 @@ var gunzip = async (input, maxBytes) => {
|
|
|
19870
20725
|
);
|
|
19871
20726
|
if (!stream) throw new OtlpDecodeError("Empty gzip body", 400);
|
|
19872
20727
|
const reader = stream.getReader();
|
|
19873
|
-
const
|
|
20728
|
+
const chunks2 = [];
|
|
19874
20729
|
let total = 0;
|
|
19875
20730
|
try {
|
|
19876
20731
|
for (; ; ) {
|
|
@@ -19885,7 +20740,7 @@ var gunzip = async (input, maxBytes) => {
|
|
|
19885
20740
|
413
|
|
19886
20741
|
);
|
|
19887
20742
|
}
|
|
19888
|
-
|
|
20743
|
+
chunks2.push(value);
|
|
19889
20744
|
}
|
|
19890
20745
|
} catch (err) {
|
|
19891
20746
|
if (err instanceof OtlpDecodeError) throw err;
|
|
@@ -19893,7 +20748,7 @@ var gunzip = async (input, maxBytes) => {
|
|
|
19893
20748
|
}
|
|
19894
20749
|
const out = new Uint8Array(total);
|
|
19895
20750
|
let offset = 0;
|
|
19896
|
-
for (const chunk of
|
|
20751
|
+
for (const chunk of chunks2) {
|
|
19897
20752
|
out.set(chunk, offset);
|
|
19898
20753
|
offset += chunk.byteLength;
|
|
19899
20754
|
}
|
|
@@ -20022,19 +20877,34 @@ var anyValueToString = (v) => {
|
|
|
20022
20877
|
return bytesToBase64(v.value.value);
|
|
20023
20878
|
case "arrayValue":
|
|
20024
20879
|
case "kvlistValue":
|
|
20025
|
-
return JSON.stringify(
|
|
20880
|
+
return JSON.stringify(adaptAnyValueAt(v, 1));
|
|
20026
20881
|
default:
|
|
20027
20882
|
return "";
|
|
20028
20883
|
}
|
|
20029
20884
|
};
|
|
20030
|
-
var
|
|
20885
|
+
var MAX_ANY_VALUE_DEPTH = 64;
|
|
20886
|
+
var TRUNCATED_DEPTH_MARKER = `[truncated: nested deeper than ${MAX_ANY_VALUE_DEPTH} levels]`;
|
|
20887
|
+
var setOwn = (out, key, value) => {
|
|
20888
|
+
if (key === "__proto__") {
|
|
20889
|
+
Object.defineProperty(out, key, {
|
|
20890
|
+
value,
|
|
20891
|
+
enumerable: true,
|
|
20892
|
+
writable: true,
|
|
20893
|
+
configurable: true
|
|
20894
|
+
});
|
|
20895
|
+
} else {
|
|
20896
|
+
out[key] = value;
|
|
20897
|
+
}
|
|
20898
|
+
};
|
|
20899
|
+
var keyValuesToRecord = (kvs) => keyValuesToRecordAt(kvs, 0);
|
|
20900
|
+
var keyValuesToRecordAt = (kvs, depth) => {
|
|
20031
20901
|
const out = {};
|
|
20032
20902
|
for (const kv of kvs) {
|
|
20033
|
-
out
|
|
20903
|
+
setOwn(out, kv.key, anyValueToJson(kv.value, depth + 1));
|
|
20034
20904
|
}
|
|
20035
20905
|
return out;
|
|
20036
20906
|
};
|
|
20037
|
-
var anyValueToJson = (v) => {
|
|
20907
|
+
var anyValueToJson = (v, depth) => {
|
|
20038
20908
|
if (!v) return null;
|
|
20039
20909
|
switch (v.value.case) {
|
|
20040
20910
|
case "stringValue":
|
|
@@ -20048,18 +20918,21 @@ var anyValueToJson = (v) => {
|
|
|
20048
20918
|
case "bytesValue":
|
|
20049
20919
|
return bytesToBase64(v.value.value);
|
|
20050
20920
|
case "arrayValue":
|
|
20051
|
-
|
|
20921
|
+
if (depth > MAX_ANY_VALUE_DEPTH) return TRUNCATED_DEPTH_MARKER;
|
|
20922
|
+
return v.value.value.values.map((x) => anyValueToJson(x, depth + 1));
|
|
20052
20923
|
case "kvlistValue":
|
|
20053
|
-
|
|
20924
|
+
if (depth > MAX_ANY_VALUE_DEPTH) return TRUNCATED_DEPTH_MARKER;
|
|
20925
|
+
return keyValuesToRecordAt(v.value.value.values, depth);
|
|
20054
20926
|
default:
|
|
20055
20927
|
return null;
|
|
20056
20928
|
}
|
|
20057
20929
|
};
|
|
20058
|
-
var adaptKeyValue = (kv) => (
|
|
20930
|
+
var adaptKeyValue = (kv) => adaptKeyValueAt(kv, 0);
|
|
20931
|
+
var adaptKeyValueAt = (kv, depth) => ({
|
|
20059
20932
|
key: kv.key,
|
|
20060
|
-
value: kv.value ?
|
|
20933
|
+
value: kv.value ? adaptAnyValueAt(kv.value, depth + 1) : void 0
|
|
20061
20934
|
});
|
|
20062
|
-
var
|
|
20935
|
+
var adaptAnyValueAt = (v, depth) => {
|
|
20063
20936
|
switch (v.value.case) {
|
|
20064
20937
|
case "stringValue":
|
|
20065
20938
|
return { stringValue: v.value.value };
|
|
@@ -20070,15 +20943,21 @@ var adaptAnyValue = (v) => {
|
|
|
20070
20943
|
case "doubleValue":
|
|
20071
20944
|
return { doubleValue: v.value.value };
|
|
20072
20945
|
case "arrayValue":
|
|
20946
|
+
if (depth > MAX_ANY_VALUE_DEPTH)
|
|
20947
|
+
return { stringValue: TRUNCATED_DEPTH_MARKER };
|
|
20073
20948
|
return {
|
|
20074
20949
|
arrayValue: {
|
|
20075
|
-
values: v.value.value.values.map(
|
|
20950
|
+
values: v.value.value.values.map(
|
|
20951
|
+
(x) => adaptAnyValueAt(x, depth + 1)
|
|
20952
|
+
)
|
|
20076
20953
|
}
|
|
20077
20954
|
};
|
|
20078
20955
|
case "kvlistValue":
|
|
20956
|
+
if (depth > MAX_ANY_VALUE_DEPTH)
|
|
20957
|
+
return { stringValue: TRUNCATED_DEPTH_MARKER };
|
|
20079
20958
|
return {
|
|
20080
20959
|
kvlistValue: {
|
|
20081
|
-
values: v.value.value.values.map(
|
|
20960
|
+
values: v.value.value.values.map((kv) => adaptKeyValueAt(kv, depth))
|
|
20082
20961
|
}
|
|
20083
20962
|
};
|
|
20084
20963
|
case "bytesValue":
|
|
@@ -20106,7 +20985,7 @@ var bytesToBase64 = (bytes) => {
|
|
|
20106
20985
|
};
|
|
20107
20986
|
|
|
20108
20987
|
// src/otlp/decode/logs.ts
|
|
20109
|
-
var
|
|
20988
|
+
var decodeLogsRequestCapped = (body, maxRecords) => {
|
|
20110
20989
|
let msg;
|
|
20111
20990
|
try {
|
|
20112
20991
|
msg = body.wireFormat === "protobuf" ? fromBinary(ExportLogsServiceRequestSchema, body.bytes) : fromJson(ExportLogsServiceRequestSchema, decodeJsonBody(body.bytes));
|
|
@@ -20117,16 +20996,25 @@ var decodeLogsRequest = (body) => {
|
|
|
20117
20996
|
);
|
|
20118
20997
|
}
|
|
20119
20998
|
const out = [];
|
|
20999
|
+
let rejected = 0;
|
|
20120
21000
|
for (const rl of msg.resourceLogs) {
|
|
21001
|
+
if (out.length >= maxRecords) {
|
|
21002
|
+
for (const sl of rl.scopeLogs) rejected += sl.logRecords.length;
|
|
21003
|
+
continue;
|
|
21004
|
+
}
|
|
20121
21005
|
const serviceName = extractServiceName(rl.resource?.attributes);
|
|
20122
21006
|
for (const sl of rl.scopeLogs) {
|
|
20123
21007
|
const loggerName = sl.scope?.name || null;
|
|
20124
21008
|
for (const log of sl.logRecords) {
|
|
21009
|
+
if (out.length >= maxRecords) {
|
|
21010
|
+
rejected++;
|
|
21011
|
+
continue;
|
|
21012
|
+
}
|
|
20125
21013
|
out.push(adaptLogRecord(log, serviceName, loggerName));
|
|
20126
21014
|
}
|
|
20127
21015
|
}
|
|
20128
21016
|
}
|
|
20129
|
-
return out;
|
|
21017
|
+
return { records: out, rejected };
|
|
20130
21018
|
};
|
|
20131
21019
|
var adaptLogRecord = (log, serviceName, loggerName) => {
|
|
20132
21020
|
const ts = log.timeUnixNano || log.observedTimeUnixNano;
|
|
@@ -20176,7 +21064,7 @@ var ExportMetricsServiceResponseSchema = /* @__PURE__ */ messageDesc6(file_opent
|
|
|
20176
21064
|
var ExportMetricsPartialSuccessSchema = /* @__PURE__ */ messageDesc6(file_opentelemetry_proto_collector_metrics_v1_metrics_service, 2);
|
|
20177
21065
|
|
|
20178
21066
|
// src/otlp/decode/metrics.ts
|
|
20179
|
-
var
|
|
21067
|
+
var decodeMetricsRequestCapped = (body, maxPoints) => {
|
|
20180
21068
|
let msg;
|
|
20181
21069
|
try {
|
|
20182
21070
|
msg = body.wireFormat === "protobuf" ? fromBinary2(ExportMetricsServiceRequestSchema, body.bytes) : fromJson2(
|
|
@@ -20190,7 +21078,18 @@ var decodeMetricsRequest = (body) => {
|
|
|
20190
21078
|
);
|
|
20191
21079
|
}
|
|
20192
21080
|
const out = [];
|
|
21081
|
+
let rejected = 0;
|
|
20193
21082
|
for (const rm of msg.resourceMetrics) {
|
|
21083
|
+
if (out.length >= maxPoints) {
|
|
21084
|
+
for (const sm of rm.scopeMetrics) rejected += countPoints(sm);
|
|
21085
|
+
continue;
|
|
21086
|
+
}
|
|
21087
|
+
const remaining = maxPoints - out.length;
|
|
21088
|
+
const total = rm.scopeMetrics.reduce((n, sm) => n + countPoints(sm), 0);
|
|
21089
|
+
if (total > remaining) {
|
|
21090
|
+
rejected += total - remaining;
|
|
21091
|
+
trimScopeMetrics(rm.scopeMetrics, remaining);
|
|
21092
|
+
}
|
|
20194
21093
|
const resourceAttrs = rm.resource?.attributes ?? [];
|
|
20195
21094
|
const resourceAttrsJson = resourceAttrs.length ? JSON.stringify(keyValuesToRecord(resourceAttrs)) : null;
|
|
20196
21095
|
const serviceName = extractServiceName(resourceAttrs);
|
|
@@ -20198,7 +21097,34 @@ var decodeMetricsRequest = (body) => {
|
|
|
20198
21097
|
addMetrics(out, rm, sm, serviceName, resourceAttrsJson);
|
|
20199
21098
|
}
|
|
20200
21099
|
}
|
|
20201
|
-
return out;
|
|
21100
|
+
return { points: out, rejected };
|
|
21101
|
+
};
|
|
21102
|
+
var metricPointCount = (metric) => {
|
|
21103
|
+
switch (metric.data.case) {
|
|
21104
|
+
case "gauge":
|
|
21105
|
+
case "sum":
|
|
21106
|
+
case "histogram":
|
|
21107
|
+
case "exponentialHistogram":
|
|
21108
|
+
case "summary":
|
|
21109
|
+
return metric.data.value.dataPoints.length;
|
|
21110
|
+
default:
|
|
21111
|
+
return 0;
|
|
21112
|
+
}
|
|
21113
|
+
};
|
|
21114
|
+
var countPoints = (sm) => sm.metrics.reduce((n, m) => n + metricPointCount(m), 0);
|
|
21115
|
+
var trimScopeMetrics = (scopeMetrics, budget) => {
|
|
21116
|
+
let left = budget;
|
|
21117
|
+
for (const sm of scopeMetrics) {
|
|
21118
|
+
for (const metric of sm.metrics) {
|
|
21119
|
+
const n = metricPointCount(metric);
|
|
21120
|
+
if (n === 0) continue;
|
|
21121
|
+
if (n > left) {
|
|
21122
|
+
const data = metric.data.value;
|
|
21123
|
+
data.dataPoints = data.dataPoints.slice(0, left);
|
|
21124
|
+
}
|
|
21125
|
+
left -= Math.min(n, left);
|
|
21126
|
+
}
|
|
21127
|
+
}
|
|
20202
21128
|
};
|
|
20203
21129
|
var addMetrics = (out, rm, sm, serviceName, resourceAttrsJson) => {
|
|
20204
21130
|
const scopeName = sm.scope?.name || null;
|
|
@@ -20442,7 +21368,31 @@ var ExportTraceServiceResponseSchema = /* @__PURE__ */ messageDesc8(file_opentel
|
|
|
20442
21368
|
var ExportTracePartialSuccessSchema = /* @__PURE__ */ messageDesc8(file_opentelemetry_proto_collector_trace_v1_trace_service, 2);
|
|
20443
21369
|
|
|
20444
21370
|
// src/otlp/decode/traces.ts
|
|
20445
|
-
var
|
|
21371
|
+
var decodeTraceRequestCapped = (body, maxSpans) => {
|
|
21372
|
+
const msg = parseTraceMessage(body);
|
|
21373
|
+
let accepted = 0;
|
|
21374
|
+
let rejected = 0;
|
|
21375
|
+
const resourceSpans = [];
|
|
21376
|
+
for (const rs of msg.resourceSpans) {
|
|
21377
|
+
for (const ss of rs.scopeSpans) {
|
|
21378
|
+
const kept = [];
|
|
21379
|
+
for (const s of ss.spans) {
|
|
21380
|
+
if (!hasValidIds(s)) continue;
|
|
21381
|
+
if (accepted >= maxSpans) {
|
|
21382
|
+
rejected++;
|
|
21383
|
+
} else {
|
|
21384
|
+
kept.push(s);
|
|
21385
|
+
accepted++;
|
|
21386
|
+
}
|
|
21387
|
+
}
|
|
21388
|
+
ss.spans = kept;
|
|
21389
|
+
}
|
|
21390
|
+
resourceSpans.push(adaptResourceSpans(rs));
|
|
21391
|
+
}
|
|
21392
|
+
return { request: { resourceSpans }, accepted, rejected };
|
|
21393
|
+
};
|
|
21394
|
+
var hasValidIds = (s) => s.traceId.length === 16 && s.spanId.length === 8;
|
|
21395
|
+
var parseTraceMessage = (body) => {
|
|
20446
21396
|
let msg;
|
|
20447
21397
|
try {
|
|
20448
21398
|
msg = body.wireFormat === "protobuf" ? fromBinary3(ExportTraceServiceRequestSchema, body.bytes) : fromJson3(ExportTraceServiceRequestSchema, decodeJsonBody(body.bytes));
|
|
@@ -20452,14 +21402,14 @@ var decodeTraceRequest = (body) => {
|
|
|
20452
21402
|
400
|
|
20453
21403
|
);
|
|
20454
21404
|
}
|
|
20455
|
-
return
|
|
21405
|
+
return msg;
|
|
20456
21406
|
};
|
|
20457
21407
|
var adaptResourceSpans = (rs) => ({
|
|
20458
21408
|
resource: rs.resource ? { attributes: rs.resource.attributes.map(adaptKeyValue) } : void 0,
|
|
20459
21409
|
scopeSpans: rs.scopeSpans.map((ss) => ({
|
|
20460
21410
|
scope: ss.scope ? { name: ss.scope.name, version: ss.scope.version } : void 0,
|
|
20461
21411
|
spans: ss.spans.flatMap((s) => {
|
|
20462
|
-
if (s
|
|
21412
|
+
if (!hasValidIds(s)) return [];
|
|
20463
21413
|
return [
|
|
20464
21414
|
{
|
|
20465
21415
|
traceId: bytesToHex(s.traceId),
|
|
@@ -20570,19 +21520,18 @@ var logsReceiverPlugin = {
|
|
|
20570
21520
|
throw err;
|
|
20571
21521
|
}
|
|
20572
21522
|
let decoded;
|
|
21523
|
+
let rejected = 0;
|
|
20573
21524
|
try {
|
|
20574
|
-
decoded =
|
|
21525
|
+
({ records: decoded, rejected } = decodeLogsRequestCapped(
|
|
21526
|
+
body,
|
|
21527
|
+
MAX_LOGS_PER_REQUEST
|
|
21528
|
+
));
|
|
20575
21529
|
} catch (err) {
|
|
20576
21530
|
if (err instanceof OtlpDecodeError) {
|
|
20577
21531
|
return c.json({ error: err.message }, err.status);
|
|
20578
21532
|
}
|
|
20579
21533
|
throw err;
|
|
20580
21534
|
}
|
|
20581
|
-
let rejected = 0;
|
|
20582
|
-
if (decoded.length > MAX_LOGS_PER_REQUEST) {
|
|
20583
|
-
rejected = decoded.length - MAX_LOGS_PER_REQUEST;
|
|
20584
|
-
decoded = decoded.slice(0, MAX_LOGS_PER_REQUEST);
|
|
20585
|
-
}
|
|
20586
21535
|
const store = new LogsStore(sqlDbFor(c.env));
|
|
20587
21536
|
const retentionHours = getConfiguredRetentionHours5(c.env.RETENTION_HOURS);
|
|
20588
21537
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -20691,19 +21640,18 @@ var metricsReceiverPlugin = {
|
|
|
20691
21640
|
throw err;
|
|
20692
21641
|
}
|
|
20693
21642
|
let points;
|
|
21643
|
+
let rejected = 0;
|
|
20694
21644
|
try {
|
|
20695
|
-
points =
|
|
21645
|
+
({ points, rejected } = decodeMetricsRequestCapped(
|
|
21646
|
+
body,
|
|
21647
|
+
MAX_POINTS_PER_REQUEST
|
|
21648
|
+
));
|
|
20696
21649
|
} catch (err) {
|
|
20697
21650
|
if (err instanceof OtlpDecodeError) {
|
|
20698
21651
|
return c.json({ error: err.message }, err.status);
|
|
20699
21652
|
}
|
|
20700
21653
|
throw err;
|
|
20701
21654
|
}
|
|
20702
|
-
let rejected = 0;
|
|
20703
|
-
if (points.length > MAX_POINTS_PER_REQUEST) {
|
|
20704
|
-
rejected = points.length - MAX_POINTS_PER_REQUEST;
|
|
20705
|
-
points = points.slice(0, MAX_POINTS_PER_REQUEST);
|
|
20706
|
-
}
|
|
20707
21655
|
const retentionHours = getConfiguredRetentionHours6(c.env.RETENTION_HOURS);
|
|
20708
21656
|
const now = /* @__PURE__ */ new Date();
|
|
20709
21657
|
const receivedAt = now.toISOString();
|
|
@@ -20960,34 +21908,18 @@ var otlpReceiverPlugin = {
|
|
|
20960
21908
|
}
|
|
20961
21909
|
throw err;
|
|
20962
21910
|
}
|
|
20963
|
-
let
|
|
21911
|
+
let capped;
|
|
20964
21912
|
try {
|
|
20965
|
-
|
|
21913
|
+
capped = decodeTraceRequestCapped(body, MAX_SPANS_PER_REQUEST);
|
|
20966
21914
|
} catch (err) {
|
|
20967
21915
|
if (err instanceof OtlpDecodeError) {
|
|
20968
21916
|
return c.json({ error: err.message }, err.status);
|
|
20969
21917
|
}
|
|
20970
21918
|
throw err;
|
|
20971
21919
|
}
|
|
20972
|
-
|
|
20973
|
-
|
|
20974
|
-
const trimmedResourceSpans =
|
|
20975
|
-
for (const rs of payload.resourceSpans ?? []) {
|
|
20976
|
-
const scopeSpans = [];
|
|
20977
|
-
for (const ss of rs.scopeSpans ?? []) {
|
|
20978
|
-
const spans2 = [];
|
|
20979
|
-
for (const s of ss.spans ?? []) {
|
|
20980
|
-
if (total >= MAX_SPANS_PER_REQUEST) {
|
|
20981
|
-
rejected++;
|
|
20982
|
-
} else {
|
|
20983
|
-
spans2.push(s);
|
|
20984
|
-
total++;
|
|
20985
|
-
}
|
|
20986
|
-
}
|
|
20987
|
-
scopeSpans.push({ ...ss, spans: spans2 });
|
|
20988
|
-
}
|
|
20989
|
-
trimmedResourceSpans.push({ ...rs, scopeSpans });
|
|
20990
|
-
}
|
|
21920
|
+
const total = capped.accepted;
|
|
21921
|
+
const rejected = capped.rejected;
|
|
21922
|
+
const trimmedResourceSpans = capped.request.resourceSpans;
|
|
20991
21923
|
const parsedSpans = toStoredSpans(
|
|
20992
21924
|
{ resourceSpans: trimmedResourceSpans },
|
|
20993
21925
|
projectId,
|
|
@@ -21190,6 +22122,7 @@ var profileId = () => {
|
|
|
21190
22122
|
for (let i = 0; i < 16; i++) randStr += ENCODING[bytes[i] % 32];
|
|
21191
22123
|
return timeStr + randStr;
|
|
21192
22124
|
};
|
|
22125
|
+
var safeFilenamePart = (value) => value.replace(/[^A-Za-z0-9_.-]/g, "_");
|
|
21193
22126
|
var parseNonNegInt = (value, fallback) => {
|
|
21194
22127
|
if (value == null) return fallback;
|
|
21195
22128
|
const parsed = Number.parseInt(value, 10);
|
|
@@ -21345,7 +22278,7 @@ var profileRoutesPlugin = {
|
|
|
21345
22278
|
headers: {
|
|
21346
22279
|
"Content-Type": "application/octet-stream",
|
|
21347
22280
|
"Content-Encoding": "gzip",
|
|
21348
|
-
"Content-Disposition": `attachment; filename="${id}-trace-${filterTraceId.slice(0, 8)}.pprof.gz"`,
|
|
22281
|
+
"Content-Disposition": `attachment; filename="${safeFilenamePart(id)}-trace-${safeFilenamePart(filterTraceId.slice(0, 8))}.pprof.gz"`,
|
|
21349
22282
|
// Surface the size shrink so the UI can show savings.
|
|
21350
22283
|
"X-Obs-Filter-Original-Bytes": String(raw.byteLength),
|
|
21351
22284
|
"X-Obs-Filter-Filtered-Bytes": String(filtered.byteLength)
|
|
@@ -21365,7 +22298,7 @@ var profileRoutesPlugin = {
|
|
|
21365
22298
|
return new Response(obj.body, {
|
|
21366
22299
|
headers: {
|
|
21367
22300
|
"Content-Type": "application/octet-stream",
|
|
21368
|
-
"Content-Disposition": `attachment; filename="${id}.pprof.gz"`
|
|
22301
|
+
"Content-Disposition": `attachment; filename="${safeFilenamePart(id)}.pprof.gz"`
|
|
21369
22302
|
}
|
|
21370
22303
|
});
|
|
21371
22304
|
}
|
|
@@ -23413,16 +24346,16 @@ var BlobStoreToR2Adapter = class {
|
|
|
23413
24346
|
};
|
|
23414
24347
|
var streamToBytes = async (stream) => {
|
|
23415
24348
|
const reader = stream.getReader();
|
|
23416
|
-
const
|
|
24349
|
+
const chunks2 = [];
|
|
23417
24350
|
while (true) {
|
|
23418
24351
|
const { value, done } = await reader.read();
|
|
23419
24352
|
if (done) break;
|
|
23420
|
-
if (value)
|
|
24353
|
+
if (value) chunks2.push(value);
|
|
23421
24354
|
}
|
|
23422
|
-
const total =
|
|
24355
|
+
const total = chunks2.reduce((sum, chunk) => sum + chunk.byteLength, 0);
|
|
23423
24356
|
const out = new Uint8Array(total);
|
|
23424
24357
|
let offset = 0;
|
|
23425
|
-
for (const chunk of
|
|
24358
|
+
for (const chunk of chunks2) {
|
|
23426
24359
|
out.set(chunk, offset);
|
|
23427
24360
|
offset += chunk.byteLength;
|
|
23428
24361
|
}
|
|
@@ -23532,16 +24465,16 @@ var toUint8Array = (value) => {
|
|
|
23532
24465
|
};
|
|
23533
24466
|
var streamToBytes2 = async (stream) => {
|
|
23534
24467
|
const reader = stream.getReader();
|
|
23535
|
-
const
|
|
24468
|
+
const chunks2 = [];
|
|
23536
24469
|
while (true) {
|
|
23537
24470
|
const { value, done } = await reader.read();
|
|
23538
24471
|
if (done) break;
|
|
23539
|
-
if (value)
|
|
24472
|
+
if (value) chunks2.push(value);
|
|
23540
24473
|
}
|
|
23541
|
-
const total =
|
|
24474
|
+
const total = chunks2.reduce((n, c) => n + c.byteLength, 0);
|
|
23542
24475
|
const out = new Uint8Array(total);
|
|
23543
24476
|
let offset = 0;
|
|
23544
|
-
for (const c of
|
|
24477
|
+
for (const c of chunks2) {
|
|
23545
24478
|
out.set(c, offset);
|
|
23546
24479
|
offset += c.byteLength;
|
|
23547
24480
|
}
|
|
@@ -23979,7 +24912,8 @@ var createDefaultCollectorApp = (config) => createTelemetryCollectorApp({
|
|
|
23979
24912
|
dashboardAuth: config?.dashboardAuth,
|
|
23980
24913
|
logger: config?.logger,
|
|
23981
24914
|
withChildSpan: config?.withChildSpan,
|
|
23982
|
-
sqlDb: config?.sqlDb
|
|
24915
|
+
sqlDb: config?.sqlDb,
|
|
24916
|
+
allowUnauthenticatedIngest: config?.allowUnauthenticatedIngest
|
|
23983
24917
|
});
|
|
23984
24918
|
var src_default = createTelemetryCollectorApp(allPlugins);
|
|
23985
24919
|
export {
|
|
@@ -24034,11 +24968,14 @@ export {
|
|
|
24034
24968
|
dispatchIncidentWebhook,
|
|
24035
24969
|
evalCasesRoutesPlugin,
|
|
24036
24970
|
evaluateAllRules,
|
|
24971
|
+
evaluateMetricQuery,
|
|
24037
24972
|
evaluateRemediationWatchdogs,
|
|
24973
|
+
evaluateRuleBatch,
|
|
24038
24974
|
evidenceRetrievalRoutesPlugin,
|
|
24039
24975
|
extractSpanLineageAndTopology,
|
|
24040
24976
|
failRemediationProposal,
|
|
24041
24977
|
findCrossTraceContention,
|
|
24978
|
+
formatAlertEmail,
|
|
24042
24979
|
genAiNormalizerPlugin,
|
|
24043
24980
|
getChangeEvent,
|
|
24044
24981
|
getIncidentInvestigation,
|
|
@@ -24048,6 +24985,7 @@ export {
|
|
|
24048
24985
|
identityReceiverPlugin,
|
|
24049
24986
|
isEvalCaseSourceType,
|
|
24050
24987
|
isEvalRunStatus,
|
|
24988
|
+
isValidEmailAddress,
|
|
24051
24989
|
issueEnrichmentPlugin,
|
|
24052
24990
|
issueInsightsPlugin,
|
|
24053
24991
|
listChangeEvents,
|
|
@@ -24069,6 +25007,7 @@ export {
|
|
|
24069
25007
|
recordChangeEvent,
|
|
24070
25008
|
recordInvestigationFinding,
|
|
24071
25009
|
redactionProcessorPlugin,
|
|
25010
|
+
reduceSeries,
|
|
24072
25011
|
rejectRemediationProposal,
|
|
24073
25012
|
replayQueryRoutesPlugin,
|
|
24074
25013
|
replayReceiverPlugin,
|
|
@@ -24083,6 +25022,7 @@ export {
|
|
|
24083
25022
|
usageReceiverPlugin,
|
|
24084
25023
|
usersQueryRoutesPlugin,
|
|
24085
25024
|
utmEnrichmentPlugin,
|
|
25025
|
+
validateMetricQuery,
|
|
24086
25026
|
verifyPairedCounterfactualProof
|
|
24087
25027
|
};
|
|
24088
25028
|
//# sourceMappingURL=index.js.map
|