@obsunified/collector 2.3.0 → 2.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +7 -3
  2. package/dist/auth.d.ts +2 -1
  3. package/dist/auth.js +1 -1
  4. package/dist/{chunk-2FALTDCE.js → chunk-VSOZJO6Y.js} +12 -1
  5. package/dist/chunk-VSOZJO6Y.js.map +1 -0
  6. package/dist/index.d.ts +271 -62
  7. package/dist/index.js +1077 -137
  8. package/dist/index.js.map +1 -1
  9. package/dist/{ingest-auth-BmP5-f96.d.ts → ingest-auth-BTowieER.d.ts} +66 -2
  10. package/package.json +3 -3
  11. package/src/framework/collector.ts +22 -0
  12. package/src/framework/env.ts +9 -1
  13. package/src/framework/ingest-auth-warning.test.ts +71 -0
  14. package/src/index.ts +15 -0
  15. package/src/lib/alert-email.ts +104 -0
  16. package/src/lib/alerts-store.test.ts +147 -2
  17. package/src/lib/alerts-store.ts +438 -63
  18. package/src/lib/metric-alerts.test.ts +769 -0
  19. package/src/lib/metric-alerts.ts +640 -0
  20. package/src/lib/metrics-store.ts +9 -2
  21. package/src/lib/projects-store.test.ts +44 -1
  22. package/src/lib/projects-store.ts +21 -0
  23. package/src/lib/sql-db-postgres.integration.test.ts +247 -0
  24. package/src/lib/store/trace-queries.ts +1 -1
  25. package/src/lib/webhook-url.test.ts +49 -0
  26. package/src/lib/webhook-url.ts +72 -14
  27. package/src/migrations/054_alert_rules_pending_secs.sql +11 -0
  28. package/src/migrations/055_alert_state_pending_since.sql +12 -0
  29. package/src/migrations/056_metric_point_series_received.sql +9 -0
  30. package/src/migrations/057_alert_evaluations_outcome.sql +11 -0
  31. package/src/migrations/058_alert_state_pending_seen_at.sql +12 -0
  32. package/src/migrations/059_alert_evaluations_error.sql +11 -0
  33. package/src/migrations-postgres/054_alert_rules_pending_secs.sql +7 -0
  34. package/src/migrations-postgres/055_alert_state_pending_since.sql +6 -0
  35. package/src/migrations-postgres/056_metric_point_series_received.sql +7 -0
  36. package/src/migrations-postgres/057_alert_evaluations_outcome.sql +6 -0
  37. package/src/migrations-postgres/058_alert_state_pending_seen_at.sql +6 -0
  38. package/src/migrations-postgres/059_alert_evaluations_error.sql +6 -0
  39. package/src/migrations-postgres/README.md +13 -0
  40. package/src/otlp/decode/logs.ts +30 -2
  41. package/src/otlp/decode/metrics.ts +67 -2
  42. package/src/otlp/decode/traces.ts +57 -4
  43. package/src/otlp/decode/values.ts +64 -10
  44. package/src/otlp/decode.test.ts +305 -0
  45. package/src/otlp/decode.ts +9 -3
  46. package/src/plugins/alerts-evaluator-pending.test.ts +322 -0
  47. package/src/plugins/alerts-evaluator-retention.test.ts +2 -0
  48. package/src/plugins/alerts-evaluator.test.ts +10 -2
  49. package/src/plugins/alerts-evaluator.ts +239 -40
  50. package/src/plugins/alerts-routes.ts +17 -2
  51. package/src/plugins/autosre-routes.ts +9 -0
  52. package/src/plugins/logs-receiver.ts +8 -8
  53. package/src/plugins/metrics-receiver.ts +10 -9
  54. package/src/plugins/otlp-receiver.ts +10 -26
  55. package/src/plugins/profile-routes.test.ts +22 -0
  56. package/src/plugins/profile-routes.ts +11 -2
  57. package/src/plugins/replay-query-routes.test.ts +63 -1
  58. package/src/plugins/replay-query-routes.ts +40 -9
  59. package/dist/chunk-2FALTDCE.js.map +0 -1
package/dist/index.js CHANGED
@@ -9,7 +9,7 @@ import {
9
9
  randomHex,
10
10
  sha256Hex,
11
11
  sqlDbFor
12
- } from "./chunk-2FALTDCE.js";
12
+ } from "./chunk-VSOZJO6Y.js";
13
13
  import {
14
14
  LlmCallError,
15
15
  aiEvaluationEvidenceReferences,
@@ -3575,22 +3575,66 @@ var isPrivateIpv4 = (o) => o[0] === 0 || // "this" network / unspecified
3575
3575
  o[0] === 10 || o[0] === 127 || o[0] === 100 && o[1] >= 64 && o[1] <= 127 || // CGNAT
3576
3576
  o[0] === 169 && o[1] === 254 || // link-local incl. cloud metadata
3577
3577
  o[0] === 172 && o[1] >= 16 && o[1] <= 31 || o[0] === 192 && o[1] === 168 || o[0] >= 224;
3578
+ var parseIpv6 = (literal) => {
3579
+ let text = literal.replace(/%.*$/, "");
3580
+ const tail = /:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(text);
3581
+ let tailGroups = [];
3582
+ if (tail) {
3583
+ const v4 = parseIpv4(tail[1]);
3584
+ if (!v4) return null;
3585
+ tailGroups = [v4[0] << 8 | v4[1], v4[2] << 8 | v4[3]];
3586
+ text = text.slice(0, -tail[1].length);
3587
+ if (text.endsWith(":") && !text.endsWith("::")) text = text.slice(0, -1);
3588
+ }
3589
+ const halves = text.split("::");
3590
+ if (halves.length > 2) return null;
3591
+ const toGroups = (part) => {
3592
+ if (part === "") return [];
3593
+ const groups = [];
3594
+ for (const g of part.split(":")) {
3595
+ if (!/^[0-9a-f]{1,4}$/.test(g)) return null;
3596
+ groups.push(Number.parseInt(g, 16));
3597
+ }
3598
+ return groups;
3599
+ };
3600
+ const head = toGroups(halves[0]);
3601
+ const rest = halves.length === 2 ? toGroups(halves[1]) : [];
3602
+ if (!head || !rest) return null;
3603
+ const explicit = head.length + rest.length + tailGroups.length;
3604
+ if (halves.length === 1) {
3605
+ return explicit === 8 ? [...head, ...tailGroups] : null;
3606
+ }
3607
+ if (explicit > 7) return null;
3608
+ return [
3609
+ ...head,
3610
+ ...new Array(8 - explicit).fill(0),
3611
+ ...rest,
3612
+ ...tailGroups
3613
+ ];
3614
+ };
3615
+ var embeddedIpv4 = (hi, lo) => [
3616
+ hi >> 8,
3617
+ hi & 255,
3618
+ lo >> 8,
3619
+ lo & 255
3620
+ ];
3578
3621
  var isPrivateIpv6 = (host) => {
3579
3622
  const literal = host.replace(/^\[|\]$/g, "").toLowerCase();
3580
- if (literal === "::" || literal === "::1") return true;
3581
- const dotted = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(literal);
3582
- if (dotted) {
3583
- const v4 = parseIpv4(dotted[1]);
3584
- return v4 ? isPrivateIpv4(v4) : true;
3585
- }
3586
- const hex = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(literal);
3587
- if (hex) {
3588
- const hi = Number.parseInt(hex[1], 16);
3589
- const lo = Number.parseInt(hex[2], 16);
3590
- return isPrivateIpv4([hi >> 8, hi & 255, lo >> 8, lo & 255]);
3591
- }
3592
- const first = Number.parseInt(literal.split(":")[0] || "0", 16);
3593
- if (Number.isNaN(first)) return true;
3623
+ const g = parseIpv6(literal);
3624
+ if (!g) return true;
3625
+ const zeroPrefix = (n) => g.slice(0, n).every((x) => x === 0);
3626
+ if (zeroPrefix(6)) {
3627
+ if (g[6] === 0 && (g[7] === 0 || g[7] === 1)) return true;
3628
+ return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
3629
+ }
3630
+ if (zeroPrefix(5) && g[5] === 65535) {
3631
+ return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
3632
+ }
3633
+ if (g[0] === 100 && g[1] === 65435 && g.slice(2, 6).every((x) => x === 0)) {
3634
+ return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
3635
+ }
3636
+ if (g[0] === 8194) return isPrivateIpv4(embeddedIpv4(g[1], g[2]));
3637
+ const first = g[0];
3594
3638
  if ((first & 65024) === 64512) return true;
3595
3639
  if ((first & 65472) === 65152) return true;
3596
3640
  if ((first & 65280) === 65280) return true;
@@ -5798,7 +5842,11 @@ var MetricsStore = class {
5798
5842
  async ingestBatch(opts) {
5799
5843
  const { projectId, points, receivedAt, expiresAt } = opts;
5800
5844
  if (points.length === 0) return;
5801
- const seriesByIdentity = await this.resolveSeries(projectId, points);
5845
+ const seriesByIdentity = await this.resolveSeries(
5846
+ projectId,
5847
+ points,
5848
+ receivedAt
5849
+ );
5802
5850
  const pointStmt = this.db.prepare(`
5803
5851
  INSERT INTO metric_point (
5804
5852
  id, series_id, project_id, ts_ns, start_ts_ns,
@@ -5917,7 +5965,7 @@ var MetricsStore = class {
5917
5965
  * Ensures a `metric_series` row exists for every unique identity in the
5918
5966
  * batch. Returns a map of identity → series row for point linking.
5919
5967
  */
5920
- async resolveSeries(projectId, points) {
5968
+ async resolveSeries(projectId, points, receivedAt) {
5921
5969
  const byIdentity = /* @__PURE__ */ new Map();
5922
5970
  for (const p of points) {
5923
5971
  if (!byIdentity.has(p.identity)) byIdentity.set(p.identity, p);
@@ -5944,7 +5992,7 @@ var MetricsStore = class {
5944
5992
  `);
5945
5993
  const toInsert = [];
5946
5994
  const attemptedIds = /* @__PURE__ */ new Set();
5947
- const createdAt = (/* @__PURE__ */ new Date()).toISOString();
5995
+ const createdAt = receivedAt;
5948
5996
  for (const [identity, p] of byIdentity.entries()) {
5949
5997
  if (existing.has(identity)) continue;
5950
5998
  const id = crypto.randomUUID();
@@ -7085,7 +7133,7 @@ async function selectTraceCandidates(db, options) {
7085
7133
  }
7086
7134
  if (options.search) {
7087
7135
  const term = `%${options.search}%`;
7088
- whereClause += " AND (span_name LIKE ? OR status_message LIKE ? OR attributes_json LIKE ? OR events_json LIKE ?)";
7136
+ whereClause += " AND (span_name LIKE ? OR status_message LIKE ? OR CAST(attributes_json AS TEXT) LIKE ? OR CAST(events_json AS TEXT) LIKE ?)";
7089
7137
  binds.push(term, term, term, term);
7090
7138
  }
7091
7139
  const having = options.status === "error" ? "HAVING SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) > 0" : options.status === "ok" ? "HAVING SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) = 0" : "";
@@ -8498,6 +8546,17 @@ var createTelemetryCollectorApp = (pluginsOrConfig) => {
8498
8546
  app.use("/v1/*", config.auth.middleware);
8499
8547
  } else if (config.auth?.ingest) {
8500
8548
  app.use("/v1/*", config.auth.ingest);
8549
+ } else if (!config.allowUnauthenticatedIngest) {
8550
+ let warned = false;
8551
+ app.use("/v1/*", async (_c, next) => {
8552
+ if (!warned) {
8553
+ warned = true;
8554
+ runtime.logger.warn(
8555
+ "[collector] WARNING: /v1/* ingest is UNAUTHENTICATED \u2014 this app was built without `auth`, so anyone who can reach it can write telemetry. Pass `auth: { middleware: createIngestAuth() }` (and set INGEST_KEY or create a key in the dashboard), or set `allowUnauthenticatedIngest: true` to acknowledge and silence this warning."
8556
+ );
8557
+ }
8558
+ await next();
8559
+ });
8501
8560
  }
8502
8561
  if (config.dashboardAuth) {
8503
8562
  const { middleware } = config.dashboardAuth;
@@ -12872,8 +12931,426 @@ var aiSpanPayloadsProcessorPlugin = {
12872
12931
  }
12873
12932
  };
12874
12933
 
12934
+ // src/lib/alert-email.ts
12935
+ var MAX_EMAIL_RECIPIENTS = 20;
12936
+ var EMAIL_RE = /^[^\s@<>,;:"()[\]\\]+@[^\s@<>,;:"()[\]\\]+\.[^\s@<>,;:"()[\]\\]+$/;
12937
+ var isValidEmailAddress = (addr) => addr.length <= 254 && EMAIL_RE.test(addr);
12938
+ var oneLine = (s) => s.replace(/[\r\n]+/g, " ").trim();
12939
+ function formatAlertEmail(payload) {
12940
+ const label = payload.state === "firing" ? "FIRING" : "RESOLVED";
12941
+ const name = oneLine(payload.rule.name);
12942
+ const lines = [
12943
+ payload.state === "firing" ? `Alert rule "${name}" is firing.` : `Alert rule "${name}" has resolved.`,
12944
+ "",
12945
+ `Rule: ${name} (${payload.rule.id})`,
12946
+ `Signal: ${payload.rule.signal}`,
12947
+ `State: ${payload.state}`,
12948
+ `Value: ${payload.value}`,
12949
+ `Condition: value ${payload.comparison} ${payload.threshold}`,
12950
+ `Threshold: ${payload.threshold}`,
12951
+ `Comparison: ${payload.comparison}`,
12952
+ `Evaluated: ${payload.evaluatedAt}`,
12953
+ `Project: ${payload.projectId}`
12954
+ ];
12955
+ if (payload.analysis) {
12956
+ lines.push(
12957
+ `Analysis: ${payload.analysis.id}${payload.analysis.status ? ` (${payload.analysis.status})` : ""}`
12958
+ );
12959
+ if (payload.analysis.narrative) {
12960
+ lines.push("", payload.analysis.narrative);
12961
+ }
12962
+ }
12963
+ lines.push("", "Sent by obs-unified.");
12964
+ return {
12965
+ subject: `[obs-unified] ${label}: ${name}`,
12966
+ text: `${lines.join("\n")}
12967
+ `
12968
+ };
12969
+ }
12970
+
12875
12971
  // src/lib/alerts-store.ts
12876
12972
  import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT4 } from "@obsunified/types";
12973
+
12974
+ // src/lib/metric-alerts.ts
12975
+ var METRIC_AGGREGATIONS = [
12976
+ "last",
12977
+ "min",
12978
+ "max",
12979
+ "avg",
12980
+ "sum",
12981
+ "increase",
12982
+ "rate",
12983
+ "age_seconds"
12984
+ ];
12985
+ var METRIC_FIELDS = [
12986
+ "value",
12987
+ "count",
12988
+ "sum"
12989
+ ];
12990
+ var TEMPORALITY_DELTA = 1;
12991
+ var MAX_METRIC_NAMES = 20;
12992
+ var MAX_ATTRIBUTE_KEYS = 32;
12993
+ var MAX_MATCHED_SERIES = 1e3;
12994
+ var MAX_LOADED_POINTS = 5e4;
12995
+ var MAX_METRIC_WINDOW_MINS = 1440;
12996
+ var DEFAULT_STALE_SECS = 300;
12997
+ var MAX_STALE_SECS = 86400;
12998
+ var METRIC_NO_DATA_POLICIES = ["zero", "keep"];
12999
+ var ID_CHUNK = 50;
13000
+ var SELECTOR_KEYS = [
13001
+ "metric",
13002
+ "attributes",
13003
+ "resourceAttributes",
13004
+ "serviceName",
13005
+ "field"
13006
+ ];
13007
+ var QUERY_KEYS = [
13008
+ ...SELECTOR_KEYS,
13009
+ "aggregation",
13010
+ "denominator",
13011
+ "minDenominator",
13012
+ "staleSecs",
13013
+ "noData"
13014
+ ];
13015
+ var DENOMINATOR_KEYS = [...SELECTOR_KEYS, "aggregation"];
13016
+ var isPlainObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
13017
+ var validateAttributes = (value, path) => {
13018
+ if (value === void 0) return;
13019
+ if (!isPlainObject(value))
13020
+ throw new Error(`${path} must be an object of attribute: value pairs`);
13021
+ const entries = Object.entries(value);
13022
+ if (entries.length > MAX_ATTRIBUTE_KEYS)
13023
+ throw new Error(`${path} may list at most ${MAX_ATTRIBUTE_KEYS} keys`);
13024
+ for (const [key, v] of entries) {
13025
+ if (!key.trim()) throw new Error(`${path} has an empty attribute name`);
13026
+ const ok = typeof v === "string" || typeof v === "boolean" || typeof v === "number" && Number.isFinite(v);
13027
+ if (!ok)
13028
+ throw new Error(
13029
+ `${path}.${key} must be a string, finite number or boolean`
13030
+ );
13031
+ }
13032
+ };
13033
+ var validateSelector = (q, path, allowedKeys) => {
13034
+ for (const key of Object.keys(q)) {
13035
+ if (!allowedKeys.includes(key))
13036
+ throw new Error(
13037
+ `${path}.${key} is not a metrics query field (allowed: ${allowedKeys.join(", ")})`
13038
+ );
13039
+ }
13040
+ const metric = q.metric;
13041
+ if (typeof metric === "string") {
13042
+ if (!metric.trim()) throw new Error(`${path}.metric must not be empty`);
13043
+ } else if (Array.isArray(metric)) {
13044
+ if (metric.length === 0)
13045
+ throw new Error(`${path}.metric must name at least one metric`);
13046
+ if (metric.length > MAX_METRIC_NAMES)
13047
+ throw new Error(
13048
+ `${path}.metric may name at most ${MAX_METRIC_NAMES} metrics`
13049
+ );
13050
+ for (const m of metric) {
13051
+ if (typeof m !== "string" || !m.trim())
13052
+ throw new Error(`${path}.metric entries must be non-empty strings`);
13053
+ }
13054
+ } else {
13055
+ throw new Error(
13056
+ `${path}.metric is required: a metric name or a list of names`
13057
+ );
13058
+ }
13059
+ validateAttributes(q.attributes, `${path}.attributes`);
13060
+ validateAttributes(q.resourceAttributes, `${path}.resourceAttributes`);
13061
+ if (q.serviceName !== void 0 && (typeof q.serviceName !== "string" || !q.serviceName.trim()))
13062
+ throw new Error(`${path}.serviceName must be a non-empty string`);
13063
+ if (q.field !== void 0 && !METRIC_FIELDS.includes(q.field))
13064
+ throw new Error(
13065
+ `${path}.field must be one of: ${METRIC_FIELDS.join(", ")}`
13066
+ );
13067
+ };
13068
+ function validateMetricQuery(query) {
13069
+ if (!isPlainObject(query))
13070
+ throw new Error("query must be an object for the metrics signal");
13071
+ validateSelector(query, "query", QUERY_KEYS);
13072
+ const aggregation = query.aggregation;
13073
+ if (!METRIC_AGGREGATIONS.includes(aggregation))
13074
+ throw new Error(
13075
+ `query.aggregation must be one of: ${METRIC_AGGREGATIONS.join(", ")}`
13076
+ );
13077
+ if (query.staleSecs !== void 0 && (!Number.isInteger(query.staleSecs) || query.staleSecs < 1 || query.staleSecs > MAX_STALE_SECS))
13078
+ throw new Error(
13079
+ `query.staleSecs must be an integer from 1 to ${MAX_STALE_SECS}`
13080
+ );
13081
+ if (query.noData !== void 0 && !METRIC_NO_DATA_POLICIES.includes(query.noData))
13082
+ throw new Error(
13083
+ `query.noData must be one of: ${METRIC_NO_DATA_POLICIES.join(", ")}`
13084
+ );
13085
+ if (query.denominator === void 0) {
13086
+ if (query.minDenominator !== void 0)
13087
+ throw new Error("query.minDenominator requires query.denominator");
13088
+ return;
13089
+ }
13090
+ if (aggregation === "age_seconds")
13091
+ throw new Error("query.denominator cannot be used with age_seconds");
13092
+ if (!isPlainObject(query.denominator))
13093
+ throw new Error("query.denominator must be an object");
13094
+ validateSelector(query.denominator, "query.denominator", DENOMINATOR_KEYS);
13095
+ const denAgg = query.denominator.aggregation;
13096
+ if (denAgg !== void 0) {
13097
+ if (denAgg === "age_seconds" || !METRIC_AGGREGATIONS.includes(denAgg))
13098
+ throw new Error(
13099
+ `query.denominator.aggregation must be one of: ${METRIC_AGGREGATIONS.filter((a) => a !== "age_seconds").join(", ")}`
13100
+ );
13101
+ }
13102
+ if (query.minDenominator !== void 0 && (typeof query.minDenominator !== "number" || !Number.isFinite(query.minDenominator) || query.minDenominator < 0))
13103
+ throw new Error("query.minDenominator must be a finite number >= 0");
13104
+ }
13105
+ var toBigInt = (ns) => {
13106
+ if (!ns) return null;
13107
+ try {
13108
+ return BigInt(ns);
13109
+ } catch {
13110
+ return null;
13111
+ }
13112
+ };
13113
+ var readField = (p, field) => {
13114
+ const raw = field === "count" ? p.count : field === "sum" ? p.sum : p.value;
13115
+ const n = typeof raw === "string" ? Number(raw) : raw;
13116
+ return typeof n === "number" && Number.isFinite(n) ? n : null;
13117
+ };
13118
+ var sortPoints = (points) => [...points].sort((a, b) => {
13119
+ const ta = toBigInt(a.tsNs) ?? 0n;
13120
+ const tb = toBigInt(b.tsNs) ?? 0n;
13121
+ if (ta !== tb) return ta < tb ? -1 : 1;
13122
+ return a.receivedAt < b.receivedAt ? -1 : a.receivedAt > b.receivedAt ? 1 : 0;
13123
+ });
13124
+ var cumulativeIncrease = (points, windowStartNs, bornInWindow) => {
13125
+ if (points.length === 0) return 0;
13126
+ const first = points[0];
13127
+ const countsFirst = first.start !== null ? first.start >= windowStartNs : bornInWindow;
13128
+ let total = countsFirst ? first.v : 0;
13129
+ for (let i = 1; i < points.length; i++) {
13130
+ const prev = points[i - 1];
13131
+ const cur = points[i];
13132
+ const restarted = cur.start !== null && prev.start !== null && cur.start !== prev.start;
13133
+ if (restarted || cur.v < prev.v) total += cur.v;
13134
+ else total += cur.v - prev.v;
13135
+ }
13136
+ return total;
13137
+ };
13138
+ function reduceSeries(points, opts) {
13139
+ const usable = [];
13140
+ for (const p of sortPoints(points)) {
13141
+ const v = readField(p, opts.field);
13142
+ if (v !== null) usable.push({ v, start: toBigInt(p.startTsNs) });
13143
+ }
13144
+ if (usable.length === 0) return null;
13145
+ const values = usable.map((u) => u.v);
13146
+ switch (opts.aggregation) {
13147
+ case "last":
13148
+ return values[values.length - 1];
13149
+ case "min":
13150
+ return values.reduce((a, b) => b < a ? b : a);
13151
+ case "max":
13152
+ return values.reduce((a, b) => b > a ? b : a);
13153
+ case "sum":
13154
+ return values.reduce((a, b) => a + b, 0);
13155
+ case "avg":
13156
+ return values.reduce((a, b) => a + b, 0) / values.length;
13157
+ case "increase":
13158
+ case "rate": {
13159
+ const increase = opts.temporality === TEMPORALITY_DELTA ? values.reduce((a, b) => a + b, 0) : cumulativeIncrease(
13160
+ usable,
13161
+ opts.windowStartNs,
13162
+ opts.bornInWindow ?? false
13163
+ );
13164
+ return opts.aggregation === "rate" ? increase / opts.windowSecs : increase;
13165
+ }
13166
+ }
13167
+ }
13168
+ var metricRatio = (numerator, denominator, minDenominator) => denominator > 0 && denominator >= minDenominator ? numerator / denominator : 0;
13169
+ var parseAttrs2 = (raw) => {
13170
+ if (raw === null || raw === void 0) return {};
13171
+ if (typeof raw === "string") {
13172
+ try {
13173
+ const parsed = JSON.parse(raw);
13174
+ return isPlainObject(parsed) ? parsed : {};
13175
+ } catch {
13176
+ return {};
13177
+ }
13178
+ }
13179
+ return isPlainObject(raw) ? raw : {};
13180
+ };
13181
+ var attrsMatch = (stored, wanted) => {
13182
+ if (!wanted) return true;
13183
+ for (const [key, expected] of Object.entries(wanted)) {
13184
+ const actual = stored[key];
13185
+ if (actual === void 0 || actual === null) return false;
13186
+ if (typeof actual === "object") return false;
13187
+ if (String(actual) !== String(expected)) return false;
13188
+ }
13189
+ return true;
13190
+ };
13191
+ var metricNames = (selector) => [
13192
+ ...new Set(
13193
+ (Array.isArray(selector.metric) ? selector.metric : [selector.metric]).map(
13194
+ (m) => m.trim()
13195
+ )
13196
+ )
13197
+ ];
13198
+ var chunks = (items, size) => {
13199
+ const out = [];
13200
+ for (let i = 0; i < items.length; i += size)
13201
+ out.push(items.slice(i, i + size));
13202
+ return out;
13203
+ };
13204
+ async function matchingSeries(db, projectId, selector) {
13205
+ const names = metricNames(selector);
13206
+ let sql = `SELECT id, temporality, created_at, attributes_json, resource_attrs_json
13207
+ FROM metric_series
13208
+ WHERE project_id = ? AND name IN (${names.map(() => "?").join(", ")})`;
13209
+ const binds = [projectId, ...names];
13210
+ if (selector.serviceName) {
13211
+ sql += ` AND service_name = ?`;
13212
+ binds.push(selector.serviceName);
13213
+ }
13214
+ const rs = await db.prepare(sql).bind(...binds).all();
13215
+ const matched = (rs.results ?? []).filter(
13216
+ (row) => attrsMatch(parseAttrs2(row.attributes_json), selector.attributes) && attrsMatch(
13217
+ parseAttrs2(row.resource_attrs_json),
13218
+ selector.resourceAttributes
13219
+ )
13220
+ );
13221
+ if (matched.length > MAX_MATCHED_SERIES)
13222
+ throw new Error(
13223
+ `metrics query matches ${matched.length} series (limit ${MAX_MATCHED_SERIES}); narrow it with attributes or serviceName`
13224
+ );
13225
+ return matched.map((row) => ({
13226
+ id: row.id,
13227
+ createdAt: row.created_at,
13228
+ temporality: row.temporality === null || row.temporality === void 0 ? null : Number(row.temporality)
13229
+ }));
13230
+ }
13231
+ var STALENESS_APPLIES = /* @__PURE__ */ new Set([
13232
+ "last",
13233
+ "min",
13234
+ "max",
13235
+ "avg",
13236
+ "sum"
13237
+ ]);
13238
+ async function evaluateSelector(db, selector, aggregation, ctx) {
13239
+ const series = await matchingSeries(db, ctx.projectId, selector);
13240
+ if (series.length === 0) return { value: 0, contributing: 0 };
13241
+ const windowStartMs = ctx.now.getTime() - ctx.windowMins * 6e4;
13242
+ const cutoff = new Date(windowStartMs).toISOString();
13243
+ const bySeries = /* @__PURE__ */ new Map();
13244
+ let loaded = 0;
13245
+ for (const chunk of chunks(series, ID_CHUNK)) {
13246
+ const cap = ctx.maxLoadedPoints ?? MAX_LOADED_POINTS;
13247
+ const budget = cap - loaded;
13248
+ const rs = await db.prepare(
13249
+ `SELECT series_id, ts_ns, start_ts_ns, value, count, sum, received_at
13250
+ FROM metric_point
13251
+ WHERE project_id = ? AND series_id IN (${chunk.map(() => "?").join(", ")})
13252
+ AND received_at >= ?
13253
+ LIMIT ?`
13254
+ ).bind(ctx.projectId, ...chunk.map((s) => s.id), cutoff, budget + 1).all();
13255
+ const rows = rs.results ?? [];
13256
+ loaded += rows.length;
13257
+ if (loaded > cap)
13258
+ throw new Error(
13259
+ `metrics query loads more than ${cap} points in its window; shorten windowMins or narrow the query`
13260
+ );
13261
+ for (const row of rows) {
13262
+ const list = bySeries.get(row.series_id) ?? [];
13263
+ list.push({
13264
+ tsNs: String(row.ts_ns),
13265
+ startTsNs: row.start_ts_ns === null ? null : String(row.start_ts_ns),
13266
+ value: row.value,
13267
+ count: row.count,
13268
+ sum: row.sum,
13269
+ receivedAt: row.received_at
13270
+ });
13271
+ bySeries.set(row.series_id, list);
13272
+ }
13273
+ }
13274
+ const field = selector.field ?? "value";
13275
+ const windowStartNs = BigInt(windowStartMs) * 1000000n;
13276
+ const freshCutoff = new Date(
13277
+ ctx.now.getTime() - (ctx.staleSecs ?? DEFAULT_STALE_SECS) * 1e3
13278
+ ).toISOString();
13279
+ let total = 0;
13280
+ let contributing = 0;
13281
+ for (const s of series) {
13282
+ const points = bySeries.get(s.id) ?? [];
13283
+ if (STALENESS_APPLIES.has(aggregation)) {
13284
+ const newest = points.reduce(
13285
+ (max, p) => max === null || p.receivedAt > max ? p.receivedAt : max,
13286
+ null
13287
+ );
13288
+ if (newest === null || newest < freshCutoff) continue;
13289
+ }
13290
+ const reduced = reduceSeries(points, {
13291
+ aggregation,
13292
+ field,
13293
+ temporality: s.temporality,
13294
+ windowStartNs,
13295
+ windowSecs: ctx.windowMins * 60,
13296
+ bornInWindow: s.createdAt >= cutoff
13297
+ });
13298
+ if (reduced !== null) {
13299
+ total += reduced;
13300
+ contributing += 1;
13301
+ }
13302
+ }
13303
+ return { value: total, contributing };
13304
+ }
13305
+ async function ageSeconds(db, selector, ctx) {
13306
+ const series = await matchingSeries(db, ctx.projectId, selector);
13307
+ let newest = null;
13308
+ for (const chunk of chunks(series, ID_CHUNK)) {
13309
+ const row = await db.prepare(
13310
+ `SELECT MAX(received_at) AS newest FROM metric_point
13311
+ WHERE project_id = ? AND series_id IN (${chunk.map(() => "?").join(", ")})`
13312
+ ).bind(ctx.projectId, ...chunk.map((s) => s.id)).first();
13313
+ if (row?.newest && (newest === null || row.newest > newest))
13314
+ newest = row.newest;
13315
+ }
13316
+ const reference = Date.parse(newest ?? ctx.ruleCreatedAt);
13317
+ if (!Number.isFinite(reference)) return 0;
13318
+ return Math.max(0, (ctx.now.getTime() - reference) / 1e3);
13319
+ }
13320
+ async function evaluateMetricQuery(db, query, ctx) {
13321
+ if (query.aggregation === "age_seconds")
13322
+ return { value: await ageSeconds(db, query, ctx), noData: false };
13323
+ const withStale = { ...ctx, staleSecs: query.staleSecs ?? ctx.staleSecs };
13324
+ const numerator = await evaluateSelector(
13325
+ db,
13326
+ query,
13327
+ query.aggregation,
13328
+ withStale
13329
+ );
13330
+ if (!query.denominator)
13331
+ return {
13332
+ value: numerator.value,
13333
+ noData: numerator.contributing === 0
13334
+ };
13335
+ const denominator = await evaluateSelector(
13336
+ db,
13337
+ query.denominator,
13338
+ query.denominator.aggregation ?? query.aggregation,
13339
+ withStale
13340
+ );
13341
+ return {
13342
+ value: metricRatio(
13343
+ numerator.value,
13344
+ denominator.value,
13345
+ query.minDenominator ?? 0
13346
+ ),
13347
+ noData: numerator.contributing === 0 && denominator.contributing === 0
13348
+ };
13349
+ }
13350
+
13351
+ // src/lib/alerts-store.ts
13352
+ var MAX_PENDING_SECS = 86400;
13353
+ var EMAIL_UNAVAILABLE_MESSAGE = "email channels need SMTP configured on the collector (ALERT_SMTP_HOST and ALERT_SMTP_FROM on the Node collector); the Cloudflare Worker collector cannot send email";
12877
13354
  var parseJsonField = (value) => {
12878
13355
  if (typeof value === "string") return JSON.parse(value);
12879
13356
  return value;
@@ -12892,6 +13369,8 @@ var rowToRule = (row, state) => ({
12892
13369
  createdAt: row.created_at,
12893
13370
  updatedAt: row.updated_at,
12894
13371
  analysisId: row.analysis_id ?? null,
13372
+ pendingSecs: Number(row.pending_secs ?? 0),
13373
+ pendingSince: state?.pending_since ?? null,
12895
13374
  currentState: state?.current_state,
12896
13375
  lastStateChange: state?.last_state_change ?? null
12897
13376
  });
@@ -12902,11 +13381,18 @@ var rowToEvaluation = (row) => ({
12902
13381
  evaluatedAt: row.evaluated_at,
12903
13382
  value: row.value,
12904
13383
  state: row.state,
12905
- notified: row.notified === 1
13384
+ notified: row.notified === 1,
13385
+ outcome: row.outcome ?? "value",
13386
+ error: row.error_message ?? null
12906
13387
  });
13388
+ var MAX_ERROR_MESSAGE = 500;
13389
+ var sanitizeEvaluationError = (message) => {
13390
+ const cleaned = message.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s"'<>]+/gi, "<url>").replace(/\s+/g, " ").trim();
13391
+ return cleaned.length > MAX_ERROR_MESSAGE ? `${cleaned.slice(0, MAX_ERROR_MESSAGE - 1)}\u2026` : cleaned;
13392
+ };
12907
13393
  var windowCutoffIso = (windowMins) => new Date(Date.now() - windowMins * 60 * 1e3).toISOString();
12908
- var SIGNALS2 = ["spans", "logs", "usage", "ai"];
12909
- var COMPARISONS = [">", ">=", "<", "<="];
13394
+ var SIGNALS2 = ["spans", "logs", "usage", "ai", "metrics"];
13395
+ var COMPARISONS = [">", ">=", "<", "<=", "==", "!="];
12910
13396
  function compareValue(value, threshold, comparison) {
12911
13397
  switch (comparison) {
12912
13398
  case ">":
@@ -12917,6 +13403,10 @@ function compareValue(value, threshold, comparison) {
12917
13403
  return value < threshold;
12918
13404
  case "<=":
12919
13405
  return value <= threshold;
13406
+ case "==":
13407
+ return value === threshold;
13408
+ case "!=":
13409
+ return value !== threshold;
12920
13410
  }
12921
13411
  }
12922
13412
  var AlertsStore = class {
@@ -12934,11 +13424,46 @@ var AlertsStore = class {
12934
13424
  throw new Error("windowMins must be >= 1");
12935
13425
  if (!COMPARISONS.includes(input.comparison))
12936
13426
  throw new Error(`comparison must be one of: ${COMPARISONS.join(", ")}`);
13427
+ if (input.pendingSecs !== void 0 && (!Number.isInteger(input.pendingSecs) || input.pendingSecs < 0 || input.pendingSecs > MAX_PENDING_SECS))
13428
+ throw new Error(
13429
+ `pendingSecs must be an integer from 0 to ${MAX_PENDING_SECS}`
13430
+ );
13431
+ if (input.signal === "metrics") {
13432
+ validateMetricQuery(input.query);
13433
+ if (input.windowMins > MAX_METRIC_WINDOW_MINS)
13434
+ throw new Error(
13435
+ `windowMins must be <= ${MAX_METRIC_WINDOW_MINS} for the metrics signal`
13436
+ );
13437
+ }
12937
13438
  if (!Array.isArray(input.channels) || input.channels.length === 0)
12938
13439
  throw new Error("at least one channel is required");
12939
13440
  for (const ch of input.channels) {
13441
+ if (ch.type === "email") {
13442
+ if (!Array.isArray(ch.to) || ch.to.length === 0)
13443
+ throw new Error("email channel.to must list at least one address");
13444
+ if (ch.to.length > MAX_EMAIL_RECIPIENTS)
13445
+ throw new Error(
13446
+ `email channel.to may list at most ${MAX_EMAIL_RECIPIENTS} addresses`
13447
+ );
13448
+ for (const addr of ch.to) {
13449
+ if (typeof addr !== "string" || !isValidEmailAddress(addr))
13450
+ throw new Error(
13451
+ `email channel.to has an invalid address: ${JSON.stringify(addr)}`
13452
+ );
13453
+ }
13454
+ const extra = Object.keys(ch).filter((k) => k !== "type" && k !== "to");
13455
+ if (extra.length > 0)
13456
+ throw new Error(
13457
+ `email channel accepts only "type" and "to" (SMTP settings come from the collector environment); got: ${extra.join(", ")}`
13458
+ );
13459
+ if (!this.options.emailDeliveryAvailable)
13460
+ throw new Error(EMAIL_UNAVAILABLE_MESSAGE);
13461
+ continue;
13462
+ }
12940
13463
  if (ch.type !== "webhook")
12941
- throw new Error(`unknown channel type: ${ch.type}`);
13464
+ throw new Error(
13465
+ `unknown channel type: ${ch.type}`
13466
+ );
12942
13467
  if (!ch.url || typeof ch.url !== "string")
12943
13468
  throw new Error("channel.url must be http(s)://");
12944
13469
  try {
@@ -12961,7 +13486,55 @@ var AlertsStore = class {
12961
13486
  const stateByRule = new Map(
12962
13487
  (stateRs.results ?? []).map((s) => [s.rule_id, s])
12963
13488
  );
12964
- return rows.map((r) => rowToRule(r, stateByRule.get(r.id)));
13489
+ const lastByRule = await this.lastEvaluations(projectId);
13490
+ return rows.map((r) => {
13491
+ const state = stateByRule.get(r.id);
13492
+ const last = lastByRule.get(r.id);
13493
+ return {
13494
+ ...rowToRule(r, state),
13495
+ notificationDelivered: state ? state.notified === void 0 || state.notified === null ? true : Number(state.notified) === 1 : null,
13496
+ lastEvaluation: last ? {
13497
+ evaluatedAt: last.evaluated_at,
13498
+ outcome: last.outcome ?? "value",
13499
+ value: Number(last.value),
13500
+ error: (last.outcome ?? "value") === "error" ? last.error_message ?? null : null
13501
+ } : null
13502
+ };
13503
+ });
13504
+ }
13505
+ /**
13506
+ * The newest evaluation of every rule in the project, one query. Falls
13507
+ * back to the pre-057/059 column set on older databases.
13508
+ */
13509
+ async lastEvaluations(projectId) {
13510
+ const run = (columns) => this.db.prepare(
13511
+ `SELECT e.rule_id, e.evaluated_at, e.value${columns}
13512
+ FROM alert_evaluations e
13513
+ WHERE e.project_id = ?
13514
+ AND e.evaluated_at = (
13515
+ SELECT MAX(e2.evaluated_at) FROM alert_evaluations e2
13516
+ WHERE e2.rule_id = e.rule_id
13517
+ )`
13518
+ ).bind(projectId).all();
13519
+ let rs;
13520
+ try {
13521
+ rs = await run(", e.outcome, e.error_message");
13522
+ } catch (err) {
13523
+ if (!isMissingColumn(err, "error_message") && !isMissingColumn(err, "outcome"))
13524
+ throw err;
13525
+ try {
13526
+ rs = await run(", e.outcome");
13527
+ } catch (inner) {
13528
+ if (!isMissingColumn(inner, "outcome")) throw inner;
13529
+ rs = await run("");
13530
+ }
13531
+ }
13532
+ const out = /* @__PURE__ */ new Map();
13533
+ for (const row of rs.results ?? []) {
13534
+ const seen = out.get(row.rule_id);
13535
+ if (!seen || row.outcome === "error") out.set(row.rule_id, row);
13536
+ }
13537
+ return out;
12965
13538
  }
12966
13539
  async getRule(id, projectId) {
12967
13540
  const row = await this.db.prepare(`SELECT * FROM alert_rules WHERE id = ? AND project_id = ?`).bind(id, projectId).first();
@@ -12973,10 +13546,8 @@ var AlertsStore = class {
12973
13546
  this.validateInput(input);
12974
13547
  const id = randomHex(16);
12975
13548
  const now = (/* @__PURE__ */ new Date()).toISOString();
12976
- await this.db.prepare(
12977
- `INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at, analysis_id)
12978
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
12979
- ).bind(
13549
+ const pendingSecs = input.pendingSecs ?? 0;
13550
+ const values = [
12980
13551
  id,
12981
13552
  projectId,
12982
13553
  input.name.trim(),
@@ -12990,7 +13561,18 @@ var AlertsStore = class {
12990
13561
  now,
12991
13562
  now,
12992
13563
  input.analysisId ?? null
12993
- ).run();
13564
+ ];
13565
+ await this.withPendingColumnFallback(
13566
+ pendingSecs,
13567
+ () => this.db.prepare(
13568
+ `INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at, analysis_id, pending_secs)
13569
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
13570
+ ).bind(...values, pendingSecs).run(),
13571
+ () => this.db.prepare(
13572
+ `INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at, analysis_id)
13573
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
13574
+ ).bind(...values).run()
13575
+ );
12994
13576
  const created = await this.getRule(id, projectId);
12995
13577
  if (!created) throw new Error("Failed to create rule");
12996
13578
  return created;
@@ -13007,16 +13589,13 @@ var AlertsStore = class {
13007
13589
  comparison: patch.comparison ?? existing.comparison,
13008
13590
  channels: patch.channels ?? existing.channels,
13009
13591
  enabled: patch.enabled ?? existing.enabled,
13010
- analysisId: "analysisId" in patch ? patch.analysisId : existing.analysisId
13592
+ analysisId: "analysisId" in patch ? patch.analysisId : existing.analysisId,
13593
+ pendingSecs: patch.pendingSecs ?? existing.pendingSecs
13011
13594
  };
13012
13595
  this.validateInput(merged);
13013
13596
  const now = (/* @__PURE__ */ new Date()).toISOString();
13014
- await this.db.prepare(
13015
- `UPDATE alert_rules SET
13016
- name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
13017
- comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?
13018
- WHERE id = ? AND project_id = ?`
13019
- ).bind(
13597
+ const pendingSecs = merged.pendingSecs ?? 0;
13598
+ const values = [
13020
13599
  merged.name.trim(),
13021
13600
  merged.signal,
13022
13601
  JSON.stringify(merged.query),
@@ -13026,12 +13605,72 @@ var AlertsStore = class {
13026
13605
  JSON.stringify(merged.channels),
13027
13606
  merged.enabled === false ? 0 : 1,
13028
13607
  now,
13029
- merged.analysisId ?? null,
13030
- id,
13031
- projectId
13032
- ).run();
13608
+ merged.analysisId ?? null
13609
+ ];
13610
+ await this.withPendingColumnFallback(
13611
+ pendingSecs,
13612
+ () => this.db.prepare(
13613
+ `UPDATE alert_rules SET
13614
+ name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
13615
+ comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?,
13616
+ pending_secs = ?
13617
+ WHERE id = ? AND project_id = ?`
13618
+ ).bind(...values, pendingSecs, id, projectId).run(),
13619
+ () => this.db.prepare(
13620
+ `UPDATE alert_rules SET
13621
+ name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
13622
+ comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?
13623
+ WHERE id = ? AND project_id = ?`
13624
+ ).bind(...values, id, projectId).run()
13625
+ );
13626
+ const evaluationChanged = merged.signal !== existing.signal || JSON.stringify(merged.query) !== JSON.stringify(existing.query) || merged.threshold !== existing.threshold || merged.windowMins !== existing.windowMins || merged.comparison !== existing.comparison || merged.enabled !== false !== existing.enabled || (merged.analysisId ?? null) !== (existing.analysisId ?? null) || pendingSecs !== (existing.pendingSecs ?? 0);
13627
+ if (evaluationChanged) await this.resetPending(id, projectId);
13033
13628
  return this.getRule(id, projectId);
13034
13629
  }
13630
+ /**
13631
+ * Clears a rule's pending period. Tolerates databases that have not
13632
+ * applied 055/058 (there is nothing to clear on them).
13633
+ */
13634
+ async resetPending(ruleId, projectId) {
13635
+ try {
13636
+ await this.db.prepare(
13637
+ `UPDATE alert_state SET pending_since = NULL, pending_seen_at = NULL
13638
+ WHERE rule_id = ? AND project_id = ?`
13639
+ ).bind(ruleId, projectId).run();
13640
+ } catch (err) {
13641
+ if (!isMissingColumn(err, "pending_seen_at")) {
13642
+ if (isMissingColumn(err, "pending_since")) return;
13643
+ throw err;
13644
+ }
13645
+ try {
13646
+ await this.db.prepare(
13647
+ `UPDATE alert_state SET pending_since = NULL
13648
+ WHERE rule_id = ? AND project_id = ?`
13649
+ ).bind(ruleId, projectId).run();
13650
+ } catch (inner) {
13651
+ if (!isMissingColumn(inner, "pending_since")) throw inner;
13652
+ }
13653
+ }
13654
+ }
13655
+ /**
13656
+ * Runs `current`; on a database that has not applied migration 054 (no
13657
+ * `alert_rules.pending_secs`) a rule without a pending duration is
13658
+ * written with the legacy statement so rolling deploys keep working. A
13659
+ * rule that needs the column is refused with a clear message instead of
13660
+ * silently losing its pending duration.
13661
+ */
13662
+ async withPendingColumnFallback(pendingSecs, current, legacy) {
13663
+ try {
13664
+ return await current();
13665
+ } catch (err) {
13666
+ if (!isMissingColumn(err, "pending_secs")) throw err;
13667
+ if (pendingSecs > 0)
13668
+ throw new Error(
13669
+ "pendingSecs needs database migration 054 (alert_rules.pending_secs); apply migrations and retry"
13670
+ );
13671
+ return legacy();
13672
+ }
13673
+ }
13035
13674
  async deleteRule(id, projectId) {
13036
13675
  const owned = await this.db.prepare(
13037
13676
  `SELECT 1 AS ok FROM alert_rules WHERE id = ? AND project_id = ?`
@@ -13102,11 +13741,32 @@ var AlertsStore = class {
13102
13741
  throw err;
13103
13742
  }
13104
13743
  }
13105
- async recordEvaluation(ruleId, projectId, value, state, notified) {
13744
+ /**
13745
+ * Record (`{ since, seenAt }`) or clear (`null`) the pending period of a
13746
+ * rule that has a pending duration (migrations 055, 058): when the
13747
+ * condition started holding, and the last evaluation that saw it hold.
13748
+ * A rule without a state row gets one in state `ok` since its creation,
13749
+ * already delivered (nothing is owed for a rule that never changed
13750
+ * state).
13751
+ */
13752
+ async setPending(ruleId, projectId, pending, ruleCreatedAt) {
13106
13753
  await this.db.prepare(
13107
- `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified)
13108
- VALUES (?, ?, ?, ?, ?, ?, ?)`
13754
+ `INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change, notified, pending_since, pending_seen_at)
13755
+ VALUES (?, ?, 'ok', ?, 1, ?, ?)
13756
+ ON CONFLICT(rule_id) DO UPDATE SET
13757
+ pending_since = excluded.pending_since,
13758
+ pending_seen_at = excluded.pending_seen_at`
13109
13759
  ).bind(
13760
+ ruleId,
13761
+ projectId,
13762
+ ruleCreatedAt,
13763
+ pending?.since ?? null,
13764
+ pending?.seenAt ?? null
13765
+ ).run();
13766
+ }
13767
+ async recordEvaluation(ruleId, projectId, value, state, notified, outcome = "value", errorMessage) {
13768
+ const error = outcome === "error" && errorMessage ? sanitizeEvaluationError(errorMessage) : null;
13769
+ const values = [
13110
13770
  randomHex(16),
13111
13771
  ruleId,
13112
13772
  projectId,
@@ -13114,15 +13774,43 @@ var AlertsStore = class {
13114
13774
  value,
13115
13775
  state,
13116
13776
  notified ? 1 : 0
13117
- ).run();
13777
+ ];
13778
+ try {
13779
+ await this.db.prepare(
13780
+ `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified, outcome, error_message)
13781
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
13782
+ ).bind(...values, outcome, error).run();
13783
+ return;
13784
+ } catch (err) {
13785
+ if (!isMissingColumn(err, "error_message")) {
13786
+ if (!isMissingColumn(err, "outcome")) throw err;
13787
+ } else {
13788
+ try {
13789
+ await this.db.prepare(
13790
+ `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified, outcome)
13791
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
13792
+ ).bind(...values, outcome).run();
13793
+ return;
13794
+ } catch (inner) {
13795
+ if (!isMissingColumn(inner, "outcome")) throw inner;
13796
+ }
13797
+ }
13798
+ await this.db.prepare(
13799
+ `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified)
13800
+ VALUES (?, ?, ?, ?, ?, ?, ?)`
13801
+ ).bind(...values).run();
13802
+ }
13118
13803
  }
13119
13804
  async listEvaluations(params) {
13120
13805
  const cutoff = new Date(
13121
13806
  Date.now() - params.hours * 60 * 60 * 1e3
13122
13807
  ).toISOString();
13123
- const rs = await this.db.prepare(
13808
+ const limit = Math.min(params.limit ?? 500, 1e3);
13809
+ const rs = params.projectId ? await this.db.prepare(
13810
+ `SELECT * FROM alert_evaluations WHERE rule_id = ? AND project_id = ? AND evaluated_at >= ? ORDER BY evaluated_at DESC LIMIT ?`
13811
+ ).bind(params.ruleId, params.projectId, cutoff, limit).all() : await this.db.prepare(
13124
13812
  `SELECT * FROM alert_evaluations WHERE rule_id = ? AND evaluated_at >= ? ORDER BY evaluated_at DESC LIMIT ?`
13125
- ).bind(params.ruleId, cutoff, Math.min(params.limit ?? 500, 1e3)).all();
13813
+ ).bind(params.ruleId, cutoff, limit).all();
13126
13814
  const rows = rs.results ?? [];
13127
13815
  const projectId = params.projectId ?? rows[0]?.project_id;
13128
13816
  const rule = projectId ? await this.getRule(params.ruleId, projectId) : null;
@@ -13136,7 +13824,21 @@ var AlertsStore = class {
13136
13824
  return response;
13137
13825
  }
13138
13826
  // ── Evaluators (count-over-window per signal) ──
13139
- async evaluateRule(rule) {
13827
+ async evaluateRule(rule, now = /* @__PURE__ */ new Date()) {
13828
+ return (await this.evaluateRuleWithStatus(rule, now)).value;
13829
+ }
13830
+ /** Like `evaluateRule`, plus whether a metrics rule found no fresh data. */
13831
+ async evaluateRuleWithStatus(rule, now = /* @__PURE__ */ new Date()) {
13832
+ if (rule.signal === "metrics" && !rule.analysisId)
13833
+ return evaluateMetricQuery(this.db, rule.query, {
13834
+ projectId: rule.projectId,
13835
+ windowMins: rule.windowMins,
13836
+ ruleCreatedAt: rule.createdAt,
13837
+ now
13838
+ });
13839
+ return { value: await this.evaluateCountRule(rule), noData: false };
13840
+ }
13841
+ async evaluateCountRule(rule) {
13140
13842
  if (rule.analysisId) {
13141
13843
  const row = await this.db.prepare(
13142
13844
  `SELECT primary_value FROM analysis_results
@@ -13154,6 +13856,12 @@ var AlertsStore = class {
13154
13856
  return this.evaluateUsageRule(rule);
13155
13857
  case "ai":
13156
13858
  return this.evaluateAIRule(rule);
13859
+ case "metrics":
13860
+ throw new Error(
13861
+ "metrics rules evaluate through evaluateRuleWithStatus"
13862
+ );
13863
+ default:
13864
+ throw new Error(`unknown alert signal: ${String(rule.signal)}`);
13157
13865
  }
13158
13866
  }
13159
13867
  /**
@@ -13239,14 +13947,46 @@ var AlertsStore = class {
13239
13947
  return row?.c ?? 0;
13240
13948
  }
13241
13949
  };
13242
- var isMissingNotifiedColumn = (err) => {
13950
+ var isMissingNotifiedColumn = (err) => isMissingColumn(err, "notified");
13951
+ var isMissingColumn = (err, column) => {
13243
13952
  const message = err instanceof Error ? err.message : String(err);
13244
- return /notified/i.test(message) && /no such column|has no column|does not exist/i.test(message);
13953
+ return message.toLowerCase().includes(column) && /no such column|has no column|does not exist/i.test(message);
13245
13954
  };
13246
13955
 
13247
13956
  // src/plugins/alerts-evaluator.ts
13248
13957
  var DEFAULT_RULE_CONCURRENCY = 5;
13249
13958
  var DEFAULT_RULE_TIMEOUT_MS = 1e4;
13959
+ var DEFAULT_MAX_EVALUATION_GAP_MS = 2 * 5 * 6e4;
13960
+ async function fireEmail(channel, payload, logger, tracer, delivery = {}) {
13961
+ const transport = delivery.email;
13962
+ if (!transport) {
13963
+ logger.error(
13964
+ "[alerts-evaluator] email channel has no SMTP transport on this collector",
13965
+ { rule_id: payload.rule.id, recipients: channel.to.length }
13966
+ );
13967
+ return false;
13968
+ }
13969
+ const exec = async (span) => {
13970
+ try {
13971
+ await transport.send({ to: channel.to, ...formatAlertEmail(payload) });
13972
+ if (span) span.setAttribute("alert.email.sent", true);
13973
+ return true;
13974
+ } catch (err) {
13975
+ if (span) span.setAttribute("alert.email.sent", false);
13976
+ logger.error("[alerts-evaluator] email delivery failed", {
13977
+ rule_id: payload.rule.id,
13978
+ recipients: channel.to.length,
13979
+ error: err instanceof Error ? err.message : String(err)
13980
+ });
13981
+ return false;
13982
+ }
13983
+ };
13984
+ if (!tracer) return exec(null);
13985
+ return tracer("email.alert", async (span) => exec(span), {
13986
+ "alert.state": payload.state,
13987
+ "alert.email.recipients": channel.to.length
13988
+ });
13989
+ }
13250
13990
  async function fireWebhook(channel, payload, logger, tracer, delivery = {}) {
13251
13991
  const target = redactWebhookUrl(channel.url);
13252
13992
  try {
@@ -13318,7 +14058,23 @@ async function fireWebhook(channel, payload, logger, tracer, delivery = {}) {
13318
14058
  async function fireChannels(channels, payload, logger, tracer, delivery = {}) {
13319
14059
  if (channels.length === 0) return true;
13320
14060
  const results = await Promise.all(
13321
- channels.filter((ch) => ch.type === "webhook").map((ch) => fireWebhook(ch, payload, logger, tracer, delivery))
14061
+ channels.map((ch) => {
14062
+ if (ch.type === "webhook")
14063
+ return fireWebhook(
14064
+ ch,
14065
+ payload,
14066
+ logger,
14067
+ tracer,
14068
+ delivery
14069
+ );
14070
+ if (ch.type === "email")
14071
+ return fireEmail(ch, payload, logger, tracer, delivery);
14072
+ logger.error("[alerts-evaluator] unknown channel type", {
14073
+ rule_id: payload.rule.id,
14074
+ type: String(ch.type)
14075
+ });
14076
+ return Promise.resolve(false);
14077
+ })
13322
14078
  );
13323
14079
  return results.every(Boolean);
13324
14080
  }
@@ -13349,19 +14105,47 @@ async function mapWithConcurrency(items, concurrency, mapper) {
13349
14105
  );
13350
14106
  return results;
13351
14107
  }
13352
- async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery, onAlertFiring) {
14108
+ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery, hold, onAlertFiring) {
13353
14109
  const none = { fired: 0, resolved: 0, redelivered: 0 };
14110
+ let previousForError = null;
13354
14111
  try {
13355
- const value = await withTimeout(
13356
- store.evaluateRule(rule),
14112
+ const priorState = await store.getState(rule.id, rule.projectId);
14113
+ const previous = priorState?.current_state ?? "ok";
14114
+ previousForError = previous;
14115
+ const raw = await withTimeout(
14116
+ store.evaluateRuleWithStatus ? store.evaluateRuleWithStatus(rule) : store.evaluateRule(rule),
13357
14117
  ruleTimeoutMs,
13358
14118
  `rule evaluation timed out after ${ruleTimeoutMs}ms`
13359
14119
  );
13360
- const shouldFire = compareValue(value, rule.threshold, rule.comparison);
13361
- const priorState = await store.getState(rule.id, rule.projectId);
13362
- const previous = priorState?.current_state ?? "ok";
13363
- const next = shouldFire ? "firing" : "ok";
14120
+ const { value, noData } = typeof raw === "number" ? { value: raw, noData: false } : raw;
13364
14121
  const now = (/* @__PURE__ */ new Date()).toISOString();
14122
+ const nowMs = Date.parse(now);
14123
+ const keepOnNoData = noData && rule.signal === "metrics" && rule.query.noData === "keep";
14124
+ const matches = !keepOnNoData && compareValue(value, rule.threshold, rule.comparison);
14125
+ const pendingSecs = rule.pendingSecs ?? 0;
14126
+ const pendingSince = priorState?.pending_since ?? null;
14127
+ const seenAt = priorState?.pending_seen_at ?? pendingSince;
14128
+ const holdBroken = pendingSince !== null && (seenAt === null || nowMs - Date.parse(seenAt) > hold.maxEvaluationGapMs || hold.holdStartedAt !== void 0 && seenAt < hold.holdStartedAt);
14129
+ let pendingActive = pendingSince !== null;
14130
+ let shouldFire = matches;
14131
+ if (matches && previous === "ok" && pendingSecs > 0) {
14132
+ const since = holdBroken || pendingSince === null ? now : pendingSince;
14133
+ await store.setPending(
14134
+ rule.id,
14135
+ rule.projectId,
14136
+ { since, seenAt: now },
14137
+ rule.createdAt
14138
+ );
14139
+ pendingActive = true;
14140
+ shouldFire = nowMs - Date.parse(since) >= pendingSecs * 1e3;
14141
+ }
14142
+ const next = keepOnNoData ? previous : shouldFire ? "firing" : "ok";
14143
+ const clearPending = async () => {
14144
+ if (pendingActive && (!matches || next === "firing")) {
14145
+ await store.setPending(rule.id, rule.projectId, null, rule.createdAt);
14146
+ }
14147
+ };
14148
+ const outcome = keepOnNoData ? "no_data" : "value";
13365
14149
  const analysisAttachment = rule.analysisId ? await store.getAnalysisNarrative(rule.projectId, rule.analysisId).then(
13366
14150
  (n) => n ? {
13367
14151
  id: rule.analysisId,
@@ -13387,6 +14171,7 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
13387
14171
  );
13388
14172
  if (previous !== next) {
13389
14173
  await store.transitionState(rule.id, rule.projectId, next, now);
14174
+ await clearPending();
13390
14175
  if (next === "firing" && onAlertFiring) {
13391
14176
  await onAlertFiring(rule).catch(() => void 0);
13392
14177
  }
@@ -13394,7 +14179,14 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
13394
14179
  if (ok) {
13395
14180
  await store.markNotified(rule.id, rule.projectId, next, now);
13396
14181
  }
13397
- await store.recordEvaluation(rule.id, rule.projectId, value, next, ok);
14182
+ await store.recordEvaluation(
14183
+ rule.id,
14184
+ rule.projectId,
14185
+ value,
14186
+ next,
14187
+ ok,
14188
+ outcome
14189
+ );
13398
14190
  if (!ok) {
13399
14191
  logger.warn(
13400
14192
  "[alerts-evaluator] state recorded but delivery failed; will retry",
@@ -13403,6 +14195,7 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
13403
14195
  }
13404
14196
  return next === "firing" ? { fired: 1, resolved: 0, redelivered: 0 } : { fired: 0, resolved: 1, redelivered: 0 };
13405
14197
  }
14198
+ await clearPending();
13406
14199
  const owed = priorState !== null && priorState.notified === 0;
13407
14200
  if (owed) {
13408
14201
  const ok = await deliver(previous);
@@ -13414,7 +14207,14 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
13414
14207
  priorState.last_state_change
13415
14208
  );
13416
14209
  }
13417
- await store.recordEvaluation(rule.id, rule.projectId, value, next, ok);
14210
+ await store.recordEvaluation(
14211
+ rule.id,
14212
+ rule.projectId,
14213
+ value,
14214
+ next,
14215
+ ok,
14216
+ outcome
14217
+ );
13418
14218
  if (!ok) {
13419
14219
  logger.warn("[alerts-evaluator] delivery retry failed; will retry", {
13420
14220
  rule_id: rule.id,
@@ -13425,13 +14225,32 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
13425
14225
  }
13426
14226
  return { fired: 0, resolved: 0, redelivered: 1 };
13427
14227
  }
13428
- await store.recordEvaluation(rule.id, rule.projectId, value, next, false);
14228
+ await store.recordEvaluation(
14229
+ rule.id,
14230
+ rule.projectId,
14231
+ value,
14232
+ next,
14233
+ false,
14234
+ outcome
14235
+ );
13429
14236
  } catch (err) {
14237
+ const message = err instanceof Error ? err.message : String(err);
13430
14238
  logger.error("[alerts-evaluator] rule evaluation failed", {
13431
14239
  rule_id: rule.id,
13432
14240
  project_id: rule.projectId,
13433
- error: err instanceof Error ? err.message : String(err)
14241
+ error: message
13434
14242
  });
14243
+ if (previousForError !== null) {
14244
+ await store.recordEvaluation(
14245
+ rule.id,
14246
+ rule.projectId,
14247
+ 0,
14248
+ previousForError,
14249
+ false,
14250
+ "error",
14251
+ message
14252
+ ).catch(() => void 0);
14253
+ }
13435
14254
  }
13436
14255
  return none;
13437
14256
  }
@@ -13439,7 +14258,8 @@ async function evaluateRuleBatch(store, rules, options) {
13439
14258
  const logger = options?.logger ?? consoleLogger;
13440
14259
  const delivery = {
13441
14260
  allowPrivateWebhookTargets: options?.allowPrivateWebhookTargets,
13442
- hostLookup: options?.hostLookup
14261
+ hostLookup: options?.hostLookup,
14262
+ email: options?.email
13443
14263
  };
13444
14264
  const concurrency = Math.max(
13445
14265
  1,
@@ -13459,6 +14279,10 @@ async function evaluateRuleBatch(store, rules, options) {
13459
14279
  options?.tracer,
13460
14280
  ruleTimeoutMs,
13461
14281
  delivery,
14282
+ {
14283
+ maxEvaluationGapMs: options?.maxEvaluationGapMs ?? DEFAULT_MAX_EVALUATION_GAP_MS,
14284
+ holdStartedAt: options?.holdStartedAt
14285
+ },
13462
14286
  options?.onAlertFiring
13463
14287
  )
13464
14288
  );
@@ -13474,7 +14298,10 @@ async function evaluateAllRules(env, options) {
13474
14298
  const tracer = options?.tracer;
13475
14299
  const allowPrivateWebhookTargets = env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true";
13476
14300
  const db = sqlDbFor(env);
13477
- const store = new AlertsStore(db, { allowPrivateWebhookTargets });
14301
+ const store = new AlertsStore(db, {
14302
+ allowPrivateWebhookTargets,
14303
+ emailDeliveryAvailable: Boolean(env.ALERT_EMAIL)
14304
+ });
13478
14305
  const rules = await store.listEnabledRules();
13479
14306
  return evaluateRuleBatch(store, rules, {
13480
14307
  logger,
@@ -13483,6 +14310,9 @@ async function evaluateAllRules(env, options) {
13483
14310
  ruleTimeoutMs: options?.ruleTimeoutMs,
13484
14311
  allowPrivateWebhookTargets,
13485
14312
  hostLookup: options?.hostLookup,
14313
+ email: env.ALERT_EMAIL,
14314
+ maxEvaluationGapMs: options?.maxEvaluationGapMs,
14315
+ holdStartedAt: options?.holdStartedAt,
13486
14316
  onAlertFiring: async (rule) => {
13487
14317
  const target = await resolveServiceIncidentTarget(
13488
14318
  db,
@@ -13511,7 +14341,8 @@ function createAlertEvaluatorHandler() {
13511
14341
  // src/plugins/alerts-routes.ts
13512
14342
  import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT5 } from "@obsunified/types";
13513
14343
  var storeOptions = (env) => ({
13514
- allowPrivateWebhookTargets: env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true"
14344
+ allowPrivateWebhookTargets: env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
14345
+ emailDeliveryAvailable: Boolean(env.ALERT_EMAIL)
13515
14346
  });
13516
14347
  var alertsRoutesPlugin = {
13517
14348
  name: "alerts-routes",
@@ -13588,9 +14419,19 @@ var alertsRoutesPlugin = {
13588
14419
  const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
13589
14420
  const rule = await store.getRule(id, projectId);
13590
14421
  if (!rule) return c.json({ error: "Rule not found" }, 404);
13591
- const value = await store.evaluateRule(rule);
14422
+ let value;
14423
+ let noData;
14424
+ try {
14425
+ ({ value, noData } = await store.evaluateRuleWithStatus(rule));
14426
+ } catch (err) {
14427
+ const message = err instanceof Error ? err.message : String(err);
14428
+ return c.json({ error: message }, 422);
14429
+ }
13592
14430
  const response = {
13593
14431
  value,
14432
+ noData,
14433
+ // The bare comparison: a rule with `pendingSecs` additionally
14434
+ // needs the condition to hold that long before it fires.
13594
14435
  wouldFire: compareValue(value, rule.threshold, rule.comparison),
13595
14436
  comparison: rule.comparison,
13596
14437
  threshold: rule.threshold,
@@ -16923,6 +17764,7 @@ var filterPprofByTraceId = async (profile, traceIdFilter) => {
16923
17764
  var DEFAULT_CHUNK_LIMIT = 100;
16924
17765
  var MAX_CHUNK_LIMIT = 500;
16925
17766
  var REPLAY_CHUNK_FETCH_CONCURRENCY = 8;
17767
+ var REPLAY_RESPONSE_BYTE_BUDGET = 64 * 1024 * 1024;
16926
17768
  async function mapWithConcurrency2(items, concurrency, mapper) {
16927
17769
  const results = new Array(items.length);
16928
17770
  let nextIndex = 0;
@@ -16937,13 +17779,30 @@ async function mapWithConcurrency2(items, concurrency, mapper) {
16937
17779
  );
16938
17780
  return results;
16939
17781
  }
17782
+ function selectReplayChunkPage(objects, offset, limit, byteBudget = REPLAY_RESPONSE_BYTE_BUDGET) {
17783
+ const window = objects.slice(offset, offset + limit);
17784
+ let bytes = 0;
17785
+ let count = 0;
17786
+ for (const obj of window) {
17787
+ const size = typeof obj.size === "number" && obj.size > 0 ? obj.size : 0;
17788
+ if (count > 0 && bytes + size > byteBudget) break;
17789
+ bytes += size;
17790
+ count++;
17791
+ }
17792
+ const selected = window.slice(0, count);
17793
+ const end = offset + count;
17794
+ return {
17795
+ selected,
17796
+ nextChunkOffset: end < objects.length ? end : null
17797
+ };
17798
+ }
16940
17799
  async function fetchReplayChunks(bucket, objects, concurrency = REPLAY_CHUNK_FETCH_CONCURRENCY) {
16941
- const chunks = await mapWithConcurrency2(objects, concurrency, async (obj) => {
17800
+ const chunks2 = await mapWithConcurrency2(objects, concurrency, async (obj) => {
16942
17801
  const objectData = await bucket.get(obj.key);
16943
17802
  if (!objectData) return [];
16944
17803
  return objectData.json();
16945
17804
  });
16946
- return chunks.flat();
17805
+ return chunks2.flat();
16947
17806
  }
16948
17807
  var replayQueryRoutesPlugin = {
16949
17808
  name: "replay-query-routes",
@@ -17003,15 +17862,11 @@ var replayQueryRoutesPlugin = {
17003
17862
  ) || DEFAULT_CHUNK_LIMIT
17004
17863
  )
17005
17864
  );
17006
- const selectedObjects = objects.slice(
17007
- chunkOffset,
17008
- chunkOffset + chunkLimit
17009
- );
17865
+ const { selected: selectedObjects, nextChunkOffset } = selectReplayChunkPage(objects, chunkOffset, chunkLimit);
17010
17866
  const events = await fetchReplayChunks(
17011
17867
  c.env.REPLAYS_BUCKET,
17012
17868
  selectedObjects
17013
17869
  );
17014
- const nextChunkOffset = chunkOffset + chunkLimit < objects.length ? chunkOffset + chunkLimit : null;
17015
17870
  return c.json({
17016
17871
  metadata,
17017
17872
  events,
@@ -19870,7 +20725,7 @@ var gunzip = async (input, maxBytes) => {
19870
20725
  );
19871
20726
  if (!stream) throw new OtlpDecodeError("Empty gzip body", 400);
19872
20727
  const reader = stream.getReader();
19873
- const chunks = [];
20728
+ const chunks2 = [];
19874
20729
  let total = 0;
19875
20730
  try {
19876
20731
  for (; ; ) {
@@ -19885,7 +20740,7 @@ var gunzip = async (input, maxBytes) => {
19885
20740
  413
19886
20741
  );
19887
20742
  }
19888
- chunks.push(value);
20743
+ chunks2.push(value);
19889
20744
  }
19890
20745
  } catch (err) {
19891
20746
  if (err instanceof OtlpDecodeError) throw err;
@@ -19893,7 +20748,7 @@ var gunzip = async (input, maxBytes) => {
19893
20748
  }
19894
20749
  const out = new Uint8Array(total);
19895
20750
  let offset = 0;
19896
- for (const chunk of chunks) {
20751
+ for (const chunk of chunks2) {
19897
20752
  out.set(chunk, offset);
19898
20753
  offset += chunk.byteLength;
19899
20754
  }
@@ -20022,19 +20877,34 @@ var anyValueToString = (v) => {
20022
20877
  return bytesToBase64(v.value.value);
20023
20878
  case "arrayValue":
20024
20879
  case "kvlistValue":
20025
- return JSON.stringify(adaptAnyValue(v));
20880
+ return JSON.stringify(adaptAnyValueAt(v, 1));
20026
20881
  default:
20027
20882
  return "";
20028
20883
  }
20029
20884
  };
20030
- var keyValuesToRecord = (kvs) => {
20885
+ var MAX_ANY_VALUE_DEPTH = 64;
20886
+ var TRUNCATED_DEPTH_MARKER = `[truncated: nested deeper than ${MAX_ANY_VALUE_DEPTH} levels]`;
20887
+ var setOwn = (out, key, value) => {
20888
+ if (key === "__proto__") {
20889
+ Object.defineProperty(out, key, {
20890
+ value,
20891
+ enumerable: true,
20892
+ writable: true,
20893
+ configurable: true
20894
+ });
20895
+ } else {
20896
+ out[key] = value;
20897
+ }
20898
+ };
20899
+ var keyValuesToRecord = (kvs) => keyValuesToRecordAt(kvs, 0);
20900
+ var keyValuesToRecordAt = (kvs, depth) => {
20031
20901
  const out = {};
20032
20902
  for (const kv of kvs) {
20033
- out[kv.key] = anyValueToJson(kv.value);
20903
+ setOwn(out, kv.key, anyValueToJson(kv.value, depth + 1));
20034
20904
  }
20035
20905
  return out;
20036
20906
  };
20037
- var anyValueToJson = (v) => {
20907
+ var anyValueToJson = (v, depth) => {
20038
20908
  if (!v) return null;
20039
20909
  switch (v.value.case) {
20040
20910
  case "stringValue":
@@ -20048,18 +20918,21 @@ var anyValueToJson = (v) => {
20048
20918
  case "bytesValue":
20049
20919
  return bytesToBase64(v.value.value);
20050
20920
  case "arrayValue":
20051
- return v.value.value.values.map((x) => anyValueToJson(x));
20921
+ if (depth > MAX_ANY_VALUE_DEPTH) return TRUNCATED_DEPTH_MARKER;
20922
+ return v.value.value.values.map((x) => anyValueToJson(x, depth + 1));
20052
20923
  case "kvlistValue":
20053
- return keyValuesToRecord(v.value.value.values);
20924
+ if (depth > MAX_ANY_VALUE_DEPTH) return TRUNCATED_DEPTH_MARKER;
20925
+ return keyValuesToRecordAt(v.value.value.values, depth);
20054
20926
  default:
20055
20927
  return null;
20056
20928
  }
20057
20929
  };
20058
- var adaptKeyValue = (kv) => ({
20930
+ var adaptKeyValue = (kv) => adaptKeyValueAt(kv, 0);
20931
+ var adaptKeyValueAt = (kv, depth) => ({
20059
20932
  key: kv.key,
20060
- value: kv.value ? adaptAnyValue(kv.value) : void 0
20933
+ value: kv.value ? adaptAnyValueAt(kv.value, depth + 1) : void 0
20061
20934
  });
20062
- var adaptAnyValue = (v) => {
20935
+ var adaptAnyValueAt = (v, depth) => {
20063
20936
  switch (v.value.case) {
20064
20937
  case "stringValue":
20065
20938
  return { stringValue: v.value.value };
@@ -20070,15 +20943,21 @@ var adaptAnyValue = (v) => {
20070
20943
  case "doubleValue":
20071
20944
  return { doubleValue: v.value.value };
20072
20945
  case "arrayValue":
20946
+ if (depth > MAX_ANY_VALUE_DEPTH)
20947
+ return { stringValue: TRUNCATED_DEPTH_MARKER };
20073
20948
  return {
20074
20949
  arrayValue: {
20075
- values: v.value.value.values.map((x) => adaptAnyValue(x))
20950
+ values: v.value.value.values.map(
20951
+ (x) => adaptAnyValueAt(x, depth + 1)
20952
+ )
20076
20953
  }
20077
20954
  };
20078
20955
  case "kvlistValue":
20956
+ if (depth > MAX_ANY_VALUE_DEPTH)
20957
+ return { stringValue: TRUNCATED_DEPTH_MARKER };
20079
20958
  return {
20080
20959
  kvlistValue: {
20081
- values: v.value.value.values.map(adaptKeyValue)
20960
+ values: v.value.value.values.map((kv) => adaptKeyValueAt(kv, depth))
20082
20961
  }
20083
20962
  };
20084
20963
  case "bytesValue":
@@ -20106,7 +20985,7 @@ var bytesToBase64 = (bytes) => {
20106
20985
  };
20107
20986
 
20108
20987
  // src/otlp/decode/logs.ts
20109
- var decodeLogsRequest = (body) => {
20988
+ var decodeLogsRequestCapped = (body, maxRecords) => {
20110
20989
  let msg;
20111
20990
  try {
20112
20991
  msg = body.wireFormat === "protobuf" ? fromBinary(ExportLogsServiceRequestSchema, body.bytes) : fromJson(ExportLogsServiceRequestSchema, decodeJsonBody(body.bytes));
@@ -20117,16 +20996,25 @@ var decodeLogsRequest = (body) => {
20117
20996
  );
20118
20997
  }
20119
20998
  const out = [];
20999
+ let rejected = 0;
20120
21000
  for (const rl of msg.resourceLogs) {
21001
+ if (out.length >= maxRecords) {
21002
+ for (const sl of rl.scopeLogs) rejected += sl.logRecords.length;
21003
+ continue;
21004
+ }
20121
21005
  const serviceName = extractServiceName(rl.resource?.attributes);
20122
21006
  for (const sl of rl.scopeLogs) {
20123
21007
  const loggerName = sl.scope?.name || null;
20124
21008
  for (const log of sl.logRecords) {
21009
+ if (out.length >= maxRecords) {
21010
+ rejected++;
21011
+ continue;
21012
+ }
20125
21013
  out.push(adaptLogRecord(log, serviceName, loggerName));
20126
21014
  }
20127
21015
  }
20128
21016
  }
20129
- return out;
21017
+ return { records: out, rejected };
20130
21018
  };
20131
21019
  var adaptLogRecord = (log, serviceName, loggerName) => {
20132
21020
  const ts = log.timeUnixNano || log.observedTimeUnixNano;
@@ -20176,7 +21064,7 @@ var ExportMetricsServiceResponseSchema = /* @__PURE__ */ messageDesc6(file_opent
20176
21064
  var ExportMetricsPartialSuccessSchema = /* @__PURE__ */ messageDesc6(file_opentelemetry_proto_collector_metrics_v1_metrics_service, 2);
20177
21065
 
20178
21066
  // src/otlp/decode/metrics.ts
20179
- var decodeMetricsRequest = (body) => {
21067
+ var decodeMetricsRequestCapped = (body, maxPoints) => {
20180
21068
  let msg;
20181
21069
  try {
20182
21070
  msg = body.wireFormat === "protobuf" ? fromBinary2(ExportMetricsServiceRequestSchema, body.bytes) : fromJson2(
@@ -20190,7 +21078,18 @@ var decodeMetricsRequest = (body) => {
20190
21078
  );
20191
21079
  }
20192
21080
  const out = [];
21081
+ let rejected = 0;
20193
21082
  for (const rm of msg.resourceMetrics) {
21083
+ if (out.length >= maxPoints) {
21084
+ for (const sm of rm.scopeMetrics) rejected += countPoints(sm);
21085
+ continue;
21086
+ }
21087
+ const remaining = maxPoints - out.length;
21088
+ const total = rm.scopeMetrics.reduce((n, sm) => n + countPoints(sm), 0);
21089
+ if (total > remaining) {
21090
+ rejected += total - remaining;
21091
+ trimScopeMetrics(rm.scopeMetrics, remaining);
21092
+ }
20194
21093
  const resourceAttrs = rm.resource?.attributes ?? [];
20195
21094
  const resourceAttrsJson = resourceAttrs.length ? JSON.stringify(keyValuesToRecord(resourceAttrs)) : null;
20196
21095
  const serviceName = extractServiceName(resourceAttrs);
@@ -20198,7 +21097,34 @@ var decodeMetricsRequest = (body) => {
20198
21097
  addMetrics(out, rm, sm, serviceName, resourceAttrsJson);
20199
21098
  }
20200
21099
  }
20201
- return out;
21100
+ return { points: out, rejected };
21101
+ };
21102
+ var metricPointCount = (metric) => {
21103
+ switch (metric.data.case) {
21104
+ case "gauge":
21105
+ case "sum":
21106
+ case "histogram":
21107
+ case "exponentialHistogram":
21108
+ case "summary":
21109
+ return metric.data.value.dataPoints.length;
21110
+ default:
21111
+ return 0;
21112
+ }
21113
+ };
21114
+ var countPoints = (sm) => sm.metrics.reduce((n, m) => n + metricPointCount(m), 0);
21115
+ var trimScopeMetrics = (scopeMetrics, budget) => {
21116
+ let left = budget;
21117
+ for (const sm of scopeMetrics) {
21118
+ for (const metric of sm.metrics) {
21119
+ const n = metricPointCount(metric);
21120
+ if (n === 0) continue;
21121
+ if (n > left) {
21122
+ const data = metric.data.value;
21123
+ data.dataPoints = data.dataPoints.slice(0, left);
21124
+ }
21125
+ left -= Math.min(n, left);
21126
+ }
21127
+ }
20202
21128
  };
20203
21129
  var addMetrics = (out, rm, sm, serviceName, resourceAttrsJson) => {
20204
21130
  const scopeName = sm.scope?.name || null;
@@ -20442,7 +21368,31 @@ var ExportTraceServiceResponseSchema = /* @__PURE__ */ messageDesc8(file_opentel
20442
21368
  var ExportTracePartialSuccessSchema = /* @__PURE__ */ messageDesc8(file_opentelemetry_proto_collector_trace_v1_trace_service, 2);
20443
21369
 
20444
21370
  // src/otlp/decode/traces.ts
20445
- var decodeTraceRequest = (body) => {
21371
+ var decodeTraceRequestCapped = (body, maxSpans) => {
21372
+ const msg = parseTraceMessage(body);
21373
+ let accepted = 0;
21374
+ let rejected = 0;
21375
+ const resourceSpans = [];
21376
+ for (const rs of msg.resourceSpans) {
21377
+ for (const ss of rs.scopeSpans) {
21378
+ const kept = [];
21379
+ for (const s of ss.spans) {
21380
+ if (!hasValidIds(s)) continue;
21381
+ if (accepted >= maxSpans) {
21382
+ rejected++;
21383
+ } else {
21384
+ kept.push(s);
21385
+ accepted++;
21386
+ }
21387
+ }
21388
+ ss.spans = kept;
21389
+ }
21390
+ resourceSpans.push(adaptResourceSpans(rs));
21391
+ }
21392
+ return { request: { resourceSpans }, accepted, rejected };
21393
+ };
21394
+ var hasValidIds = (s) => s.traceId.length === 16 && s.spanId.length === 8;
21395
+ var parseTraceMessage = (body) => {
20446
21396
  let msg;
20447
21397
  try {
20448
21398
  msg = body.wireFormat === "protobuf" ? fromBinary3(ExportTraceServiceRequestSchema, body.bytes) : fromJson3(ExportTraceServiceRequestSchema, decodeJsonBody(body.bytes));
@@ -20452,14 +21402,14 @@ var decodeTraceRequest = (body) => {
20452
21402
  400
20453
21403
  );
20454
21404
  }
20455
- return { resourceSpans: msg.resourceSpans.map(adaptResourceSpans) };
21405
+ return msg;
20456
21406
  };
20457
21407
  var adaptResourceSpans = (rs) => ({
20458
21408
  resource: rs.resource ? { attributes: rs.resource.attributes.map(adaptKeyValue) } : void 0,
20459
21409
  scopeSpans: rs.scopeSpans.map((ss) => ({
20460
21410
  scope: ss.scope ? { name: ss.scope.name, version: ss.scope.version } : void 0,
20461
21411
  spans: ss.spans.flatMap((s) => {
20462
- if (s.traceId.length !== 16 || s.spanId.length !== 8) return [];
21412
+ if (!hasValidIds(s)) return [];
20463
21413
  return [
20464
21414
  {
20465
21415
  traceId: bytesToHex(s.traceId),
@@ -20570,19 +21520,18 @@ var logsReceiverPlugin = {
20570
21520
  throw err;
20571
21521
  }
20572
21522
  let decoded;
21523
+ let rejected = 0;
20573
21524
  try {
20574
- decoded = decodeLogsRequest(body);
21525
+ ({ records: decoded, rejected } = decodeLogsRequestCapped(
21526
+ body,
21527
+ MAX_LOGS_PER_REQUEST
21528
+ ));
20575
21529
  } catch (err) {
20576
21530
  if (err instanceof OtlpDecodeError) {
20577
21531
  return c.json({ error: err.message }, err.status);
20578
21532
  }
20579
21533
  throw err;
20580
21534
  }
20581
- let rejected = 0;
20582
- if (decoded.length > MAX_LOGS_PER_REQUEST) {
20583
- rejected = decoded.length - MAX_LOGS_PER_REQUEST;
20584
- decoded = decoded.slice(0, MAX_LOGS_PER_REQUEST);
20585
- }
20586
21535
  const store = new LogsStore(sqlDbFor(c.env));
20587
21536
  const retentionHours = getConfiguredRetentionHours5(c.env.RETENTION_HOURS);
20588
21537
  const now = /* @__PURE__ */ new Date();
@@ -20691,19 +21640,18 @@ var metricsReceiverPlugin = {
20691
21640
  throw err;
20692
21641
  }
20693
21642
  let points;
21643
+ let rejected = 0;
20694
21644
  try {
20695
- points = decodeMetricsRequest(body);
21645
+ ({ points, rejected } = decodeMetricsRequestCapped(
21646
+ body,
21647
+ MAX_POINTS_PER_REQUEST
21648
+ ));
20696
21649
  } catch (err) {
20697
21650
  if (err instanceof OtlpDecodeError) {
20698
21651
  return c.json({ error: err.message }, err.status);
20699
21652
  }
20700
21653
  throw err;
20701
21654
  }
20702
- let rejected = 0;
20703
- if (points.length > MAX_POINTS_PER_REQUEST) {
20704
- rejected = points.length - MAX_POINTS_PER_REQUEST;
20705
- points = points.slice(0, MAX_POINTS_PER_REQUEST);
20706
- }
20707
21655
  const retentionHours = getConfiguredRetentionHours6(c.env.RETENTION_HOURS);
20708
21656
  const now = /* @__PURE__ */ new Date();
20709
21657
  const receivedAt = now.toISOString();
@@ -20960,34 +21908,18 @@ var otlpReceiverPlugin = {
20960
21908
  }
20961
21909
  throw err;
20962
21910
  }
20963
- let payload;
21911
+ let capped;
20964
21912
  try {
20965
- payload = decodeTraceRequest(body);
21913
+ capped = decodeTraceRequestCapped(body, MAX_SPANS_PER_REQUEST);
20966
21914
  } catch (err) {
20967
21915
  if (err instanceof OtlpDecodeError) {
20968
21916
  return c.json({ error: err.message }, err.status);
20969
21917
  }
20970
21918
  throw err;
20971
21919
  }
20972
- let total = 0;
20973
- let rejected = 0;
20974
- const trimmedResourceSpans = [];
20975
- for (const rs of payload.resourceSpans ?? []) {
20976
- const scopeSpans = [];
20977
- for (const ss of rs.scopeSpans ?? []) {
20978
- const spans2 = [];
20979
- for (const s of ss.spans ?? []) {
20980
- if (total >= MAX_SPANS_PER_REQUEST) {
20981
- rejected++;
20982
- } else {
20983
- spans2.push(s);
20984
- total++;
20985
- }
20986
- }
20987
- scopeSpans.push({ ...ss, spans: spans2 });
20988
- }
20989
- trimmedResourceSpans.push({ ...rs, scopeSpans });
20990
- }
21920
+ const total = capped.accepted;
21921
+ const rejected = capped.rejected;
21922
+ const trimmedResourceSpans = capped.request.resourceSpans;
20991
21923
  const parsedSpans = toStoredSpans(
20992
21924
  { resourceSpans: trimmedResourceSpans },
20993
21925
  projectId,
@@ -21190,6 +22122,7 @@ var profileId = () => {
21190
22122
  for (let i = 0; i < 16; i++) randStr += ENCODING[bytes[i] % 32];
21191
22123
  return timeStr + randStr;
21192
22124
  };
22125
+ var safeFilenamePart = (value) => value.replace(/[^A-Za-z0-9_.-]/g, "_");
21193
22126
  var parseNonNegInt = (value, fallback) => {
21194
22127
  if (value == null) return fallback;
21195
22128
  const parsed = Number.parseInt(value, 10);
@@ -21345,7 +22278,7 @@ var profileRoutesPlugin = {
21345
22278
  headers: {
21346
22279
  "Content-Type": "application/octet-stream",
21347
22280
  "Content-Encoding": "gzip",
21348
- "Content-Disposition": `attachment; filename="${id}-trace-${filterTraceId.slice(0, 8)}.pprof.gz"`,
22281
+ "Content-Disposition": `attachment; filename="${safeFilenamePart(id)}-trace-${safeFilenamePart(filterTraceId.slice(0, 8))}.pprof.gz"`,
21349
22282
  // Surface the size shrink so the UI can show savings.
21350
22283
  "X-Obs-Filter-Original-Bytes": String(raw.byteLength),
21351
22284
  "X-Obs-Filter-Filtered-Bytes": String(filtered.byteLength)
@@ -21365,7 +22298,7 @@ var profileRoutesPlugin = {
21365
22298
  return new Response(obj.body, {
21366
22299
  headers: {
21367
22300
  "Content-Type": "application/octet-stream",
21368
- "Content-Disposition": `attachment; filename="${id}.pprof.gz"`
22301
+ "Content-Disposition": `attachment; filename="${safeFilenamePart(id)}.pprof.gz"`
21369
22302
  }
21370
22303
  });
21371
22304
  }
@@ -23413,16 +24346,16 @@ var BlobStoreToR2Adapter = class {
23413
24346
  };
23414
24347
  var streamToBytes = async (stream) => {
23415
24348
  const reader = stream.getReader();
23416
- const chunks = [];
24349
+ const chunks2 = [];
23417
24350
  while (true) {
23418
24351
  const { value, done } = await reader.read();
23419
24352
  if (done) break;
23420
- if (value) chunks.push(value);
24353
+ if (value) chunks2.push(value);
23421
24354
  }
23422
- const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
24355
+ const total = chunks2.reduce((sum, chunk) => sum + chunk.byteLength, 0);
23423
24356
  const out = new Uint8Array(total);
23424
24357
  let offset = 0;
23425
- for (const chunk of chunks) {
24358
+ for (const chunk of chunks2) {
23426
24359
  out.set(chunk, offset);
23427
24360
  offset += chunk.byteLength;
23428
24361
  }
@@ -23532,16 +24465,16 @@ var toUint8Array = (value) => {
23532
24465
  };
23533
24466
  var streamToBytes2 = async (stream) => {
23534
24467
  const reader = stream.getReader();
23535
- const chunks = [];
24468
+ const chunks2 = [];
23536
24469
  while (true) {
23537
24470
  const { value, done } = await reader.read();
23538
24471
  if (done) break;
23539
- if (value) chunks.push(value);
24472
+ if (value) chunks2.push(value);
23540
24473
  }
23541
- const total = chunks.reduce((n, c) => n + c.byteLength, 0);
24474
+ const total = chunks2.reduce((n, c) => n + c.byteLength, 0);
23542
24475
  const out = new Uint8Array(total);
23543
24476
  let offset = 0;
23544
- for (const c of chunks) {
24477
+ for (const c of chunks2) {
23545
24478
  out.set(c, offset);
23546
24479
  offset += c.byteLength;
23547
24480
  }
@@ -23979,7 +24912,8 @@ var createDefaultCollectorApp = (config) => createTelemetryCollectorApp({
23979
24912
  dashboardAuth: config?.dashboardAuth,
23980
24913
  logger: config?.logger,
23981
24914
  withChildSpan: config?.withChildSpan,
23982
- sqlDb: config?.sqlDb
24915
+ sqlDb: config?.sqlDb,
24916
+ allowUnauthenticatedIngest: config?.allowUnauthenticatedIngest
23983
24917
  });
23984
24918
  var src_default = createTelemetryCollectorApp(allPlugins);
23985
24919
  export {
@@ -24034,11 +24968,14 @@ export {
24034
24968
  dispatchIncidentWebhook,
24035
24969
  evalCasesRoutesPlugin,
24036
24970
  evaluateAllRules,
24971
+ evaluateMetricQuery,
24037
24972
  evaluateRemediationWatchdogs,
24973
+ evaluateRuleBatch,
24038
24974
  evidenceRetrievalRoutesPlugin,
24039
24975
  extractSpanLineageAndTopology,
24040
24976
  failRemediationProposal,
24041
24977
  findCrossTraceContention,
24978
+ formatAlertEmail,
24042
24979
  genAiNormalizerPlugin,
24043
24980
  getChangeEvent,
24044
24981
  getIncidentInvestigation,
@@ -24048,6 +24985,7 @@ export {
24048
24985
  identityReceiverPlugin,
24049
24986
  isEvalCaseSourceType,
24050
24987
  isEvalRunStatus,
24988
+ isValidEmailAddress,
24051
24989
  issueEnrichmentPlugin,
24052
24990
  issueInsightsPlugin,
24053
24991
  listChangeEvents,
@@ -24069,6 +25007,7 @@ export {
24069
25007
  recordChangeEvent,
24070
25008
  recordInvestigationFinding,
24071
25009
  redactionProcessorPlugin,
25010
+ reduceSeries,
24072
25011
  rejectRemediationProposal,
24073
25012
  replayQueryRoutesPlugin,
24074
25013
  replayReceiverPlugin,
@@ -24083,6 +25022,7 @@ export {
24083
25022
  usageReceiverPlugin,
24084
25023
  usersQueryRoutesPlugin,
24085
25024
  utmEnrichmentPlugin,
25025
+ validateMetricQuery,
24086
25026
  verifyPairedCounterfactualProof
24087
25027
  };
24088
25028
  //# sourceMappingURL=index.js.map