@obsunified/collector 2.3.0 → 2.4.3

package/dist/index.js

@@ -5798,7 +5798,11 @@ var MetricsStore = class {
   async ingestBatch(opts) {
     const { projectId, points, receivedAt, expiresAt } = opts;
     if (points.length === 0) return;
-    const seriesByIdentity = await this.resolveSeries(projectId, points);
+    const seriesByIdentity = await this.resolveSeries(
+      projectId,
+      points,
+      receivedAt
+    );
     const pointStmt = this.db.prepare(`
 			INSERT INTO metric_point (
 				id, series_id, project_id, ts_ns, start_ts_ns,
@@ -5917,7 +5921,7 @@ var MetricsStore = class {
    * Ensures a `metric_series` row exists for every unique identity in the
    * batch. Returns a map of identity → series row for point linking.
    */
-  async resolveSeries(projectId, points) {
+  async resolveSeries(projectId, points, receivedAt) {
     const byIdentity = /* @__PURE__ */ new Map();
     for (const p of points) {
       if (!byIdentity.has(p.identity)) byIdentity.set(p.identity, p);
@@ -5944,7 +5948,7 @@ var MetricsStore = class {
 		`);
     const toInsert = [];
     const attemptedIds = /* @__PURE__ */ new Set();
-    const createdAt = (/* @__PURE__ */ new Date()).toISOString();
+    const createdAt = receivedAt;
     for (const [identity, p] of byIdentity.entries()) {
       if (existing.has(identity)) continue;
       const id = crypto.randomUUID();
@@ -7085,7 +7089,7 @@ async function selectTraceCandidates(db, options) {
   }
   if (options.search) {
     const term = `%${options.search}%`;
-    whereClause += " AND (span_name LIKE ? OR status_message LIKE ? OR attributes_json LIKE ? OR events_json LIKE ?)";
+    whereClause += " AND (span_name LIKE ? OR status_message LIKE ? OR CAST(attributes_json AS TEXT) LIKE ? OR CAST(events_json AS TEXT) LIKE ?)";
     binds.push(term, term, term, term);
   }
   const having = options.status === "error" ? "HAVING SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) > 0" : options.status === "ok" ? "HAVING SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) = 0" : "";
@@ -12872,8 +12876,426 @@ var aiSpanPayloadsProcessorPlugin = {
   }
 };
 
+// src/lib/alert-email.ts
+var MAX_EMAIL_RECIPIENTS = 20;
+var EMAIL_RE = /^[^\s@<>,;:"()[\]\\]+@[^\s@<>,;:"()[\]\\]+\.[^\s@<>,;:"()[\]\\]+$/;
+var isValidEmailAddress = (addr) => addr.length <= 254 && EMAIL_RE.test(addr);
+var oneLine = (s) => s.replace(/[\r\n]+/g, " ").trim();
+function formatAlertEmail(payload) {
+  const label = payload.state === "firing" ? "FIRING" : "RESOLVED";
+  const name = oneLine(payload.rule.name);
+  const lines = [
+    payload.state === "firing" ? `Alert rule "${name}" is firing.` : `Alert rule "${name}" has resolved.`,
+    "",
+    `Rule:        ${name} (${payload.rule.id})`,
+    `Signal:      ${payload.rule.signal}`,
+    `State:       ${payload.state}`,
+    `Value:       ${payload.value}`,
+    `Condition:   value ${payload.comparison} ${payload.threshold}`,
+    `Threshold:   ${payload.threshold}`,
+    `Comparison:  ${payload.comparison}`,
+    `Evaluated:   ${payload.evaluatedAt}`,
+    `Project:     ${payload.projectId}`
+  ];
+  if (payload.analysis) {
+    lines.push(
+      `Analysis:    ${payload.analysis.id}${payload.analysis.status ? ` (${payload.analysis.status})` : ""}`
+    );
+    if (payload.analysis.narrative) {
+      lines.push("", payload.analysis.narrative);
+    }
+  }
+  lines.push("", "Sent by obs-unified.");
+  return {
+    subject: `[obs-unified] ${label}: ${name}`,
+    text: `${lines.join("\n")}
+`
+  };
+}
+
 // src/lib/alerts-store.ts
 import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT4 } from "@obsunified/types";
+
+// src/lib/metric-alerts.ts
+var METRIC_AGGREGATIONS = [
+  "last",
+  "min",
+  "max",
+  "avg",
+  "sum",
+  "increase",
+  "rate",
+  "age_seconds"
+];
+var METRIC_FIELDS = [
+  "value",
+  "count",
+  "sum"
+];
+var TEMPORALITY_DELTA = 1;
+var MAX_METRIC_NAMES = 20;
+var MAX_ATTRIBUTE_KEYS = 32;
+var MAX_MATCHED_SERIES = 1e3;
+var MAX_LOADED_POINTS = 5e4;
+var MAX_METRIC_WINDOW_MINS = 1440;
+var DEFAULT_STALE_SECS = 300;
+var MAX_STALE_SECS = 86400;
+var METRIC_NO_DATA_POLICIES = ["zero", "keep"];
+var ID_CHUNK = 50;
+var SELECTOR_KEYS = [
+  "metric",
+  "attributes",
+  "resourceAttributes",
+  "serviceName",
+  "field"
+];
+var QUERY_KEYS = [
+  ...SELECTOR_KEYS,
+  "aggregation",
+  "denominator",
+  "minDenominator",
+  "staleSecs",
+  "noData"
+];
+var DENOMINATOR_KEYS = [...SELECTOR_KEYS, "aggregation"];
+var isPlainObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+var validateAttributes = (value, path) => {
+  if (value === void 0) return;
+  if (!isPlainObject(value))
+    throw new Error(`${path} must be an object of attribute: value pairs`);
+  const entries = Object.entries(value);
+  if (entries.length > MAX_ATTRIBUTE_KEYS)
+    throw new Error(`${path} may list at most ${MAX_ATTRIBUTE_KEYS} keys`);
+  for (const [key, v] of entries) {
+    if (!key.trim()) throw new Error(`${path} has an empty attribute name`);
+    const ok = typeof v === "string" || typeof v === "boolean" || typeof v === "number" && Number.isFinite(v);
+    if (!ok)
+      throw new Error(
+        `${path}.${key} must be a string, finite number or boolean`
+      );
+  }
+};
+var validateSelector = (q, path, allowedKeys) => {
+  for (const key of Object.keys(q)) {
+    if (!allowedKeys.includes(key))
+      throw new Error(
+        `${path}.${key} is not a metrics query field (allowed: ${allowedKeys.join(", ")})`
+      );
+  }
+  const metric = q.metric;
+  if (typeof metric === "string") {
+    if (!metric.trim()) throw new Error(`${path}.metric must not be empty`);
+  } else if (Array.isArray(metric)) {
+    if (metric.length === 0)
+      throw new Error(`${path}.metric must name at least one metric`);
+    if (metric.length > MAX_METRIC_NAMES)
+      throw new Error(
+        `${path}.metric may name at most ${MAX_METRIC_NAMES} metrics`
+      );
+    for (const m of metric) {
+      if (typeof m !== "string" || !m.trim())
+        throw new Error(`${path}.metric entries must be non-empty strings`);
+    }
+  } else {
+    throw new Error(
+      `${path}.metric is required: a metric name or a list of names`
+    );
+  }
+  validateAttributes(q.attributes, `${path}.attributes`);
+  validateAttributes(q.resourceAttributes, `${path}.resourceAttributes`);
+  if (q.serviceName !== void 0 && (typeof q.serviceName !== "string" || !q.serviceName.trim()))
+    throw new Error(`${path}.serviceName must be a non-empty string`);
+  if (q.field !== void 0 && !METRIC_FIELDS.includes(q.field))
+    throw new Error(
+      `${path}.field must be one of: ${METRIC_FIELDS.join(", ")}`
+    );
+};
+function validateMetricQuery(query) {
+  if (!isPlainObject(query))
+    throw new Error("query must be an object for the metrics signal");
+  validateSelector(query, "query", QUERY_KEYS);
+  const aggregation = query.aggregation;
+  if (!METRIC_AGGREGATIONS.includes(aggregation))
+    throw new Error(
+      `query.aggregation must be one of: ${METRIC_AGGREGATIONS.join(", ")}`
+    );
+  if (query.staleSecs !== void 0 && (!Number.isInteger(query.staleSecs) || query.staleSecs < 1 || query.staleSecs > MAX_STALE_SECS))
+    throw new Error(
+      `query.staleSecs must be an integer from 1 to ${MAX_STALE_SECS}`
+    );
+  if (query.noData !== void 0 && !METRIC_NO_DATA_POLICIES.includes(query.noData))
+    throw new Error(
+      `query.noData must be one of: ${METRIC_NO_DATA_POLICIES.join(", ")}`
+    );
+  if (query.denominator === void 0) {
+    if (query.minDenominator !== void 0)
+      throw new Error("query.minDenominator requires query.denominator");
+    return;
+  }
+  if (aggregation === "age_seconds")
+    throw new Error("query.denominator cannot be used with age_seconds");
+  if (!isPlainObject(query.denominator))
+    throw new Error("query.denominator must be an object");
+  validateSelector(query.denominator, "query.denominator", DENOMINATOR_KEYS);
+  const denAgg = query.denominator.aggregation;
+  if (denAgg !== void 0) {
+    if (denAgg === "age_seconds" || !METRIC_AGGREGATIONS.includes(denAgg))
+      throw new Error(
+        `query.denominator.aggregation must be one of: ${METRIC_AGGREGATIONS.filter((a) => a !== "age_seconds").join(", ")}`
+      );
+  }
+  if (query.minDenominator !== void 0 && (typeof query.minDenominator !== "number" || !Number.isFinite(query.minDenominator) || query.minDenominator < 0))
+    throw new Error("query.minDenominator must be a finite number >= 0");
+}
+var toBigInt = (ns) => {
+  if (!ns) return null;
+  try {
+    return BigInt(ns);
+  } catch {
+    return null;
+  }
+};
+var readField = (p, field) => {
+  const raw = field === "count" ? p.count : field === "sum" ? p.sum : p.value;
+  const n = typeof raw === "string" ? Number(raw) : raw;
+  return typeof n === "number" && Number.isFinite(n) ? n : null;
+};
+var sortPoints = (points) => [...points].sort((a, b) => {
+  const ta = toBigInt(a.tsNs) ?? 0n;
+  const tb = toBigInt(b.tsNs) ?? 0n;
+  if (ta !== tb) return ta < tb ? -1 : 1;
+  return a.receivedAt < b.receivedAt ? -1 : a.receivedAt > b.receivedAt ? 1 : 0;
+});
+var cumulativeIncrease = (points, windowStartNs, bornInWindow) => {
+  if (points.length === 0) return 0;
+  const first = points[0];
+  const countsFirst = first.start !== null ? first.start >= windowStartNs : bornInWindow;
+  let total = countsFirst ? first.v : 0;
+  for (let i = 1; i < points.length; i++) {
+    const prev = points[i - 1];
+    const cur = points[i];
+    const restarted = cur.start !== null && prev.start !== null && cur.start !== prev.start;
+    if (restarted || cur.v < prev.v) total += cur.v;
+    else total += cur.v - prev.v;
+  }
+  return total;
+};
+function reduceSeries(points, opts) {
+  const usable = [];
+  for (const p of sortPoints(points)) {
+    const v = readField(p, opts.field);
+    if (v !== null) usable.push({ v, start: toBigInt(p.startTsNs) });
+  }
+  if (usable.length === 0) return null;
+  const values = usable.map((u) => u.v);
+  switch (opts.aggregation) {
+    case "last":
+      return values[values.length - 1];
+    case "min":
+      return values.reduce((a, b) => b < a ? b : a);
+    case "max":
+      return values.reduce((a, b) => b > a ? b : a);
+    case "sum":
+      return values.reduce((a, b) => a + b, 0);
+    case "avg":
+      return values.reduce((a, b) => a + b, 0) / values.length;
+    case "increase":
+    case "rate": {
+      const increase = opts.temporality === TEMPORALITY_DELTA ? values.reduce((a, b) => a + b, 0) : cumulativeIncrease(
+        usable,
+        opts.windowStartNs,
+        opts.bornInWindow ?? false
+      );
+      return opts.aggregation === "rate" ? increase / opts.windowSecs : increase;
+    }
+  }
+}
+var metricRatio = (numerator, denominator, minDenominator) => denominator > 0 && denominator >= minDenominator ? numerator / denominator : 0;
+var parseAttrs2 = (raw) => {
+  if (raw === null || raw === void 0) return {};
+  if (typeof raw === "string") {
+    try {
+      const parsed = JSON.parse(raw);
+      return isPlainObject(parsed) ? parsed : {};
+    } catch {
+      return {};
+    }
+  }
+  return isPlainObject(raw) ? raw : {};
+};
+var attrsMatch = (stored, wanted) => {
+  if (!wanted) return true;
+  for (const [key, expected] of Object.entries(wanted)) {
+    const actual = stored[key];
+    if (actual === void 0 || actual === null) return false;
+    if (typeof actual === "object") return false;
+    if (String(actual) !== String(expected)) return false;
+  }
+  return true;
+};
+var metricNames = (selector) => [
+  ...new Set(
+    (Array.isArray(selector.metric) ? selector.metric : [selector.metric]).map(
+      (m) => m.trim()
+    )
+  )
+];
+var chunks = (items, size) => {
+  const out = [];
+  for (let i = 0; i < items.length; i += size)
+    out.push(items.slice(i, i + size));
+  return out;
+};
+async function matchingSeries(db, projectId, selector) {
+  const names = metricNames(selector);
+  let sql = `SELECT id, temporality, created_at, attributes_json, resource_attrs_json
+		FROM metric_series
+		WHERE project_id = ? AND name IN (${names.map(() => "?").join(", ")})`;
+  const binds = [projectId, ...names];
+  if (selector.serviceName) {
+    sql += ` AND service_name = ?`;
+    binds.push(selector.serviceName);
+  }
+  const rs = await db.prepare(sql).bind(...binds).all();
+  const matched = (rs.results ?? []).filter(
+    (row) => attrsMatch(parseAttrs2(row.attributes_json), selector.attributes) && attrsMatch(
+      parseAttrs2(row.resource_attrs_json),
+      selector.resourceAttributes
+    )
+  );
+  if (matched.length > MAX_MATCHED_SERIES)
+    throw new Error(
+      `metrics query matches ${matched.length} series (limit ${MAX_MATCHED_SERIES}); narrow it with attributes or serviceName`
+    );
+  return matched.map((row) => ({
+    id: row.id,
+    createdAt: row.created_at,
+    temporality: row.temporality === null || row.temporality === void 0 ? null : Number(row.temporality)
+  }));
+}
+var STALENESS_APPLIES = /* @__PURE__ */ new Set([
+  "last",
+  "min",
+  "max",
+  "avg",
+  "sum"
+]);
+async function evaluateSelector(db, selector, aggregation, ctx) {
+  const series = await matchingSeries(db, ctx.projectId, selector);
+  if (series.length === 0) return { value: 0, contributing: 0 };
+  const windowStartMs = ctx.now.getTime() - ctx.windowMins * 6e4;
+  const cutoff = new Date(windowStartMs).toISOString();
+  const bySeries = /* @__PURE__ */ new Map();
+  let loaded = 0;
+  for (const chunk of chunks(series, ID_CHUNK)) {
+    const cap = ctx.maxLoadedPoints ?? MAX_LOADED_POINTS;
+    const budget = cap - loaded;
+    const rs = await db.prepare(
+      `SELECT series_id, ts_ns, start_ts_ns, value, count, sum, received_at
+				FROM metric_point
+				WHERE project_id = ? AND series_id IN (${chunk.map(() => "?").join(", ")})
+				  AND received_at >= ?
+				LIMIT ?`
+    ).bind(ctx.projectId, ...chunk.map((s) => s.id), cutoff, budget + 1).all();
+    const rows = rs.results ?? [];
+    loaded += rows.length;
+    if (loaded > cap)
+      throw new Error(
+        `metrics query loads more than ${cap} points in its window; shorten windowMins or narrow the query`
+      );
+    for (const row of rows) {
+      const list = bySeries.get(row.series_id) ?? [];
+      list.push({
+        tsNs: String(row.ts_ns),
+        startTsNs: row.start_ts_ns === null ? null : String(row.start_ts_ns),
+        value: row.value,
+        count: row.count,
+        sum: row.sum,
+        receivedAt: row.received_at
+      });
+      bySeries.set(row.series_id, list);
+    }
+  }
+  const field = selector.field ?? "value";
+  const windowStartNs = BigInt(windowStartMs) * 1000000n;
+  const freshCutoff = new Date(
+    ctx.now.getTime() - (ctx.staleSecs ?? DEFAULT_STALE_SECS) * 1e3
+  ).toISOString();
+  let total = 0;
+  let contributing = 0;
+  for (const s of series) {
+    const points = bySeries.get(s.id) ?? [];
+    if (STALENESS_APPLIES.has(aggregation)) {
+      const newest = points.reduce(
+        (max, p) => max === null || p.receivedAt > max ? p.receivedAt : max,
+        null
+      );
+      if (newest === null || newest < freshCutoff) continue;
+    }
+    const reduced = reduceSeries(points, {
+      aggregation,
+      field,
+      temporality: s.temporality,
+      windowStartNs,
+      windowSecs: ctx.windowMins * 60,
+      bornInWindow: s.createdAt >= cutoff
+    });
+    if (reduced !== null) {
+      total += reduced;
+      contributing += 1;
+    }
+  }
+  return { value: total, contributing };
+}
+async function ageSeconds(db, selector, ctx) {
+  const series = await matchingSeries(db, ctx.projectId, selector);
+  let newest = null;
+  for (const chunk of chunks(series, ID_CHUNK)) {
+    const row = await db.prepare(
+      `SELECT MAX(received_at) AS newest FROM metric_point
+				WHERE project_id = ? AND series_id IN (${chunk.map(() => "?").join(", ")})`
+    ).bind(ctx.projectId, ...chunk.map((s) => s.id)).first();
+    if (row?.newest && (newest === null || row.newest > newest))
+      newest = row.newest;
+  }
+  const reference = Date.parse(newest ?? ctx.ruleCreatedAt);
+  if (!Number.isFinite(reference)) return 0;
+  return Math.max(0, (ctx.now.getTime() - reference) / 1e3);
+}
+async function evaluateMetricQuery(db, query, ctx) {
+  if (query.aggregation === "age_seconds")
+    return { value: await ageSeconds(db, query, ctx), noData: false };
+  const withStale = { ...ctx, staleSecs: query.staleSecs ?? ctx.staleSecs };
+  const numerator = await evaluateSelector(
+    db,
+    query,
+    query.aggregation,
+    withStale
+  );
+  if (!query.denominator)
+    return {
+      value: numerator.value,
+      noData: numerator.contributing === 0
+    };
+  const denominator = await evaluateSelector(
+    db,
+    query.denominator,
+    query.denominator.aggregation ?? query.aggregation,
+    withStale
+  );
+  return {
+    value: metricRatio(
+      numerator.value,
+      denominator.value,
+      query.minDenominator ?? 0
+    ),
+    noData: numerator.contributing === 0 && denominator.contributing === 0
+  };
+}
+
+// src/lib/alerts-store.ts
+var MAX_PENDING_SECS = 86400;
+var EMAIL_UNAVAILABLE_MESSAGE = "email channels need SMTP configured on the collector (ALERT_SMTP_HOST and ALERT_SMTP_FROM on the Node collector); the Cloudflare Worker collector cannot send email";
 var parseJsonField = (value) => {
   if (typeof value === "string") return JSON.parse(value);
   return value;
@@ -12892,6 +13314,8 @@ var rowToRule = (row, state) => ({
   createdAt: row.created_at,
   updatedAt: row.updated_at,
   analysisId: row.analysis_id ?? null,
+  pendingSecs: Number(row.pending_secs ?? 0),
+  pendingSince: state?.pending_since ?? null,
   currentState: state?.current_state,
   lastStateChange: state?.last_state_change ?? null
 });
@@ -12902,11 +13326,18 @@ var rowToEvaluation = (row) => ({
   evaluatedAt: row.evaluated_at,
   value: row.value,
   state: row.state,
-  notified: row.notified === 1
+  notified: row.notified === 1,
+  outcome: row.outcome ?? "value",
+  error: row.error_message ?? null
 });
+var MAX_ERROR_MESSAGE = 500;
+var sanitizeEvaluationError = (message) => {
+  const cleaned = message.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s"'<>]+/gi, "<url>").replace(/\s+/g, " ").trim();
+  return cleaned.length > MAX_ERROR_MESSAGE ? `${cleaned.slice(0, MAX_ERROR_MESSAGE - 1)}\u2026` : cleaned;
+};
 var windowCutoffIso = (windowMins) => new Date(Date.now() - windowMins * 60 * 1e3).toISOString();
-var SIGNALS2 = ["spans", "logs", "usage", "ai"];
-var COMPARISONS = [">", ">=", "<", "<="];
+var SIGNALS2 = ["spans", "logs", "usage", "ai", "metrics"];
+var COMPARISONS = [">", ">=", "<", "<=", "==", "!="];
 function compareValue(value, threshold, comparison) {
   switch (comparison) {
     case ">":
@@ -12917,6 +13348,10 @@ function compareValue(value, threshold, comparison) {
       return value < threshold;
     case "<=":
       return value <= threshold;
+    case "==":
+      return value === threshold;
+    case "!=":
+      return value !== threshold;
   }
 }
 var AlertsStore = class {
@@ -12934,11 +13369,46 @@ var AlertsStore = class {
       throw new Error("windowMins must be >= 1");
     if (!COMPARISONS.includes(input.comparison))
       throw new Error(`comparison must be one of: ${COMPARISONS.join(", ")}`);
+    if (input.pendingSecs !== void 0 && (!Number.isInteger(input.pendingSecs) || input.pendingSecs < 0 || input.pendingSecs > MAX_PENDING_SECS))
+      throw new Error(
+        `pendingSecs must be an integer from 0 to ${MAX_PENDING_SECS}`
+      );
+    if (input.signal === "metrics") {
+      validateMetricQuery(input.query);
+      if (input.windowMins > MAX_METRIC_WINDOW_MINS)
+        throw new Error(
+          `windowMins must be <= ${MAX_METRIC_WINDOW_MINS} for the metrics signal`
+        );
+    }
     if (!Array.isArray(input.channels) || input.channels.length === 0)
       throw new Error("at least one channel is required");
     for (const ch of input.channels) {
+      if (ch.type === "email") {
+        if (!Array.isArray(ch.to) || ch.to.length === 0)
+          throw new Error("email channel.to must list at least one address");
+        if (ch.to.length > MAX_EMAIL_RECIPIENTS)
+          throw new Error(
+            `email channel.to may list at most ${MAX_EMAIL_RECIPIENTS} addresses`
+          );
+        for (const addr of ch.to) {
+          if (typeof addr !== "string" || !isValidEmailAddress(addr))
+            throw new Error(
+              `email channel.to has an invalid address: ${JSON.stringify(addr)}`
+            );
+        }
+        const extra = Object.keys(ch).filter((k) => k !== "type" && k !== "to");
+        if (extra.length > 0)
+          throw new Error(
+            `email channel accepts only "type" and "to" (SMTP settings come from the collector environment); got: ${extra.join(", ")}`
+          );
+        if (!this.options.emailDeliveryAvailable)
+          throw new Error(EMAIL_UNAVAILABLE_MESSAGE);
+        continue;
+      }
       if (ch.type !== "webhook")
-        throw new Error(`unknown channel type: ${ch.type}`);
+        throw new Error(
+          `unknown channel type: ${ch.type}`
+        );
       if (!ch.url || typeof ch.url !== "string")
         throw new Error("channel.url must be http(s)://");
       try {
@@ -12961,7 +13431,55 @@ var AlertsStore = class {
     const stateByRule = new Map(
       (stateRs.results ?? []).map((s) => [s.rule_id, s])
     );
-    return rows.map((r) => rowToRule(r, stateByRule.get(r.id)));
+    const lastByRule = await this.lastEvaluations(projectId);
+    return rows.map((r) => {
+      const state = stateByRule.get(r.id);
+      const last = lastByRule.get(r.id);
+      return {
+        ...rowToRule(r, state),
+        notificationDelivered: state ? state.notified === void 0 || state.notified === null ? true : Number(state.notified) === 1 : null,
+        lastEvaluation: last ? {
+          evaluatedAt: last.evaluated_at,
+          outcome: last.outcome ?? "value",
+          value: Number(last.value),
+          error: (last.outcome ?? "value") === "error" ? last.error_message ?? null : null
+        } : null
+      };
+    });
+  }
+  /**
+   * The newest evaluation of every rule in the project, one query. Falls
+   * back to the pre-057/059 column set on older databases.
+   */
+  async lastEvaluations(projectId) {
+    const run = (columns) => this.db.prepare(
+      `SELECT e.rule_id, e.evaluated_at, e.value${columns}
+				 FROM alert_evaluations e
+				 WHERE e.project_id = ?
+				   AND e.evaluated_at = (
+				     SELECT MAX(e2.evaluated_at) FROM alert_evaluations e2
+				     WHERE e2.rule_id = e.rule_id
+				   )`
+    ).bind(projectId).all();
+    let rs;
+    try {
+      rs = await run(", e.outcome, e.error_message");
+    } catch (err) {
+      if (!isMissingColumn(err, "error_message") && !isMissingColumn(err, "outcome"))
+        throw err;
+      try {
+        rs = await run(", e.outcome");
+      } catch (inner) {
+        if (!isMissingColumn(inner, "outcome")) throw inner;
+        rs = await run("");
+      }
+    }
+    const out = /* @__PURE__ */ new Map();
+    for (const row of rs.results ?? []) {
+      const seen = out.get(row.rule_id);
+      if (!seen || row.outcome === "error") out.set(row.rule_id, row);
+    }
+    return out;
   }
   async getRule(id, projectId) {
     const row = await this.db.prepare(`SELECT * FROM alert_rules WHERE id = ? AND project_id = ?`).bind(id, projectId).first();
@@ -12973,10 +13491,8 @@ var AlertsStore = class {
     this.validateInput(input);
     const id = randomHex(16);
     const now = (/* @__PURE__ */ new Date()).toISOString();
-    await this.db.prepare(
-      `INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at, analysis_id)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
-    ).bind(
+    const pendingSecs = input.pendingSecs ?? 0;
+    const values = [
       id,
       projectId,
       input.name.trim(),
@@ -12990,7 +13506,18 @@ var AlertsStore = class {
       now,
       now,
       input.analysisId ?? null
-    ).run();
+    ];
+    await this.withPendingColumnFallback(
+      pendingSecs,
+      () => this.db.prepare(
+        `INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at, analysis_id, pending_secs)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+      ).bind(...values, pendingSecs).run(),
+      () => this.db.prepare(
+        `INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at, analysis_id)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+      ).bind(...values).run()
+    );
     const created = await this.getRule(id, projectId);
     if (!created) throw new Error("Failed to create rule");
     return created;
@@ -13007,16 +13534,13 @@ var AlertsStore = class {
       comparison: patch.comparison ?? existing.comparison,
       channels: patch.channels ?? existing.channels,
       enabled: patch.enabled ?? existing.enabled,
-      analysisId: "analysisId" in patch ? patch.analysisId : existing.analysisId
+      analysisId: "analysisId" in patch ? patch.analysisId : existing.analysisId,
+      pendingSecs: patch.pendingSecs ?? existing.pendingSecs
     };
     this.validateInput(merged);
     const now = (/* @__PURE__ */ new Date()).toISOString();
-    await this.db.prepare(
-      `UPDATE alert_rules SET
-           name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
-           comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?
-         WHERE id = ? AND project_id = ?`
-    ).bind(
+    const pendingSecs = merged.pendingSecs ?? 0;
+    const values = [
       merged.name.trim(),
       merged.signal,
       JSON.stringify(merged.query),
@@ -13026,12 +13550,72 @@ var AlertsStore = class {
       JSON.stringify(merged.channels),
       merged.enabled === false ? 0 : 1,
       now,
-      merged.analysisId ?? null,
-      id,
-      projectId
-    ).run();
+      merged.analysisId ?? null
+    ];
+    await this.withPendingColumnFallback(
+      pendingSecs,
+      () => this.db.prepare(
+        `UPDATE alert_rules SET
+           name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
+           comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?,
+           pending_secs = ?
+         WHERE id = ? AND project_id = ?`
+      ).bind(...values, pendingSecs, id, projectId).run(),
+      () => this.db.prepare(
+        `UPDATE alert_rules SET
+           name = ?, signal = ?, query_json = ?, threshold = ?, window_mins = ?,
+           comparison = ?, channels_json = ?, enabled = ?, updated_at = ?, analysis_id = ?
+         WHERE id = ? AND project_id = ?`
+      ).bind(...values, id, projectId).run()
+    );
+    const evaluationChanged = merged.signal !== existing.signal || JSON.stringify(merged.query) !== JSON.stringify(existing.query) || merged.threshold !== existing.threshold || merged.windowMins !== existing.windowMins || merged.comparison !== existing.comparison || merged.enabled !== false !== existing.enabled || (merged.analysisId ?? null) !== (existing.analysisId ?? null) || pendingSecs !== (existing.pendingSecs ?? 0);
+    if (evaluationChanged) await this.resetPending(id, projectId);
     return this.getRule(id, projectId);
   }
+  /**
+   * Clears a rule's pending period. Tolerates databases that have not
+   * applied 055/058 (there is nothing to clear on them).
+   */
+  async resetPending(ruleId, projectId) {
+    try {
+      await this.db.prepare(
+        `UPDATE alert_state SET pending_since = NULL, pending_seen_at = NULL
+				 WHERE rule_id = ? AND project_id = ?`
+      ).bind(ruleId, projectId).run();
+    } catch (err) {
+      if (!isMissingColumn(err, "pending_seen_at")) {
+        if (isMissingColumn(err, "pending_since")) return;
+        throw err;
+      }
+      try {
+        await this.db.prepare(
+          `UPDATE alert_state SET pending_since = NULL
+					 WHERE rule_id = ? AND project_id = ?`
+        ).bind(ruleId, projectId).run();
+      } catch (inner) {
+        if (!isMissingColumn(inner, "pending_since")) throw inner;
+      }
+    }
+  }
+  /**
+   * Runs `current`; on a database that has not applied migration 054 (no
+   * `alert_rules.pending_secs`) a rule without a pending duration is
+   * written with the legacy statement so rolling deploys keep working. A
+   * rule that needs the column is refused with a clear message instead of
+   * silently losing its pending duration.
+   */
+  async withPendingColumnFallback(pendingSecs, current, legacy) {
+    try {
+      return await current();
+    } catch (err) {
+      if (!isMissingColumn(err, "pending_secs")) throw err;
+      if (pendingSecs > 0)
+        throw new Error(
+          "pendingSecs needs database migration 054 (alert_rules.pending_secs); apply migrations and retry"
+        );
+      return legacy();
+    }
+  }
   async deleteRule(id, projectId) {
     const owned = await this.db.prepare(
       `SELECT 1 AS ok FROM alert_rules WHERE id = ? AND project_id = ?`
@@ -13102,11 +13686,32 @@ var AlertsStore = class {
       throw err;
     }
   }
-  async recordEvaluation(ruleId, projectId, value, state, notified) {
+  /**
+   * Record (`{ since, seenAt }`) or clear (`null`) the pending period of a
+   * rule that has a pending duration (migrations 055, 058): when the
+   * condition started holding, and the last evaluation that saw it hold.
+   * A rule without a state row gets one in state `ok` since its creation,
+   * already delivered (nothing is owed for a rule that never changed
+   * state).
+   */
+  async setPending(ruleId, projectId, pending, ruleCreatedAt) {
     await this.db.prepare(
-      `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified)
-         VALUES (?, ?, ?, ?, ?, ?, ?)`
+      `INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change, notified, pending_since, pending_seen_at)
+         VALUES (?, ?, 'ok', ?, 1, ?, ?)
+         ON CONFLICT(rule_id) DO UPDATE SET
+           pending_since = excluded.pending_since,
+           pending_seen_at = excluded.pending_seen_at`
     ).bind(
+      ruleId,
+      projectId,
+      ruleCreatedAt,
+      pending?.since ?? null,
+      pending?.seenAt ?? null
+    ).run();
+  }
+  async recordEvaluation(ruleId, projectId, value, state, notified, outcome = "value", errorMessage) {
+    const error = outcome === "error" && errorMessage ? sanitizeEvaluationError(errorMessage) : null;
+    const values = [
       randomHex(16),
       ruleId,
       projectId,
@@ -13114,7 +13719,32 @@ var AlertsStore = class {
       value,
       state,
       notified ? 1 : 0
-    ).run();
+    ];
+    try {
+      await this.db.prepare(
+        `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified, outcome, error_message)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+      ).bind(...values, outcome, error).run();
+      return;
+    } catch (err) {
+      if (!isMissingColumn(err, "error_message")) {
+        if (!isMissingColumn(err, "outcome")) throw err;
+      } else {
+        try {
+          await this.db.prepare(
+            `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified, outcome)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+          ).bind(...values, outcome).run();
+          return;
+        } catch (inner) {
+          if (!isMissingColumn(inner, "outcome")) throw inner;
+        }
+      }
+      await this.db.prepare(
+        `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified)
+         VALUES (?, ?, ?, ?, ?, ?, ?)`
+      ).bind(...values).run();
+    }
   }
   async listEvaluations(params) {
     const cutoff = new Date(
@@ -13136,7 +13766,21 @@ var AlertsStore = class {
     return response;
   }
   // ── Evaluators (count-over-window per signal) ──
-  async evaluateRule(rule) {
+  async evaluateRule(rule, now = /* @__PURE__ */ new Date()) {
+    return (await this.evaluateRuleWithStatus(rule, now)).value;
+  }
+  /** Like `evaluateRule`, plus whether a metrics rule found no fresh data. */
+  async evaluateRuleWithStatus(rule, now = /* @__PURE__ */ new Date()) {
+    if (rule.signal === "metrics" && !rule.analysisId)
+      return evaluateMetricQuery(this.db, rule.query, {
+        projectId: rule.projectId,
+        windowMins: rule.windowMins,
+        ruleCreatedAt: rule.createdAt,
+        now
+      });
+    return { value: await this.evaluateCountRule(rule), noData: false };
+  }
+  async evaluateCountRule(rule) {
     if (rule.analysisId) {
       const row = await this.db.prepare(
         `SELECT primary_value FROM analysis_results
@@ -13154,6 +13798,12 @@ var AlertsStore = class {
         return this.evaluateUsageRule(rule);
       case "ai":
         return this.evaluateAIRule(rule);
+      case "metrics":
+        throw new Error(
+          "metrics rules evaluate through evaluateRuleWithStatus"
+        );
+      default:
+        throw new Error(`unknown alert signal: ${String(rule.signal)}`);
     }
   }
   /**
@@ -13239,14 +13889,46 @@ var AlertsStore = class {
     return row?.c ?? 0;
   }
 };
-var isMissingNotifiedColumn = (err) => {
+var isMissingNotifiedColumn = (err) => isMissingColumn(err, "notified");
+var isMissingColumn = (err, column) => {
   const message = err instanceof Error ? err.message : String(err);
-  return /notified/i.test(message) && /no such column|has no column|does not exist/i.test(message);
+  return message.toLowerCase().includes(column) && /no such column|has no column|does not exist/i.test(message);
 };
 
 // src/plugins/alerts-evaluator.ts
 var DEFAULT_RULE_CONCURRENCY = 5;
 var DEFAULT_RULE_TIMEOUT_MS = 1e4;
+var DEFAULT_MAX_EVALUATION_GAP_MS = 2 * 5 * 6e4;
+async function fireEmail(channel, payload, logger, tracer, delivery = {}) {
+  const transport = delivery.email;
+  if (!transport) {
+    logger.error(
+      "[alerts-evaluator] email channel has no SMTP transport on this collector",
+      { rule_id: payload.rule.id, recipients: channel.to.length }
+    );
+    return false;
+  }
+  const exec = async (span) => {
+    try {
+      await transport.send({ to: channel.to, ...formatAlertEmail(payload) });
+      if (span) span.setAttribute("alert.email.sent", true);
+      return true;
+    } catch (err) {
+      if (span) span.setAttribute("alert.email.sent", false);
+      logger.error("[alerts-evaluator] email delivery failed", {
+        rule_id: payload.rule.id,
+        recipients: channel.to.length,
+        error: err instanceof Error ? err.message : String(err)
+      });
+      return false;
+    }
+  };
+  if (!tracer) return exec(null);
+  return tracer("email.alert", async (span) => exec(span), {
+    "alert.state": payload.state,
+    "alert.email.recipients": channel.to.length
+  });
+}
 async function fireWebhook(channel, payload, logger, tracer, delivery = {}) {
   const target = redactWebhookUrl(channel.url);
   try {
@@ -13318,7 +14000,23 @@ async function fireWebhook(channel, payload, logger, tracer, delivery = {}) {
 async function fireChannels(channels, payload, logger, tracer, delivery = {}) {
   if (channels.length === 0) return true;
   const results = await Promise.all(
-    channels.filter((ch) => ch.type === "webhook").map((ch) => fireWebhook(ch, payload, logger, tracer, delivery))
+    channels.map((ch) => {
+      if (ch.type === "webhook")
+        return fireWebhook(
+          ch,
+          payload,
+          logger,
+          tracer,
+          delivery
+        );
+      if (ch.type === "email")
+        return fireEmail(ch, payload, logger, tracer, delivery);
+      logger.error("[alerts-evaluator] unknown channel type", {
+        rule_id: payload.rule.id,
+        type: String(ch.type)
+      });
+      return Promise.resolve(false);
+    })
   );
   return results.every(Boolean);
 }
@@ -13349,19 +14047,47 @@ async function mapWithConcurrency(items, concurrency, mapper) {
   );
   return results;
 }
-async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery, onAlertFiring) {
+async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, delivery, hold, onAlertFiring) {
   const none = { fired: 0, resolved: 0, redelivered: 0 };
+  let previousForError = null;
   try {
-    const value = await withTimeout(
-      store.evaluateRule(rule),
+    const priorState = await store.getState(rule.id, rule.projectId);
+    const previous = priorState?.current_state ?? "ok";
+    previousForError = previous;
+    const raw = await withTimeout(
+      store.evaluateRuleWithStatus ? store.evaluateRuleWithStatus(rule) : store.evaluateRule(rule),
       ruleTimeoutMs,
       `rule evaluation timed out after ${ruleTimeoutMs}ms`
     );
-    const shouldFire = compareValue(value, rule.threshold, rule.comparison);
-    const priorState = await store.getState(rule.id, rule.projectId);
-    const previous = priorState?.current_state ?? "ok";
-    const next = shouldFire ? "firing" : "ok";
+    const { value, noData } = typeof raw === "number" ? { value: raw, noData: false } : raw;
     const now = (/* @__PURE__ */ new Date()).toISOString();
+    const nowMs = Date.parse(now);
+    const keepOnNoData = noData && rule.signal === "metrics" && rule.query.noData === "keep";
+    const matches = !keepOnNoData && compareValue(value, rule.threshold, rule.comparison);
+    const pendingSecs = rule.pendingSecs ?? 0;
+    const pendingSince = priorState?.pending_since ?? null;
+    const seenAt = priorState?.pending_seen_at ?? pendingSince;
+    const holdBroken = pendingSince !== null && (seenAt === null || nowMs - Date.parse(seenAt) > hold.maxEvaluationGapMs || hold.holdStartedAt !== void 0 && seenAt < hold.holdStartedAt);
+    let pendingActive = pendingSince !== null;
+    let shouldFire = matches;
+    if (matches && previous === "ok" && pendingSecs > 0) {
+      const since = holdBroken || pendingSince === null ? now : pendingSince;
+      await store.setPending(
+        rule.id,
+        rule.projectId,
+        { since, seenAt: now },
+        rule.createdAt
+      );
+      pendingActive = true;
+      shouldFire = nowMs - Date.parse(since) >= pendingSecs * 1e3;
+    }
+    const next = keepOnNoData ? previous : shouldFire ? "firing" : "ok";
+    const clearPending = async () => {
+      if (pendingActive && (!matches || next === "firing")) {
+        await store.setPending(rule.id, rule.projectId, null, rule.createdAt);
+      }
+    };
+    const outcome = keepOnNoData ? "no_data" : "value";
     const analysisAttachment = rule.analysisId ? await store.getAnalysisNarrative(rule.projectId, rule.analysisId).then(
       (n) => n ? {
         id: rule.analysisId,
@@ -13387,6 +14113,7 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
     );
     if (previous !== next) {
       await store.transitionState(rule.id, rule.projectId, next, now);
+      await clearPending();
       if (next === "firing" && onAlertFiring) {
         await onAlertFiring(rule).catch(() => void 0);
       }
@@ -13394,7 +14121,14 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
       if (ok) {
         await store.markNotified(rule.id, rule.projectId, next, now);
       }
-      await store.recordEvaluation(rule.id, rule.projectId, value, next, ok);
+      await store.recordEvaluation(
+        rule.id,
+        rule.projectId,
+        value,
+        next,
+        ok,
+        outcome
+      );
       if (!ok) {
         logger.warn(
           "[alerts-evaluator] state recorded but delivery failed; will retry",
@@ -13403,6 +14137,7 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
       }
       return next === "firing" ? { fired: 1, resolved: 0, redelivered: 0 } : { fired: 0, resolved: 1, redelivered: 0 };
     }
+    await clearPending();
     const owed = priorState !== null && priorState.notified === 0;
     if (owed) {
       const ok = await deliver(previous);
@@ -13414,7 +14149,14 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
           priorState.last_state_change
         );
       }
-      await store.recordEvaluation(rule.id, rule.projectId, value, next, ok);
+      await store.recordEvaluation(
+        rule.id,
+        rule.projectId,
+        value,
+        next,
+        ok,
+        outcome
+      );
       if (!ok) {
         logger.warn("[alerts-evaluator] delivery retry failed; will retry", {
           rule_id: rule.id,
@@ -13425,13 +14167,32 @@ async function evaluateOneRule(store, rule, logger, tracer, ruleTimeoutMs, deliv
       }
       return { fired: 0, resolved: 0, redelivered: 1 };
     }
-    await store.recordEvaluation(rule.id, rule.projectId, value, next, false);
+    await store.recordEvaluation(
+      rule.id,
+      rule.projectId,
+      value,
+      next,
+      false,
+      outcome
+    );
   } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
     logger.error("[alerts-evaluator] rule evaluation failed", {
       rule_id: rule.id,
       project_id: rule.projectId,
-      error: err instanceof Error ? err.message : String(err)
+      error: message
     });
+    if (previousForError !== null) {
+      await store.recordEvaluation(
+        rule.id,
+        rule.projectId,
+        0,
+        previousForError,
+        false,
+        "error",
+        message
+      ).catch(() => void 0);
+    }
   }
   return none;
 }
@@ -13439,7 +14200,8 @@ async function evaluateRuleBatch(store, rules, options) {
   const logger = options?.logger ?? consoleLogger;
   const delivery = {
     allowPrivateWebhookTargets: options?.allowPrivateWebhookTargets,
-    hostLookup: options?.hostLookup
+    hostLookup: options?.hostLookup,
+    email: options?.email
   };
   const concurrency = Math.max(
     1,
@@ -13459,6 +14221,10 @@ async function evaluateRuleBatch(store, rules, options) {
       options?.tracer,
       ruleTimeoutMs,
       delivery,
+      {
+        maxEvaluationGapMs: options?.maxEvaluationGapMs ?? DEFAULT_MAX_EVALUATION_GAP_MS,
+        holdStartedAt: options?.holdStartedAt
+      },
       options?.onAlertFiring
     )
   );
@@ -13474,7 +14240,10 @@ async function evaluateAllRules(env, options) {
   const tracer = options?.tracer;
   const allowPrivateWebhookTargets = env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true";
   const db = sqlDbFor(env);
-  const store = new AlertsStore(db, { allowPrivateWebhookTargets });
+  const store = new AlertsStore(db, {
+    allowPrivateWebhookTargets,
+    emailDeliveryAvailable: Boolean(env.ALERT_EMAIL)
+  });
   const rules = await store.listEnabledRules();
   return evaluateRuleBatch(store, rules, {
     logger,
@@ -13483,6 +14252,9 @@ async function evaluateAllRules(env, options) {
     ruleTimeoutMs: options?.ruleTimeoutMs,
     allowPrivateWebhookTargets,
     hostLookup: options?.hostLookup,
+    email: env.ALERT_EMAIL,
+    maxEvaluationGapMs: options?.maxEvaluationGapMs,
+    holdStartedAt: options?.holdStartedAt,
     onAlertFiring: async (rule) => {
       const target = await resolveServiceIncidentTarget(
         db,
@@ -13511,7 +14283,8 @@ function createAlertEvaluatorHandler() {
 // src/plugins/alerts-routes.ts
 import { EVIDENCE_REFERENCE_CONTRACT as EVIDENCE_REFERENCE_CONTRACT5 } from "@obsunified/types";
 var storeOptions = (env) => ({
-  allowPrivateWebhookTargets: env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true"
+  allowPrivateWebhookTargets: env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
+  emailDeliveryAvailable: Boolean(env.ALERT_EMAIL)
 });
 var alertsRoutesPlugin = {
   name: "alerts-routes",
@@ -13588,9 +14361,19 @@ var alertsRoutesPlugin = {
       const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
       const rule = await store.getRule(id, projectId);
       if (!rule) return c.json({ error: "Rule not found" }, 404);
-      const value = await store.evaluateRule(rule);
+      let value;
+      let noData;
+      try {
+        ({ value, noData } = await store.evaluateRuleWithStatus(rule));
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return c.json({ error: message }, 422);
+      }
       const response = {
         value,
+        noData,
+        // The bare comparison: a rule with `pendingSecs` additionally
+        // needs the condition to hold that long before it fires.
         wouldFire: compareValue(value, rule.threshold, rule.comparison),
         comparison: rule.comparison,
         threshold: rule.threshold,
@@ -16938,12 +17721,12 @@ async function mapWithConcurrency2(items, concurrency, mapper) {
   return results;
 }
 async function fetchReplayChunks(bucket, objects, concurrency = REPLAY_CHUNK_FETCH_CONCURRENCY) {
-  const chunks = await mapWithConcurrency2(objects, concurrency, async (obj) => {
+  const chunks2 = await mapWithConcurrency2(objects, concurrency, async (obj) => {
     const objectData = await bucket.get(obj.key);
     if (!objectData) return [];
     return objectData.json();
   });
-  return chunks.flat();
+  return chunks2.flat();
 }
 var replayQueryRoutesPlugin = {
   name: "replay-query-routes",
@@ -19870,7 +20653,7 @@ var gunzip = async (input, maxBytes) => {
   );
   if (!stream) throw new OtlpDecodeError("Empty gzip body", 400);
   const reader = stream.getReader();
-  const chunks = [];
+  const chunks2 = [];
   let total = 0;
   try {
     for (; ; ) {
@@ -19885,7 +20668,7 @@ var gunzip = async (input, maxBytes) => {
           413
         );
       }
-      chunks.push(value);
+      chunks2.push(value);
     }
   } catch (err) {
     if (err instanceof OtlpDecodeError) throw err;
@@ -19893,7 +20676,7 @@ var gunzip = async (input, maxBytes) => {
   }
   const out = new Uint8Array(total);
   let offset = 0;
-  for (const chunk of chunks) {
+  for (const chunk of chunks2) {
     out.set(chunk, offset);
     offset += chunk.byteLength;
   }
@@ -23413,16 +24196,16 @@ var BlobStoreToR2Adapter = class {
 };
 var streamToBytes = async (stream) => {
   const reader = stream.getReader();
-  const chunks = [];
+  const chunks2 = [];
   while (true) {
     const { value, done } = await reader.read();
     if (done) break;
-    if (value) chunks.push(value);
+    if (value) chunks2.push(value);
   }
-  const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+  const total = chunks2.reduce((sum, chunk) => sum + chunk.byteLength, 0);
   const out = new Uint8Array(total);
   let offset = 0;
-  for (const chunk of chunks) {
+  for (const chunk of chunks2) {
     out.set(chunk, offset);
     offset += chunk.byteLength;
   }
@@ -23532,16 +24315,16 @@ var toUint8Array = (value) => {
 };
 var streamToBytes2 = async (stream) => {
   const reader = stream.getReader();
-  const chunks = [];
+  const chunks2 = [];
   while (true) {
     const { value, done } = await reader.read();
     if (done) break;
-    if (value) chunks.push(value);
+    if (value) chunks2.push(value);
   }
-  const total = chunks.reduce((n, c) => n + c.byteLength, 0);
+  const total = chunks2.reduce((n, c) => n + c.byteLength, 0);
   const out = new Uint8Array(total);
   let offset = 0;
-  for (const c of chunks) {
+  for (const c of chunks2) {
     out.set(c, offset);
     offset += c.byteLength;
   }
@@ -24034,11 +24817,14 @@ export {
   dispatchIncidentWebhook,
   evalCasesRoutesPlugin,
   evaluateAllRules,
+  evaluateMetricQuery,
   evaluateRemediationWatchdogs,
+  evaluateRuleBatch,
   evidenceRetrievalRoutesPlugin,
   extractSpanLineageAndTopology,
   failRemediationProposal,
   findCrossTraceContention,
+  formatAlertEmail,
   genAiNormalizerPlugin,
   getChangeEvent,
   getIncidentInvestigation,
@@ -24048,6 +24834,7 @@ export {
   identityReceiverPlugin,
   isEvalCaseSourceType,
   isEvalRunStatus,
+  isValidEmailAddress,
   issueEnrichmentPlugin,
   issueInsightsPlugin,
   listChangeEvents,
@@ -24069,6 +24856,7 @@ export {
   recordChangeEvent,
   recordInvestigationFinding,
   redactionProcessorPlugin,
+  reduceSeries,
   rejectRemediationProposal,
   replayQueryRoutesPlugin,
   replayReceiverPlugin,
@@ -24083,6 +24871,7 @@ export {
   usageReceiverPlugin,
   usersQueryRoutesPlugin,
   utmEnrichmentPlugin,
+  validateMetricQuery,
   verifyPairedCounterfactualProof
 };
 //# sourceMappingURL=index.js.map