@oneuptime/common 10.0.94 → 10.0.97

package/Server/Services/MetricService.ts

@@ -57,6 +57,20 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
     columns: Array<string>;
   } {
     if (!isPercentileAggregation(aggregateBy.aggregationType)) {
+      /*
+       * Try the per-host MV first — host detail pages are the
+       * dominant attribute-filtered path and the per-host MV is
+       * the only one that can serve them. If it doesn't apply
+       * (no host filter, or extra attrs/groupBy), fall through
+       * to the project/serviceId MV, then to the base table.
+       */
+      const hostMvStatement: {
+        statement: Statement;
+        columns: Array<string>;
+      } | null = this.tryBuildHostAggregateMVStatement(aggregateBy);
+      if (hostMvStatement) {
+        return hostMvStatement;
+      }
       const mvStatement: {
         statement: Statement;
         columns: Array<string>;
@@ -401,6 +415,192 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
     };
   }
 
+  /*
+   * Per-host materialized-view fast path.
+   *
+   * Returns a statement that reads from MetricItemAggMV1mByHost
+   * (created by AddMetricMinuteAggregateByHostMaterializedView)
+   * when:
+   *
+   *   - The aggregation is Sum/Avg/Min/Max/Count over `value`.
+   *   - The only attribute filter is `resource.host.name` as a
+   *     bare-string equality (the dashboard's host detail page
+   *     pattern).
+   *   - The query carries no group-by other than the time
+   *     bucket — the MV is keyed by hostIdentifier and does not
+   *     preserve other attribute breakdowns.
+   *
+   * Returns `null` if any condition fails so the caller falls
+   * through to the next fast path / base table. The result row
+   * shape (columns: aggregateColumn, timestampColumn) matches
+   * the base statement so downstream code needs no changes.
+   */
+  private tryBuildHostAggregateMVStatement(
+    aggregateBy: AggregateBy<Metric>,
+  ): { statement: Statement; columns: Array<string> } | null {
+    const aggType: AggregationType = aggregateBy.aggregationType;
+    const supported: ReadonlyArray<AggregationType> = [
+      AggregationType.Sum,
+      AggregationType.Avg,
+      AggregationType.Min,
+      AggregationType.Max,
+      AggregationType.Count,
+    ];
+    if (!supported.includes(aggType)) {
+      return null;
+    }
+
+    if (
+      aggregateBy.aggregateColumnName.toString() !== "value" ||
+      aggregateBy.aggregationTimestampColumnName.toString() !== "time"
+    ) {
+      return null;
+    }
+
+    if (aggregateBy.groupBy && Object.keys(aggregateBy.groupBy).length > 0) {
+      return null;
+    }
+
+    /*
+     * Inspect the attribute filter. This MV is only safe when
+     * the user is filtering by exactly one attribute,
+     * `resource.host.name`, with a bare-string value (the
+     * canonical Overview/Metrics-page pattern). Anything else —
+     * extra attribute filters, NotEqual, Search, etc. — has to
+     * fall back so the result stays correct.
+     */
+    const queryRecord: Record<string, unknown> =
+      (aggregateBy.query as unknown as Record<string, unknown>) || {};
+    const attrs: unknown = queryRecord["attributes"];
+    if (!attrs || typeof attrs !== "object") {
+      return null;
+    }
+    const attrEntries: Array<[string, unknown]> = Object.entries(
+      attrs as Record<string, unknown>,
+    );
+    if (attrEntries.length !== 1) {
+      return null;
+    }
+    const [attrKey, attrValue] = attrEntries[0]!;
+    if (attrKey !== "resource.host.name") {
+      return null;
+    }
+    if (attrValue === undefined || attrValue === null) {
+      return null;
+    }
+    const hostIdentifier: string =
+      typeof attrValue === "string" ? attrValue : "";
+    if (!hostIdentifier) {
+      return null;
+    }
+
+    const interval: AggregationInterval = AggregateUtil.getAggregationInterval({
+      startDate: aggregateBy.startTimestamp!,
+      endDate: aggregateBy.endTimestamp!,
+    });
+    void interval;
+
+    if (!this.database) {
+      this.useDefaultDatabase();
+    }
+    const databaseName: string = this.database.getDatasourceOptions().database!;
+
+    const intervalLower: string = interval.toLowerCase();
+
+    let mergedExpr: string;
+    if (aggType === AggregationType.Sum) {
+      mergedExpr = `sumMerge(valueSumState)`;
+    } else if (aggType === AggregationType.Count) {
+      mergedExpr = `countMerge(valueCountState)`;
+    } else if (aggType === AggregationType.Min) {
+      mergedExpr = `minMerge(valueMinState)`;
+    } else if (aggType === AggregationType.Max) {
+      mergedExpr = `maxMerge(valueMaxState)`;
+    } else {
+      mergedExpr = `if(countMerge(valueCountState) = 0, 0, sumMerge(valueSumState) / countMerge(valueCountState))`;
+    }
+
+    /*
+     * Strip both `time` (column doesn't exist on the MV; we
+     * inject an explicit bucketTime range below) and
+     * `attributes` (the attribute filter is now an explicit
+     * `hostIdentifier =` predicate against an MV column).
+     */
+    const filteredQuery: typeof aggregateBy.query =
+      this.stripAttributesAndTimeFromQuery(
+        aggregateBy.query,
+      ) as typeof aggregateBy.query;
+    const nonTimeWhere: Statement =
+      this.statementGenerator.toWhereStatement(filteredQuery);
+    const sortStatement: Statement = this.statementGenerator.toSortStatement(
+      aggregateBy.sort!,
+    );
+
+    const statement: Statement = SQL``;
+
+    statement.append(
+      `SELECT ${mergedExpr} as value, date_trunc('${intervalLower}', toStartOfInterval(bucketTime, INTERVAL 1 ${intervalLower})) as time`,
+    );
+    statement.append(SQL` FROM ${databaseName}.MetricItemAggMV1mByHost`);
+    statement.append(
+      ` WHERE bucketTime >= toDateTime('${this.formatDateTime(aggregateBy.startTimestamp!)}') AND bucketTime <= toDateTime('${this.formatDateTime(aggregateBy.endTimestamp!)}')`,
+    );
+    statement.append(
+      SQL` AND hostIdentifier = ${{
+        value: hostIdentifier,
+        type: TableColumnType.Text,
+      }}`,
+    );
+    statement.append(SQL` `).append(nonTimeWhere);
+
+    statement.append(SQL` GROUP BY `).append(`time`);
+    statement.append(SQL` ORDER BY `).append(sortStatement);
+    statement.append(
+      SQL` LIMIT ${{
+        value: Number(aggregateBy.limit),
+        type: TableColumnType.Number,
+      }}`,
+    );
+    statement.append(
+      SQL` OFFSET ${{
+        value: Number(aggregateBy.skip),
+        type: TableColumnType.Number,
+      }} `,
+    );
+    statement.append(
+      ` SETTINGS optimize_aggregation_in_order=1, optimize_move_to_prewhere=1, max_threads=4`,
+    );
+
+    logger.debug(`${this.model.tableName} Host MV Aggregate Statement`, {
+      tableName: this.model.tableName,
+    } as LogAttributes);
+    logger.debug(statement, {
+      tableName: this.model.tableName,
+    } as LogAttributes);
+
+    return {
+      statement,
+      columns: [
+        aggregateBy.aggregateColumnName.toString(),
+        aggregateBy.aggregationTimestampColumnName.toString(),
+      ],
+    };
+  }
+
+  private stripAttributesAndTimeFromQuery(query: unknown): typeof query {
+    if (!query || typeof query !== "object") {
+      return query;
+    }
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(query as Record<string, unknown>)) {
+      if (k === "time" || k === "attributes") {
+        continue;
+      }
+      out[k] = v;
+    }
+    return out as typeof query;
+  }
+
   private stripTimeFromQuery(query: unknown): typeof query {
     if (!query || typeof query !== "object") {
       return query;

package/Server/Services/MonitorTemplateService.ts

@@ -16,18 +16,20 @@ export interface SyncLinkedMonitorsResult {
 
 /**
  * Subset of Monitor fields that a template push can overwrite. Anything
- * outside this set (name, description, labels, monitorType, etc.) is
- * intentionally never touched by sync — those are per-monitor concerns.
+ * outside this set (name, description, monitorType, etc.) is intentionally
+ * never touched by sync — those are per-monitor concerns.
  */
 export type SyncableTemplateField =
   | "monitorSteps"
   | "monitoringInterval"
-  | "minimumProbeAgreement";
+  | "minimumProbeAgreement"
+  | "labels";
 
 const ALL_SYNCABLE_FIELDS: ReadonlyArray<SyncableTemplateField> = [
   "monitorSteps",
   "monitoringInterval",
   "minimumProbeAgreement",
+  "labels",
 ];
 
 export class Service extends DatabaseService<Model> {
@@ -125,6 +127,9 @@ export class Service extends DatabaseService<Model> {
         monitorSteps: true,
         monitoringInterval: true,
         minimumProbeAgreement: true,
+        labels: {
+          _id: true,
+        },
       },
       props: data.props,
     });
@@ -203,6 +208,9 @@ export class Service extends DatabaseService<Model> {
         monitorSteps: true,
         monitoringInterval: true,
         minimumProbeAgreement: true,
+        labels: {
+          _id: true,
+        },
       },
       props: data.props,
     });

package/Server/Services/TraceAggregationService.ts

@@ -503,14 +503,19 @@ export class TraceAggregationService {
       for (const [attrKey, attrValue] of Object.entries(request.attributes)) {
         TraceAggregationService.validateFacetKey(attrKey);
 
+        /*
+         * Match attribute keys case-insensitively — see the matching note in
+         * LogAggregationService.appendCommonFilters. Casings vary across
+         * OTEL conventions and app-emitted attributes.
+         */
         statement.append(
-          SQL` AND attributes[${{
+          SQL` AND arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
             type: TableColumnType.Text,
             value: attrKey,
-          }}] = ${{
+          }}) AND v = ${{
             type: TableColumnType.Text,
             value: attrValue,
-          }}`,
+          }}, mapKeys(attributes), mapValues(attributes))`,
         );
       }
     }

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -463,10 +463,35 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           /*
-           * ClickHouse Map columns return the value type's default for
-           * missing keys (empty string for String values), so to express
-           * "is empty" we have to cover both the missing-key and the
-           * empty-string case explicitly.
+           * Map filters split into two paths:
+           *
+           * 1. Programmatic equality / null / numeric comparisons —
+           *    EqualTo, NotEqual, IsNull, NotNull, GreaterThan, etc.,
+           *    or bare string/number values. Callers are dashboard
+           *    pages and services that pass canonical keys already
+           *    matching the stored casing, so we use ClickHouse's
+           *    direct Map subscript `attributes['k']`. That's an O(1)
+           *    hash lookup per row and lets the query planner push the
+           *    predicate into PREWHERE, instead of paying the
+           *    `arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(...))`
+           *    cost which materializes mapKeys/mapValues per row and
+           *    lowercases every stored key on every query. Restoring
+           *    this fast path is the single biggest performance fix
+           *    for Host / Logs / Traces detail pages.
+           *
+           * 2. User-typed substring/wildcard operators — Search,
+           *    StartsWith, EndsWith, NotContains. These come from the
+           *    search bar where users shouldn't have to remember
+           *    whether the attribute key is `requestId` or `requestid`,
+           *    so we keep the case-insensitive `arrayExists` form. The
+           *    cost is acceptable because a search-bar query is
+           *    bounded (one user, one click) and these operators
+           *    already imply a row scan.
+           *
+           * ClickHouse Map subscripts return the value type's default
+           * for missing keys (empty string for String values), which
+           * is what the IsNull / NotNull / NotEqual branches below
+           * mirror to preserve the previous semantics.
            */
           if (mapEntry instanceof IsNull) {
             whereStatement.append(
@@ -496,13 +521,13 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
           if (mapEntry instanceof Search) {
             whereStatement.append(
-              SQL`AND ${key}[${{
+              SQL`AND arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
                 value: mapKey,
                 type: TableColumnType.Text,
-              }}] ILIKE ${{
+              }}) AND v ILIKE ${{
                 value: mapEntry as Search<string>,
                 type: TableColumnType.Text,
-              }}`,
+              }}, mapKeys(${key}), mapValues(${key}))`,
             );
             continue;
           }
@@ -510,13 +535,13 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           if (mapEntry instanceof NotContains) {
             const literalValue: string = `%${(mapEntry.value as string) || ""}%`;
             whereStatement.append(
-              SQL`AND ${key}[${{
+              SQL`AND NOT arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
                 value: mapKey,
                 type: TableColumnType.Text,
-              }}] NOT ILIKE ${{
+              }}) AND v ILIKE ${{
                 value: literalValue,
                 type: TableColumnType.Text,
-              }}`,
+              }}, mapKeys(${key}), mapValues(${key}))`,
             );
             continue;
           }
@@ -524,13 +549,13 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           if (mapEntry instanceof StartsWith) {
             const literalValue: string = `${(mapEntry.value as string) || ""}%`;
             whereStatement.append(
-              SQL`AND ${key}[${{
+              SQL`AND arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
                 value: mapKey,
                 type: TableColumnType.Text,
-              }}] ILIKE ${{
+              }}) AND v ILIKE ${{
                 value: literalValue,
                 type: TableColumnType.Text,
-              }}`,
+              }}, mapKeys(${key}), mapValues(${key}))`,
             );
             continue;
           }
@@ -538,13 +563,13 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           if (mapEntry instanceof EndsWith) {
             const literalValue: string = `%${(mapEntry.value as string) || ""}`;
             whereStatement.append(
-              SQL`AND ${key}[${{
+              SQL`AND arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
                 value: mapKey,
                 type: TableColumnType.Text,
-              }}] ILIKE ${{
+              }}) AND v ILIKE ${{
                 value: literalValue,
                 type: TableColumnType.Text,
-              }}`,
+              }}, mapKeys(${key}), mapValues(${key}))`,
             );
             continue;
           }
@@ -577,7 +602,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
           /*
            * Map values are stored as text; cast to Float64 for numeric
-           * comparisons and skip rows where the cast fails (non-numeric).
+           * comparisons. toFloat64OrNull yields NULL for non-numeric
+           * values (including the empty-string default for missing
+           * keys), which compares to false against any numeric
+           * threshold and naturally drops those rows.
            */
           if (mapEntry instanceof GreaterThan) {
             whereStatement.append(
@@ -631,7 +659,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
             continue;
           }
 
-          // Bare string/number/boolean — back-compat with existing data.
+          // Bare string/number/boolean — direct Map subscript.
           whereStatement.append(
             SQL`AND ${key}[${{
               value: mapKey,

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -1,6 +1,7 @@
 import Alert from "../../../Models/DatabaseModels/Alert";
 import AlertSeverity from "../../../Models/DatabaseModels/AlertSeverity";
 import AlertStateTimeline from "../../../Models/DatabaseModels/AlertStateTimeline";
+import Host from "../../../Models/DatabaseModels/Host";
 import Label from "../../../Models/DatabaseModels/Label";
 import Monitor from "../../../Models/DatabaseModels/Monitor";
 import OnCallDutyPolicy from "../../../Models/DatabaseModels/OnCallDutyPolicy";
@@ -17,6 +18,7 @@ import { DisableAutomaticAlertCreation } from "../../EnvironmentConfig";
 import AlertService from "../../Services/AlertService";
 import AlertSeverityService from "../../Services/AlertSeverityService";
 import AlertStateTimelineService from "../../Services/AlertStateTimelineService";
+import HostService from "../../Services/HostService";
 import logger, { LogAttributes } from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 import DataToProcess from "./DataToProcess";
@@ -268,6 +270,35 @@ export default class MonitorAlert {
         }
         if (seriesLabels && Object.keys(seriesLabels).length > 0) {
           alert.seriesLabels = seriesLabels;
+
+          /*
+           * Link the alert to the Host that emitted this series, if
+           * the metric carried a `host.name` resource attribute. The
+           * Host record was auto-discovered during OTel ingestion.
+           */
+          const hostName: string | undefined =
+            typeof seriesLabels["host.name"] === "string"
+              ? (seriesLabels["host.name"] as string)
+              : undefined;
+
+          if (hostName) {
+            const host: Host | null = await HostService.findOneBy({
+              query: {
+                projectId: input.monitor.projectId!,
+                hostIdentifier: hostName,
+              },
+              select: {
+                _id: true,
+              },
+              props: {
+                isRoot: true,
+              },
+            });
+
+            if (host) {
+              alert.hosts = [host];
+            }
+          }
         }
 
         alert.onCallDutyPolicies =

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -1,3 +1,4 @@
+import Host from "../../../Models/DatabaseModels/Host";
 import Incident from "../../../Models/DatabaseModels/Incident";
 import IncidentSeverity from "../../../Models/DatabaseModels/IncidentSeverity";
 import IncidentStateTimeline from "../../../Models/DatabaseModels/IncidentStateTimeline";
@@ -15,6 +16,7 @@ import ObjectID from "../../../Types/ObjectID";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
 import { DisableAutomaticIncidentCreation } from "../../EnvironmentConfig";
+import HostService from "../../Services/HostService";
 import IncidentService from "../../Services/IncidentService";
 import IncidentSeverityService from "../../Services/IncidentSeverityService";
 import IncidentStateTimelineService from "../../Services/IncidentStateTimelineService";
@@ -313,6 +315,35 @@ export default class MonitorIncident {
         }
         if (seriesLabels && Object.keys(seriesLabels).length > 0) {
           incident.seriesLabels = seriesLabels;
+
+          /*
+           * Link the incident to the Host that emitted this series, if
+           * the metric carried a `host.name` resource attribute. The
+           * Host record was auto-discovered during OTel ingestion.
+           */
+          const hostName: string | undefined =
+            typeof seriesLabels["host.name"] === "string"
+              ? (seriesLabels["host.name"] as string)
+              : undefined;
+
+          if (hostName) {
+            const host: Host | null = await HostService.findOneBy({
+              query: {
+                projectId: input.monitor.projectId!,
+                hostIdentifier: hostName,
+              },
+              select: {
+                _id: true,
+              },
+              props: {
+                isRoot: true,
+              },
+            });
+
+            if (host) {
+              incident.hosts = [host];
+            }
+          }
         }
 
         incident.onCallDutyPolicies =

package/Server/Utils/VM/VMRunner.ts

@@ -17,6 +17,67 @@ import vm, { Context } from "vm";
  */
 const PROXY_TARGET_SYMBOL: unique symbol = Symbol("sandboxProxyTarget");
 
+/**
+ * Hardening prelude injected before user code in `runCodeInNodeVM`.
+ *
+ * Node's `vm` module is not a security boundary. The published PoC for
+ * GHSA-g9cp-35m2-fjv6 forces a stack-overflow `RangeError`, walks
+ * `e.__proto__.__proto__.__proto__` to `Object.prototype`, then reads
+ * `.toString.constructor` to obtain a `Function` constructor that compiles
+ * code in a realm where `process.binding('spawn_sync')` is reachable.
+ *
+ * This prelude closes that path by:
+ *  - severing `Error.prototype`'s link to `Object.prototype` so the 3-level
+ *    walk lands on `null` instead of `Object.prototype`;
+ *  - deleting `.constructor` from every built-in prototype, so even a
+ *    different walk (e.g. `(0).constructor.constructor`) cannot resolve to a
+ *    function constructor;
+ *  - clearing `Function` / `eval` from the sandbox global;
+ *  - freezing the affected prototypes so user code cannot reattach them.
+ *
+ * This is a hotfix for the public PoC. The durable fix is to drop
+ * `runCodeInNodeVM` in favor of running synthetic monitor scripts in an
+ * out-of-process sandbox (tracked on the `probe-runner` branch).
+ */
+const VM_HARDENING_PRELUDE: string = `(() => {
+  const _ctors = [
+    Object, Function, Array, String, Number, Boolean, RegExp,
+    Error, RangeError, TypeError, SyntaxError, ReferenceError, EvalError, URIError,
+    Symbol, Date, Map, Set, WeakMap, WeakSet, Promise, Proxy,
+    ArrayBuffer, DataView,
+    Int8Array, Uint8Array, Uint8ClampedArray,
+    Int16Array, Uint16Array, Int32Array, Uint32Array,
+    Float32Array, Float64Array,
+  ];
+  if (typeof BigInt !== 'undefined') _ctors.push(BigInt);
+
+  for (const C of _ctors) {
+    try { if (C && C.prototype) delete C.prototype.constructor; } catch (_) {}
+  }
+
+  // Generator / async-function prototypes have no named global — reach via syntax.
+  try { delete Object.getPrototypeOf(function*(){}).constructor; } catch (_) {}
+  try { delete Object.getPrototypeOf(async function(){}).constructor; } catch (_) {}
+  try { delete Object.getPrototypeOf(async function*(){}).constructor; } catch (_) {}
+
+  try { Object.setPrototypeOf(Error.prototype, null); } catch (_) {}
+
+  try {
+    Object.defineProperty(globalThis, 'Function', {
+      value: undefined, writable: false, configurable: false,
+    });
+  } catch (_) {}
+  try {
+    Object.defineProperty(globalThis, 'eval', {
+      value: undefined, writable: false, configurable: false,
+    });
+  } catch (_) {}
+
+  for (const C of _ctors) {
+    try { if (C && C.prototype) Object.freeze(C.prototype); } catch (_) {}
+  }
+})();`;
+
 /** Properties blocked on every host-realm object exposed to the sandbox. */
 const BLOCKED_SANDBOX_PROPERTIES: ReadonlySet<string> = new Set([
   "constructor",
@@ -465,6 +526,7 @@ export default class VMRunner {
     });
 
     const script: string = `(async()=>{
+        ${VM_HARDENING_PRELUDE}
         ${code}
       })()`;
 

package/Tests/Server/Services/LogAggregationService.test.ts

@@ -80,4 +80,29 @@ describe("LogAggregationService", () => {
       });
     }).toThrow("Invalid facetKey");
   });
+
+  test("histogram attribute filter matches attribute keys case-insensitively", () => {
+    /*
+     * Users typing `requestid` should still match data stored with the key
+     * `requestId` (camelCase). The histogram filter shares the same WHERE
+     * clause builder (`appendCommonFilters`) with the list/facet queries, so
+     * verifying it on histogram covers all three.
+     */
+    const statement: Statement = (
+      LogAggregationService as any
+    ).buildHistogramStatement({
+      ...defaultRequest,
+      facetKey: undefined,
+      attributes: { requestid: "uuid-123" },
+    });
+
+    expect(statement.query).toContain(
+      "arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(",
+    );
+    expect(statement.query).toContain(
+      ", mapKeys(attributes), mapValues(attributes))",
+    );
+    expect(Object.values(statement.query_params)).toContain("requestid");
+    expect(Object.values(statement.query_params)).toContain("uuid-123");
+  });
 });