@oneuptime/common 12.0.9 → 12.0.11

package/Server/Services/RunnerJobService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/RunnerJob";
 import RunnerJobStatus from "../../Types/Runbook/RunnerJobStatus";
 import RunnerJobOrigin from "../../Types/Runbook/RunnerJobOrigin";
 import RunbookStepType, {
+  isPayloadCarryingStepType,
   isRunnerExecutedStepType,
 } from "../../Types/Runbook/RunbookStepType";
 import { AI_COMMAND_STEP_TYPES } from "../../Types/AutoRemediation/AiRemediationCommandPlan";
@@ -106,6 +107,27 @@ export class Service extends DatabaseService<Model> {
       );
     }
 
+    /*
+     * Each runner-executed type carries its instruction in exactly one place:
+     * a script, or a structured payload. Enforced here rather than as a
+     * required column because it is per-type — the column check would reject
+     * an SSH job's empty script, which is the shape SSH jobs are supposed to
+     * have. A job dispatched with neither would reach a Runner that has
+     * nothing to do and report success, which looks identical to a step that
+     * actually ran.
+     */
+    if (isPayloadCarryingStepType(data.stepType)) {
+      if (!data.payload) {
+        throw new BadDataException(
+          `A ${data.stepType} job needs structured instructions to send to the Runner.`,
+        );
+      }
+    } else if (!data.script) {
+      throw new BadDataException(
+        `A ${data.stepType} job needs a script for the Runner to execute.`,
+      );
+    }
+
     const claimDeadlineAt: Date = OneUptimeDate.addRemoveSeconds(
       OneUptimeDate.getCurrentDate(),
       Math.ceil((data.claimTimeoutInMs ?? DEFAULT_CLAIM_TIMEOUT_MS) / 1000),
@@ -118,7 +140,8 @@ export class Service extends DatabaseService<Model> {
     row.stepId = data.stepId;
     row.stepType = data.stepType;
     row.targetAgentId = data.targetAgentId;
-    row.script = data.script;
+    // The column is NOT NULL, so a payload-carrying job stores "" rather than null.
+    row.script = data.script || "";
     if (data.payload) {
       row.payload = data.payload;
     }

package/Server/Types/Billing/MeteredPlan/AllMeteredPlans.ts

@@ -4,6 +4,19 @@ import ServerMeteredPlan from "./ServerMeteredPlan";
 import TelemetryMeteredPlanType from "./TelemetryMeteredPlan";
 import BadDataException from "../../../../Types/Exception/BadDataException";
 import ProductType from "../../../../Types/MeteredPlan/ProductType";
+import {
+  SESSION_REPLAY_PRICE_IN_USD_PER_GB,
+  TELEMETRY_PRICE_IN_USD_PER_GB,
+  TELEMETRY_PRICE_RETENTION_IN_DAYS,
+} from "../../../../Types/Billing/PayAsYouGoPricing";
+
+/*
+ * Per GB per day, derived from the advertised per-GB-per-15-days rate so the
+ * price the dashboard quotes a Free plan user and the price we actually meter
+ * cannot drift apart.
+ */
+const TELEMETRY_UNIT_COST_IN_USD: number =
+  TELEMETRY_PRICE_IN_USD_PER_GB / TELEMETRY_PRICE_RETENTION_IN_DAYS;
 
 export const ActiveMonitoringMeteredPlan: ActiveMonitoringMeteredPlanType =
   new ActiveMonitoringMeteredPlanType();
@@ -11,25 +24,25 @@ export const ActiveMonitoringMeteredPlan: ActiveMonitoringMeteredPlanType =
 export const LogDataIngestMeteredPlan: TelemetryMeteredPlanType =
   new TelemetryMeteredPlanType({
     productType: ProductType.Logs,
-    unitCostInUSD: 0.1 / 15, // 0.10 per 15 days per GB
+    unitCostInUSD: TELEMETRY_UNIT_COST_IN_USD, // 0.10 per 15 days per GB
   });
 
 export const MetricsDataIngestMeteredPlan: TelemetryMeteredPlanType =
   new TelemetryMeteredPlanType({
     productType: ProductType.Metrics,
-    unitCostInUSD: 0.1 / 15, // 0.10 per 15 days per GB
+    unitCostInUSD: TELEMETRY_UNIT_COST_IN_USD, // 0.10 per 15 days per GB
   });
 
 export const TracesDataIngestMetredPlan: TelemetryMeteredPlanType =
   new TelemetryMeteredPlanType({
     productType: ProductType.Traces,
-    unitCostInUSD: 0.1 / 15, // 0.10 per 15 days per GB
+    unitCostInUSD: TELEMETRY_UNIT_COST_IN_USD, // 0.10 per 15 days per GB
   });
 
 export const ProfilesDataIngestMeteredPlan: TelemetryMeteredPlanType =
   new TelemetryMeteredPlanType({
     productType: ProductType.Profiles,
-    unitCostInUSD: 0.1 / 15, // 0.10 per 15 days per GB
+    unitCostInUSD: TELEMETRY_UNIT_COST_IN_USD, // 0.10 per 15 days per GB
   });
 
 /*
@@ -43,7 +56,8 @@ export const ProfilesDataIngestMeteredPlan: TelemetryMeteredPlanType =
 export const SessionReplayDataIngestMeteredPlan: TelemetryMeteredPlanType =
   new TelemetryMeteredPlanType({
     productType: ProductType.SessionReplay,
-    unitCostInUSD: 2.0 / 15, // 2.00 per 15 days per GB
+    unitCostInUSD:
+      SESSION_REPLAY_PRICE_IN_USD_PER_GB / TELEMETRY_PRICE_RETENTION_IN_DAYS, // 2.00 per 15 days per GB
   });
 
 const AllMeteredPlans: Array<ServerMeteredPlan> = [

package/Server/Types/Database/JSONColumnQuery.ts

@@ -154,6 +154,18 @@ const toScalar: ToScalarFunction = (value: unknown): ScalarValue => {
     return value.toString();
   }
 
+  /*
+   * A Date only reaches here from a query built server-side: one that arrived
+   * over HTTP has already been through JSON, which renders a Date as its ISO
+   * string. Both paths must produce the same text, because the ordering
+   * comparisons below are lexicographic and only ISO-8601 makes that
+   * chronological. The `String(value)` fallback would yield
+   * "Sun Aug 17 2026 00:00:00 GMT+0000", which sorts by weekday name.
+   */
+  if (value instanceof Date) {
+    return value.toISOString();
+  }
+
   if (value === null || value === undefined) {
     return "";
   }
@@ -313,6 +325,32 @@ class KeyPredicateBuilder {
     };
   }
 
+  /**
+   * An ordering comparison against a value whose type we only learn at runtime.
+   *
+   * A jsonb key holds whatever the project put there, so `>` has two readings.
+   * Numbers compare numerically — "10" must be greater than "9", which text
+   * comparison gets backwards. Everything else compares as text, and the case
+   * that matters is a date: custom field dates are stored as ISO-8601 strings,
+   * and ISO-8601 is designed so that lexicographic order IS chronological
+   * order, so `>=` over the raw text is already the right question.
+   *
+   * Routing dates through compareNumeric — which is what these operators used
+   * to do unconditionally — was silently wrong rather than loudly wrong.
+   * `Number("2026-08-17T00:00:00.000Z")` is NaN, so the bound parameter became
+   * `CAST(NaN AS NUMERIC)`, and `numericExpression` independently returns NULL
+   * for text that does not look like a number. The predicate was therefore
+   * `NULL > NaN` on every row: no error, no rows, and a lit filter chip over an
+   * empty table.
+   *
+   * InBetween has always branched this way; these four now agree with it.
+   */
+  private compareOrdered(operator: string, value: unknown): PredicateFragment {
+    return isNumericValue(toScalar(value))
+      ? this.compareNumeric(operator, value)
+      : this.compareText(operator, value);
+  }
+
   /**
    * "No value here" — the key is absent, explicitly JSON null, an empty
    * string, or an empty array. A multi-select cleared in the UI leaves `[]`
@@ -466,19 +504,19 @@ class KeyPredicateBuilder {
     }
 
     if (value instanceof GreaterThanOrEqual) {
-      return this.compareNumeric(">=", value.value);
+      return this.compareOrdered(">=", value.value);
     }
 
     if (value instanceof GreaterThan) {
-      return this.compareNumeric(">", value.value);
+      return this.compareOrdered(">", value.value);
     }
 
     if (value instanceof LessThanOrEqual) {
-      return this.compareNumeric("<=", value.value);
+      return this.compareOrdered("<=", value.value);
     }
 
     if (value instanceof LessThan) {
-      return this.compareNumeric("<", value.value);
+      return this.compareOrdered("<", value.value);
     }
 
     /*

package/Server/Utils/LLM/LLMService.ts

@@ -49,6 +49,12 @@ export interface LLMCompletionRequest {
   requestTimeoutInMs?: number | undefined;
   /** Number of retries after the initial request. */
   requestRetries?: number | undefined;
+  /**
+   * Wall-clock budget for the whole retry ladder. Defaults to
+   * getRetryDeadlineInMs — raise it for a caller that can afford to wait
+   * longer than an interactive chat turn.
+   */
+  requestRetryDeadlineInMs?: number | undefined;
   /** Re-apply caller-owned request fields after provider-level overrides. */
   protectRequestParameters?: boolean | undefined;
   /**
@@ -121,6 +127,44 @@ interface OpenAIRequestAdaptation {
 }
 
 export default class LLMService {
+  /*
+   * How many times a provider call is attempted before it is reported as a
+   * failure. The provider hop is the flakiest one in the product — hosted
+   * endpoints rate limit, self-hosted ones fall behind their own queue, and a
+   * cold model can miss the per-attempt timeout while the next attempt sails
+   * through — and every one of those used to surface as a dead chat turn
+   * after three tries.
+   *
+   * This is a count of ATTEMPTS; API takes a count of retries, which is one
+   * fewer.
+   */
+  public static readonly DEFAULT_REQUEST_ATTEMPTS: number = 10;
+
+  /*
+   * Ten doublings would put the last wait seventeen minutes out. Capped at
+   * eight seconds the whole ladder (2, 4, 8, 8, ...) costs about a minute of
+   * sleep before jitter, which is the point: ride out a provider blip without
+   * turning one chat turn into a coffee break.
+   */
+  public static readonly MAX_BACKOFF_IN_MS: number = 8 * 1000;
+
+  /*
+   * Ten attempts is the right answer for failures that fail FAST — a 429, a
+   * refused connection, a 502 from a load balancer. Ten TIMEOUTS are a
+   * different story: at the 120s default that is twenty minutes, against a
+   * ChatAgentRunner budget of five for an entire turn, so the ladder gets a
+   * wall clock of its own.
+   */
+  private static readonly DEFAULT_RETRY_DEADLINE_IN_MS: number = 5 * 60 * 1000;
+
+  /*
+   * The floor keeps the deadline from ever costing a provider attempts it had
+   * before this budget existed: three full-timeout attempts is what the old
+   * two-retry policy allowed, and a slow provider (Ollama at 300s) still gets
+   * exactly that.
+   */
+  private static readonly MIN_FULL_TIMEOUT_ATTEMPTS_WITHIN_DEADLINE: number = 3;
+
   /*
    * Provider-level parameters allowed on protected, unattended completions.
    * This is deliberately a generation-only allowlist: capability fields such
@@ -196,10 +240,10 @@ export default class LLMService {
 
   /*
    * Remembers what a given endpoint actually accepted, so only the first
-   * completion per provider pays for the discovery. API.post retries 4xx, so
-   * a rejected request costs several full prompt uploads plus backoff —
-   * re-learning that on every call would be expensive on hot paths like the
-   * AI agent loop.
+   * completion per provider pays for the discovery. A rejected request still
+   * costs a full prompt upload and round trip per adaptation — re-learning
+   * that on every call would be expensive on hot paths like the AI agent
+   * loop.
    *
    * Keyed by provider + base URL + model, so editing any of them re-probes.
    * Entries also expire: what a deployment accepts is not fixed forever — the
@@ -650,6 +694,45 @@ export default class LLMService {
     return undefined;
   }
 
+  /**
+   * The retry policy every provider call runs under.
+   *
+   * Ten attempts, backed off and jittered, and deliberately spent only on
+   * failures a repeat could fix: a provider that rejects the request itself —
+   * bad key, unknown model, malformed body — answers the same way every time,
+   * and burning the ladder on it only hides the error the operator has to see.
+   */
+  private static buildRequestPolicy(data: {
+    request: LLMCompletionRequest;
+    defaultTimeoutInMs: number;
+  }): RequestOptions {
+    const timeoutInMs: number =
+      data.request.requestTimeoutInMs ?? data.defaultTimeoutInMs;
+
+    return {
+      retries: data.request.requestRetries ?? this.DEFAULT_REQUEST_ATTEMPTS - 1,
+      exponentialBackoff: true,
+      maxBackoffInMs: this.MAX_BACKOFF_IN_MS,
+      retryOnlyOnRetryableErrors: true,
+      totalTimeoutInMs: this.getRetryDeadlineInMs(data.request, timeoutInMs),
+      timeout: timeoutInMs,
+    };
+  }
+
+  private static getRetryDeadlineInMs(
+    request: LLMCompletionRequest,
+    timeoutInMs: number,
+  ): number {
+    if (request.requestRetryDeadlineInMs !== undefined) {
+      return request.requestRetryDeadlineInMs;
+    }
+
+    return Math.max(
+      this.DEFAULT_RETRY_DEADLINE_IN_MS,
+      timeoutInMs * this.MIN_FULL_TIMEOUT_ATTEMPTS_WITHIN_DEADLINE,
+    );
+  }
+
   /**
    * POST an OpenAI-style chat completion, reshaping and retrying when the
    * endpoint rejects a generation parameter it does not support.
@@ -720,11 +803,13 @@ export default class LLMService {
      * DNS round trips (and a rebind window between them).
      */
     const requestOptions: RequestOptions =
-      await this.buildGuardedRequestOptions(data.requestUrl, {
-        retries: data.request.requestRetries ?? 2,
-        exponentialBackoff: true,
-        timeout: data.request.requestTimeoutInMs ?? 120000,
-      });
+      await this.buildGuardedRequestOptions(
+        data.requestUrl,
+        this.buildRequestPolicy({
+          request: data.request,
+          defaultTimeoutInMs: 120000,
+        }),
+      );
 
     const post: () => Promise<
       HTTPResponse<JSONObject> | HTTPErrorResponse
@@ -1310,11 +1395,13 @@ export default class LLMService {
           "anthropic-version": "2023-06-01",
           "Content-Type": "application/json",
         },
-        options: await this.buildGuardedRequestOptions(anthropicRequestUrl, {
-          retries: request.requestRetries ?? 2,
-          exponentialBackoff: true,
-          timeout: request.requestTimeoutInMs ?? 120000,
-        }),
+        options: await this.buildGuardedRequestOptions(
+          anthropicRequestUrl,
+          this.buildRequestPolicy({
+            request: request,
+            defaultTimeoutInMs: 120000,
+          }),
+        ),
       });
 
     const anthropicLogAttributes: LogAttributes = {
@@ -1467,11 +1554,13 @@ export default class LLMService {
         headers: {
           "Content-Type": "application/json",
         },
-        options: await this.buildGuardedRequestOptions(ollamaRequestUrl, {
-          retries: request.requestRetries ?? 2,
-          exponentialBackoff: true,
-          timeout: request.requestTimeoutInMs ?? 300000, // Ollama may be slower
-        }),
+        options: await this.buildGuardedRequestOptions(
+          ollamaRequestUrl,
+          this.buildRequestPolicy({
+            request: request,
+            defaultTimeoutInMs: 300000, // Ollama may be slower
+          }),
+        ),
       });
 
     const ollamaLogAttributes: LogAttributes = {

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -1,15 +1,9 @@
 import Alert from "../../../Models/DatabaseModels/Alert";
 import AlertSeverity from "../../../Models/DatabaseModels/AlertSeverity";
 import AlertStateTimeline from "../../../Models/DatabaseModels/AlertStateTimeline";
-import CephCluster from "../../../Models/DatabaseModels/CephCluster";
-import DockerSwarmCluster from "../../../Models/DatabaseModels/DockerSwarmCluster";
-import Host from "../../../Models/DatabaseModels/Host";
 import Label from "../../../Models/DatabaseModels/Label";
 import Monitor from "../../../Models/DatabaseModels/Monitor";
 import OnCallDutyPolicy from "../../../Models/DatabaseModels/OnCallDutyPolicy";
-import ProxmoxCluster from "../../../Models/DatabaseModels/ProxmoxCluster";
-import IoTFleet from "../../../Models/DatabaseModels/IoTFleet";
-import Service from "../../../Models/DatabaseModels/Service";
 import SortOrder from "../../../Types/BaseDatabase/SortOrder";
 import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
 import Dictionary from "../../../Types/Dictionary";
@@ -24,11 +18,9 @@ import AlertService from "../../Services/AlertService";
 import AlertSeverityService from "../../Services/AlertSeverityService";
 import ProjectScopedReferenceValidator from "../Database/ProjectScopedReferenceValidator";
 import AlertStateTimelineService from "../../Services/AlertStateTimelineService";
-import HostService from "../../Services/HostService";
 import NetworkDeviceOwnerUserService, {
   NetworkDeviceOwners,
 } from "../../Services/NetworkDeviceOwnerUserService";
-import ServiceService from "../../Services/ServiceService";
 import logger, { LogAttributes } from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 import DataToProcess from "./DataToProcess";
@@ -36,6 +28,7 @@ import MonitorClusterContextUtil, {
   MonitorClusterContext,
 } from "./MonitorClusterContext";
 import MonitorTemplateUtil from "./MonitorTemplateUtil";
+import SeriesResourceLinker from "./SeriesResourceLinker";
 import MonitorDependencySuppression, {
   DependencySuppressionResult,
 } from "./MonitorDependencySuppression";
@@ -553,118 +546,32 @@ export default class MonitorAlert {
           alert.seriesLabels = seriesLabels;
 
           /*
-           * Link the alert to the Host that emitted this series, if
-           * the metric carried a `host.name` resource attribute. The
-           * Host record was auto-discovered during OTel ingestion.
-           * Metric attributes are stored with the `resource.` prefix in
-           * ClickHouse, so the group-by dropdown surfaces
-           * `resource.host.name` — but accept the bare `host.name` too
-           * for any non-OTel ingest paths.
+           * Attach every resource this series identifies — host, docker
+           * host, podman host, k8s cluster, service, and the Proxmox /
+           * Ceph / Swarm / IoT clusters — resolved from the shared label
+           * key map. Same call the incident path makes, so the two can't
+           * drift apart again.
            */
-          const hostName: string | undefined =
-            typeof seriesLabels["resource.host.name"] === "string"
-              ? (seriesLabels["resource.host.name"] as string)
-              : typeof seriesLabels["host.name"] === "string"
-                ? (seriesLabels["host.name"] as string)
-                : undefined;
-
-          if (hostName) {
-            const host: Host | null = await HostService.findOneBy({
-              query: {
-                projectId: input.monitor.projectId!,
-                hostIdentifier: hostName,
-              },
-              select: {
-                _id: true,
-              },
-              props: {
-                isRoot: true,
-              },
-            });
-
-            if (host) {
-              alert.hosts = [host];
-            }
-          }
-
-          /*
-           * Same idea for Service — OTel ingest stamps `service.name` and
-           * auto-creates a Service row keyed by that name (see
-           * OpenTelemetryIngestService.telemetryServiceFromName). When the
-           * breaching series carries that attribute, link the alert to the
-           * emitting service so the on-call/labels pipeline can fan out.
-           */
-          const serviceName: string | undefined =
-            typeof seriesLabels["resource.service.name"] === "string"
-              ? (seriesLabels["resource.service.name"] as string)
-              : typeof seriesLabels["service.name"] === "string"
-                ? (seriesLabels["service.name"] as string)
-                : undefined;
-
-          if (serviceName) {
-            const service: Service | null = await ServiceService.findOneBy({
-              query: {
-                projectId: input.monitor.projectId!,
-                name: serviceName,
-              },
-              select: {
-                _id: true,
-              },
-              props: {
-                isRoot: true,
-              },
-            });
-
-            if (service) {
-              alert.services = [service];
-            }
-          }
+          await SeriesResourceLinker.linkSeriesResourcesToModel({
+            model: alert,
+            seriesLabels,
+            projectId: input.monitor.projectId!,
+          });
         }
 
         /*
-         * Deterministic Proxmox/Ceph cluster link from the monitor's
-         * step config (resolved once above). Series labels never carry
-         * cluster identity for these monitor types, so this — not the
-         * label path above — is what makes the per-cluster Activity
-         * tabs and badge counts see monitor-created alerts. Runs for
-         * both grouped and ungrouped alerts.
+         * Deterministic Proxmox/Ceph/Swarm/IoT cluster link from the
+         * monitor's step config (resolved once above). The shipped
+         * templates for those monitor types group by datapoint labels
+         * that carry no cluster identity, so this — not the label path
+         * above — is what makes the per-cluster Activity tabs and badge
+         * counts see monitor-created alerts. Runs for both grouped and
+         * ungrouped alerts, and merges with whatever the labels resolved.
          */
-        if (clusterContext.proxmoxClusterIds.length > 0) {
-          alert.proxmoxClusters = clusterContext.proxmoxClusterIds.map(
-            (id: string): ProxmoxCluster => {
-              const cluster: ProxmoxCluster = new ProxmoxCluster();
-              cluster._id = id;
-              return cluster;
-            },
-          );
-        }
-        if (clusterContext.cephClusterIds.length > 0) {
-          alert.cephClusters = clusterContext.cephClusterIds.map(
-            (id: string): CephCluster => {
-              const cluster: CephCluster = new CephCluster();
-              cluster._id = id;
-              return cluster;
-            },
-          );
-        }
-        if (clusterContext.dockerSwarmClusterIds.length > 0) {
-          alert.dockerSwarmClusters = clusterContext.dockerSwarmClusterIds.map(
-            (id: string): DockerSwarmCluster => {
-              const cluster: DockerSwarmCluster = new DockerSwarmCluster();
-              cluster._id = id;
-              return cluster;
-            },
-          );
-        }
-        if (clusterContext.iotFleetIds.length > 0) {
-          alert.iotFleets = clusterContext.iotFleetIds.map(
-            (id: string): IoTFleet => {
-              const fleet: IoTFleet = new IoTFleet();
-              fleet._id = id;
-              return fleet;
-            },
-          );
-        }
+        SeriesResourceLinker.attachClusterContext({
+          model: alert,
+          clusterContext,
+        });
 
         alert.onCallDutyPolicies =
           criteriaAlert.onCallPolicyIds?.map((id: ObjectID) => {