@oneuptime/common 11.4.0 → 11.5.0

package/Server/Services/AIService.ts

@@ -19,10 +19,30 @@ import LlmProvider from "../../Models/DatabaseModels/LlmProvider";
 import LlmLog from "../../Models/DatabaseModels/LlmLog";
 import LlmLogStatus from "../../Types/LlmLogStatus";
 import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
 import BadDataException from "../../Types/Exception/BadDataException";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import logger, { LogAttributes } from "../Utils/Logger";
 
+/*
+ * Features that run WITHOUT a human in the loop. The per-project daily token
+ * budget (Project.aiDailyAutonomousTokenLimit, G4) applies only to these —
+ * interactive chat and explicitly user-triggered AI are never budget-blocked.
+ * Auto-postmortem is deliberately excluded for now: it is one call per
+ * resolved incident, not storm-shaped; include it when it moves to the queue.
+ */
+export const AUTONOMOUS_AI_FEATURES: Array<string> = [
+  "Sentinel Incident Investigation",
+  "Sentinel Alert Investigation",
+];
+
+export interface AutonomousBudgetStatus {
+  exhausted: boolean;
+  // null when the project has no limit configured.
+  limitInTokens: number | null;
+  usedTokensToday: number;
+}
+
 export interface AILogRequest {
   projectId: ObjectID;
   userId?: ObjectID | undefined;
@@ -59,6 +79,54 @@ export class Service extends BaseService {
     super();
   }
 
+  /*
+   * G4 daily budget: has this project consumed its daily autonomous-token
+   * allowance (UTC day)? Counts only AUTONOMOUS_AI_FEATURES tokens, so chat
+   * usage neither eats the autonomous budget nor is blocked by it.
+   */
+  @CaptureSpan()
+  public async getAutonomousDailyBudgetStatus(
+    projectId: ObjectID,
+  ): Promise<AutonomousBudgetStatus> {
+    const project: Project | null = await ProjectService.findOneById({
+      id: projectId,
+      select: { aiDailyAutonomousTokenLimit: true },
+      props: { isRoot: true },
+    });
+
+    const limitInTokens: number | null =
+      project?.aiDailyAutonomousTokenLimit ?? null;
+
+    if (limitInTokens === null) {
+      return { exhausted: false, limitInTokens: null, usedTokensToday: 0 };
+    }
+
+    /*
+     * A limit of 0 (or negative) pauses autonomous runs entirely — the safe
+     * reading of "0 tokens allowed", and doubles as a spend kill-switch.
+     */
+    if (limitInTokens <= 0) {
+      return { exhausted: true, limitInTokens, usedTokensToday: 0 };
+    }
+
+    const usedTokensToday: number = await LlmLogService.getTotalTokensUsedSince(
+      {
+        projectId,
+        since: OneUptimeDate.getStartOfDay(
+          OneUptimeDate.getCurrentDate(),
+          "UTC",
+        ),
+        features: AUTONOMOUS_AI_FEATURES,
+      },
+    );
+
+    return {
+      exhausted: usedTokensToday >= limitInTokens,
+      limitInTokens,
+      usedTokensToday,
+    };
+  }
+
   @CaptureSpan()
   public async executeWithLogging(
     request: AILogRequest,
@@ -169,6 +237,34 @@ export class Service extends BaseService {
       }
     }
 
+    /*
+     * G4 daily budget enforcement — autonomous features only, mirroring the
+     * InsufficientBalance path above. Autonomous runs fail closed on budget
+     * (G9); interactive features are never blocked here. A run that crosses
+     * the limit mid-flight errors on its next LLM call, which marks the AIRun
+     * Error with this message — visible in the investigation panel.
+     */
+    if (AUTONOMOUS_AI_FEATURES.includes(request.feature)) {
+      const budget: AutonomousBudgetStatus =
+        await this.getAutonomousDailyBudgetStatus(request.projectId);
+
+      if (budget.exhausted) {
+        const budgetMessage: string = `Daily autonomous AI token budget exhausted (${budget.usedTokensToday.toLocaleString()} of ${budget.limitInTokens?.toLocaleString()} tokens used today). Autonomous investigations resume tomorrow (UTC) — raise or unset the limit in the AI settings pages.`;
+
+        logEntry.status = LlmLogStatus.BudgetExceeded;
+        logEntry.statusMessage = budgetMessage.substring(0, 490);
+        logEntry.requestCompletedAt = new Date();
+        logEntry.durationMs = new Date().getTime() - startTime.getTime();
+
+        await LlmLogService.create({
+          data: logEntry,
+          props: { isRoot: true },
+        });
+
+        throw new BadDataException(budgetMessage);
+      }
+    }
+
     try {
       // Build LLM config
       const llmConfig: LLMProviderConfig = {
@@ -194,6 +290,9 @@ export class Service extends BaseService {
         temperature: request.temperature ?? 0.7,
         maxTokens: request.maxTokens,
         tools: request.tools,
+        ...(llmProvider.additionalParams
+          ? { additionalParams: llmProvider.additionalParams }
+          : {}),
       });
 
       const endTime: Date = new Date();
@@ -201,6 +300,8 @@ export class Service extends BaseService {
       // Update log with success info
       logEntry.status = LlmLogStatus.Success;
       logEntry.totalTokens = response.usage?.totalTokens || 0;
+      logEntry.cachedInputTokens = response.usage?.cachedInputTokens || 0;
+      logEntry.cacheCreationTokens = response.usage?.cacheCreationTokens || 0;
       logEntry.responsePreview = storeContentPreviews
         ? response.content.substring(0, 2000) // Store first 2000 chars
         : "[Redacted — this content is private to the requesting user]";

package/Server/Services/AnalyticsDatabaseService.ts

@@ -813,6 +813,19 @@ export default class AnalyticsDatabaseService<
         ? Object.keys(aggregateBy.groupBy)
         : [];
 
+      /*
+       * Attribute-key grouping returns the selected keys folded into a
+       * single `attributes` map column (see MetricService); carry it
+       * onto the aggregated rows so series splitters can read labels.
+       */
+      if (
+        aggregateBy.groupByAttributeKeys &&
+        aggregateBy.groupByAttributeKeys.length > 0 &&
+        !groupByColumnNames.includes("attributes")
+      ) {
+        groupByColumnNames.push("attributes");
+      }
+
       for (const item of items) {
         if (
           !(item as JSONObject)[
@@ -1171,6 +1184,20 @@ export default class AnalyticsDatabaseService<
     statement: Statement;
     columns: Array<string>;
   } {
+    /*
+     * Attribute-key grouping needs model-specific SQL over a map column
+     * (MetricService builds it); reaching this generic path with keys
+     * set would silently return ungrouped results.
+     */
+    if (
+      aggregateBy.groupByAttributeKeys &&
+      aggregateBy.groupByAttributeKeys.length > 0
+    ) {
+      throw new BadDataException(
+        `groupByAttributeKeys is not supported for ${this.model.tableName}.`,
+      );
+    }
+
     if (!this.database) {
       this.useDefaultDatabase();
     }

package/Server/Services/IncidentService.ts

@@ -83,12 +83,7 @@ import Dictionary from "../../Types/Dictionary";
 import ProjectService from "./ProjectService";
 import IncidentTemplateService from "./IncidentTemplateService";
 import IncidentTemplate from "../../Models/DatabaseModels/IncidentTemplate";
-import LLMService, {
-  LLMProviderConfig,
-  LLMCompletionResponse,
-} from "../Utils/LLM/LLMService";
-import LlmProviderService from "./LlmProviderService";
-import LlmProvider from "../../Models/DatabaseModels/LlmProvider";
+import AIService, { AILogResponse } from "./AIService";
 import SentinelIncidentInvestigationRunner from "../Utils/AI/Sentinel/IncidentInvestigationRunner";
 import IncidentAIContextBuilder, {
   AIGenerationContext,
@@ -3453,22 +3448,6 @@ ${incidentSeverity.name}
       throw new BadDataException("Incident not found");
     }
 
-    // Get LLM provider for the project
-    const llmProvider: LlmProvider | null =
-      await LlmProviderService.getLLMProviderForProject(incident.projectId);
-
-    if (!llmProvider) {
-      throw new BadDataException(
-        "No LLM provider configured for this project. Please configure an LLM provider in Settings > AI > LLM Providers.",
-      );
-    }
-
-    if (!llmProvider.llmType) {
-      throw new BadDataException(
-        "LLM provider type is not configured properly.",
-      );
-    }
-
     // Build incident context - always include workspace messages
     const contextData: IncidentContextData =
       await IncidentAIContextBuilder.buildIncidentContext({
@@ -3484,27 +3463,21 @@ ${incidentSeverity.name}
         data.template,
       );
 
-    // Generate postmortem using LLM
-    const llmConfig: LLMProviderConfig = {
-      llmType: llmProvider.llmType,
-    };
-
-    if (llmProvider.apiKey) {
-      llmConfig.apiKey = llmProvider.apiKey;
-    }
-
-    if (llmProvider.baseUrl) {
-      llmConfig.baseUrl = llmProvider.baseUrl.toString();
-    }
-
-    if (llmProvider.modelName) {
-      llmConfig.modelName = llmProvider.modelName;
-    }
-
-    const response: LLMCompletionResponse = await LLMService.getCompletion({
-      llmProviderConfig: llmConfig,
+    /*
+     * Route through AIService so the call is metered, billed and budget-
+     * checked like every other AI feature — the previous direct
+     * LLMService.getCompletion bypassed LlmLog and cloud billing entirely.
+     * Previews stay off: the prompt embeds incident context (including
+     * private notes and workspace messages) whose read ACLs are narrower
+     * than LlmLog's (G8).
+     */
+    const response: AILogResponse = await AIService.executeWithLogging({
+      projectId: incident.projectId,
+      feature: "Incident Postmortem",
+      incidentId: data.incidentId,
       messages: aiContext.messages,
       temperature: 0.2,
+      storeContentPreviews: false,
     });
 
     return response.content;

package/Server/Services/Index.ts

@@ -36,6 +36,13 @@ import IncidentStateTimelineService from "./IncidentStateTimelineService";
 import LabelService from "./LabelService";
 import KubernetesClusterService from "./KubernetesClusterService";
 import DockerHostService from "./DockerHostService";
+import NetworkDeviceService from "./NetworkDeviceService";
+import NetworkDeviceOwnerTeamService from "./NetworkDeviceOwnerTeamService";
+import NetworkDeviceOwnerUserService from "./NetworkDeviceOwnerUserService";
+import NetworkDeviceOwnerRuleService from "./NetworkDeviceOwnerRuleService";
+import NetworkDeviceLabelRuleService from "./NetworkDeviceLabelRuleService";
+import NetworkDeviceDiscoveryScanService from "./NetworkDeviceDiscoveryScanService";
+import NetworkInterfaceService from "./NetworkInterfaceService";
 import PodmanHostService from "./PodmanHostService";
 import ProxmoxClusterService from "./ProxmoxClusterService";
 import DockerSwarmClusterService from "./DockerSwarmClusterService";
@@ -301,6 +308,13 @@ const services: Array<BaseService> = [
   LabelService,
   KubernetesClusterService,
   DockerHostService,
+  NetworkDeviceService,
+  NetworkDeviceOwnerTeamService,
+  NetworkDeviceOwnerUserService,
+  NetworkDeviceOwnerRuleService,
+  NetworkDeviceLabelRuleService,
+  NetworkDeviceDiscoveryScanService,
+  NetworkInterfaceService,
   PodmanHostService,
   ProxmoxClusterService,
   DockerSwarmClusterService,

package/Server/Services/IoTDeviceCredentialService.ts

@@ -0,0 +1,351 @@
+import CreateBy from "../Types/Database/CreateBy";
+import DeleteBy from "../Types/Database/DeleteBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
+import DatabaseService from "./DatabaseService";
+import IoTFleetService from "./IoTFleetService";
+import IoTFleet from "../../Models/DatabaseModels/IoTFleet";
+import GlobalCache from "../Infrastructure/GlobalCache";
+import InMemoryTTLCache from "../Infrastructure/InMemoryTTLCache";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import PositiveNumber from "../../Types/PositiveNumber";
+import BadDataException from "../../Types/Exception/BadDataException";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import Model from "../../Models/DatabaseModels/IoTDeviceCredential";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import logger from "../Utils/Logger";
+
+/*
+ * Cache TTLs mirror TelemetryIngestionKeyService: the positive TTL is
+ * the worst-case revocation lag on the MQTT hot path (the broker
+ * re-validates each publish through this cache, so disabling or
+ * deleting a credential cuts a connected device off within ~60s);
+ * the shorter negative TTL keeps invalid-credential floods off
+ * Postgres without pinning misses for long.
+ */
+const POSITIVE_TTL_MS: number = 60 * 1000;
+const NEGATIVE_TTL_MS: number = 10 * 1000;
+
+// lastConnectedAt heartbeat writes are throttled through Redis.
+const LAST_CONNECTED_CACHE_NAMESPACE: string =
+  "iot-device-credential-last-connected";
+const LAST_CONNECTED_THROTTLE_SECONDS: number = 60;
+
+/*
+ * Everything the MQTT broker needs to authenticate and scope a
+ * device-credential CONNECT: the tenant, and the (fleet, device)
+ * identity the client is allowed to publish under.
+ */
+export interface IoTDeviceCredentialContext {
+  credentialId: string;
+  projectId: string;
+  iotFleetId: string;
+  fleetName: string;
+  externalId: string;
+  secretKey: string;
+}
+
+export class Service extends DatabaseService<Model> {
+  private contextCache: InMemoryTTLCache<IoTDeviceCredentialContext | null> =
+    new InMemoryTTLCache(10_000);
+
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    /*
+     * Always server-generate the secret. secretKey is a computed
+     * column (create ACL []), and computed columns are skipped rather
+     * than stripped by the create-permission check — so a
+     * client-supplied value would otherwise survive verbatim and
+     * defeat the unpredictability the MQTT password relies on.
+     */
+    createBy.data.secretKey = ObjectID.generate();
+
+    /*
+     * The tenant column (projectId) is forced to the caller's project
+     * by the framework, but the iotFleetId relation is not — reject a
+     * fleet that belongs to another project so a credential cannot be
+     * attached to a victim fleet (which would let it steer that
+     * fleet's inventory cleanup and mint a cross-tenant auth context).
+     */
+    await this.validateFleetBelongsToProject(createBy);
+
+    /*
+     * Byte-exact duplicate guard. The DB unique index is
+     * case-sensitive (device.id labels are matched byte-exact), so the
+     * check must be too — a case-insensitive @UniqueColumnBy would
+     * wrongly reject two legitimately-distinct device ids that differ
+     * only in case.
+     */
+    await this.validateDeviceIdIsUnique(createBy);
+
+    return { createBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  private async validateFleetBelongsToProject(
+    createBy: CreateBy<Model>,
+  ): Promise<void> {
+    if (!createBy.data.iotFleetId || !createBy.data.projectId) {
+      return;
+    }
+
+    const fleet: IoTFleet | null = await IoTFleetService.findOneById({
+      id: createBy.data.iotFleetId,
+      select: {
+        _id: true,
+        projectId: true,
+      },
+      props: { isRoot: true },
+    });
+
+    if (
+      !fleet?.projectId ||
+      fleet.projectId.toString() !== createBy.data.projectId.toString()
+    ) {
+      throw new BadDataException("IoT Fleet not found in this project.");
+    }
+  }
+
+  @CaptureSpan()
+  private async validateDeviceIdIsUnique(
+    createBy: CreateBy<Model>,
+  ): Promise<void> {
+    if (
+      !createBy.data.projectId ||
+      !createBy.data.iotFleetId ||
+      !createBy.data.externalId
+    ) {
+      return;
+    }
+
+    const existingCount: PositiveNumber = await this.countBy({
+      query: {
+        projectId: createBy.data.projectId,
+        iotFleetId: createBy.data.iotFleetId,
+        externalId: createBy.data.externalId,
+      },
+      props: { isRoot: true },
+    });
+
+    if (existingCount.toNumber() > 0) {
+      throw new BadDataException(
+        "A device with this Device ID is already registered in this fleet.",
+      );
+    }
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<Model>,
+  ): Promise<OnDelete<Model>> {
+    /*
+     * We don't know which credential(s) are being deleted without an
+     * extra query; clear the whole cache so revocation-by-delete takes
+     * effect on the broker within one cache miss.
+     */
+    this.contextCache.clear();
+    return { deleteBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    // isEnabled flips must reach the broker — same reasoning as delete.
+    this.contextCache.clear();
+    return { updateBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  protected override async onUpdateSuccess(
+    onUpdate: OnUpdate<Model>,
+    _updatedItemIds: Array<ObjectID>,
+  ): Promise<OnUpdate<Model>> {
+    /*
+     * Clear again AFTER the write commits: the before-hook clear races
+     * a concurrent getCredentialContext that could re-cache the stale
+     * (pre-revocation) row before the UPDATE lands. A second clear on
+     * success closes that window.
+     */
+    this.contextCache.clear();
+    return onUpdate;
+  }
+
+  @CaptureSpan()
+  protected override async onDeleteSuccess(
+    onDelete: OnDelete<Model>,
+    _itemIdsBeforeDelete: Array<ObjectID>,
+  ): Promise<OnDelete<Model>> {
+    this.contextCache.clear();
+    return onDelete;
+  }
+
+  /**
+   * Resolve a credential id to its auth context, with a short-lived
+   * in-process cache to keep the MQTT CONNECT/publish hot path off
+   * Postgres. Returns null for unknown, disabled, or deleted
+   * credentials (also cached, for a shorter TTL). The caller must
+   * still compare the presented secret against context.secretKey.
+   */
+  @CaptureSpan()
+  public async getCredentialContext(
+    credentialId: string,
+  ): Promise<IoTDeviceCredentialContext | null> {
+    const cached: IoTDeviceCredentialContext | null | undefined =
+      this.contextCache.get(credentialId);
+    if (cached !== undefined) {
+      return cached;
+    }
+
+    let idObject: ObjectID;
+    try {
+      idObject = new ObjectID(credentialId);
+    } catch {
+      this.contextCache.set(credentialId, null, NEGATIVE_TTL_MS);
+      return null;
+    }
+
+    const credential: Model | null = await this.findOneBy({
+      query: { _id: idObject },
+      select: {
+        projectId: true,
+        iotFleetId: true,
+        externalId: true,
+        secretKey: true,
+        isEnabled: true,
+        iotFleet: {
+          name: true,
+        },
+      },
+      props: { isRoot: true },
+    });
+
+    if (
+      !credential ||
+      credential.isEnabled === false ||
+      !credential.projectId ||
+      !credential.iotFleetId ||
+      !credential.externalId ||
+      !credential.secretKey ||
+      !credential.iotFleet?.name
+    ) {
+      this.contextCache.set(credentialId, null, NEGATIVE_TTL_MS);
+      return null;
+    }
+
+    const context: IoTDeviceCredentialContext = {
+      credentialId: credentialId,
+      projectId: credential.projectId.toString(),
+      iotFleetId: credential.iotFleetId.toString(),
+      fleetName: credential.iotFleet.name,
+      externalId: credential.externalId,
+      secretKey: credential.secretKey.toString(),
+    };
+
+    this.contextCache.set(credentialId, context, POSITIVE_TTL_MS);
+    return context;
+  }
+
+  /**
+   * Stamp lastConnectedAt for a credential, throttled to one write
+   * per minute per credential (heartbeat write — same hot-row
+   * pattern as IoTFleetService.updateLastSeen).
+   */
+  @CaptureSpan()
+  public async markConnected(credentialId: ObjectID): Promise<void> {
+    const cacheKey: string = credentialId.toString();
+
+    let cached: string | null = null;
+    try {
+      cached = await GlobalCache.getString(
+        LAST_CONNECTED_CACHE_NAMESPACE,
+        cacheKey,
+      );
+    } catch {
+      /*
+       * Fail open: a Redis blip must never skip the DB write —
+       * lastConnectedAt drives the registry UI's liveness column.
+       */
+      cached = null;
+    }
+
+    if (cached === "1") {
+      return;
+    }
+
+    try {
+      await GlobalCache.setString(
+        LAST_CONNECTED_CACHE_NAMESPACE,
+        cacheKey,
+        "1",
+        {
+          expiresInSeconds: LAST_CONNECTED_THROTTLE_SECONDS,
+        },
+      );
+    } catch (err) {
+      logger.debug(
+        `IoTDeviceCredential markConnected cache write failed: ${err}`,
+      );
+    }
+
+    /*
+     * Single no-hooks, no-version-bump UPDATE — a heartbeat write must
+     * not contend on hooks or lock convoys at connect storms.
+     */
+    await this.updateColumnsByIdWithoutHooks({
+      id: credentialId,
+      data: {
+        lastConnectedAt: OneUptimeDate.getCurrentDate(),
+      },
+    });
+  }
+
+  /**
+   * The expected-device set for a fleet — ENABLED registered device
+   * ids, VERBATIM (device.id labels are matched byte-exact; do not
+   * canonicalize). Feeds ONLY the absent-series injection in IoT
+   * Device monitor evaluation.
+   *
+   * Note: stale-cleanup preservation (IoTDeviceService.deleteStaleForFleet)
+   * deliberately does NOT use this method — it preserves any
+   * non-deleted credential row regardless of isEnabled, so a revoked
+   * (disabled) device stays in the inventory as Offline until its
+   * credential row is deleted, while its silent-death alerting pauses.
+   */
+  @CaptureSpan()
+  public async getExpectedDeviceExternalIds(data: {
+    projectId: ObjectID;
+    iotFleetId: ObjectID;
+  }): Promise<Array<string>> {
+    const credentials: Array<Model> = await this.findBy({
+      query: {
+        projectId: data.projectId,
+        iotFleetId: data.iotFleetId,
+        isEnabled: true,
+      },
+      select: {
+        externalId: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    const externalIds: Array<string> = [];
+    for (const credential of credentials) {
+      if (credential.externalId) {
+        externalIds.push(credential.externalId);
+      }
+    }
+    return externalIds;
+  }
+}
+
+export default new Service();