@oneuptime/common 12.0.4 → 12.0.5

package/Server/Utils/AI/SRE/SubjectCodeFixRun.ts

@@ -3,16 +3,32 @@ import AIRunType from "../../../../Types/AI/AIRunType";
 import AIRunStatus, {
   AIRunStatusHelper,
 } from "../../../../Types/AI/AIRunStatus";
+import AIRunCodeFixRecommendation from "../../../../Types/AI/AIRunCodeFixRecommendation";
 import CodeFixTaskType from "../../../../Types/AI/CodeFixTaskType";
-import CodeFixTaskContext from "../../../../Types/AI/CodeFixTaskContext";
+import CodeFixTaskContext, {
+  getInvestigationCodeFixTaskSnapshot,
+  InvestigationCodeFixTaskSnapshot,
+} from "../../../../Types/AI/CodeFixTaskContext";
 import CodeRepositoryType from "../../../../Types/CodeRepository/CodeRepositoryType";
+import BadDataException from "../../../../Types/Exception/BadDataException";
 import { LIMIT_PER_PROJECT } from "../../../../Types/Database/LimitMax";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
 import AIRun from "../../../../Models/DatabaseModels/AIRun";
 import AIRunService from "../../../Services/AIRunService";
 import CodeRepositoryService from "../../../Services/CodeRepositoryService";
 import FixRunBudget from "../CodeFix/FixRunBudget";
 import QueryHelper from "../../../Types/Database/QueryHelper";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
+import InvestigationSubjectLock from "./InvestigationSubjectLock";
+
+interface SubjectCodeFixRunInput {
+  projectId: ObjectID;
+  taskType: CodeFixTaskType;
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
+  userId?: ObjectID | undefined;
+  taskContext?: CodeFixTaskContext | undefined;
+}
 
 /*
  * Shared plumbing for the non-exception CodeFix recipes: the incident/alert
@@ -60,6 +76,12 @@ export default class SubjectCodeFixRun {
     incidentId?: ObjectID | undefined;
     alertId?: ObjectID | undefined;
   }): Promise<AIRun | null> {
+    if (Boolean(data.incidentId) === Boolean(data.alertId)) {
+      throw new BadDataException(
+        "Exactly one incident or alert subject is required to find an existing fix task.",
+      );
+    }
+
     return AIRunService.findOneBy({
       query: {
         runType: AIRunType.CodeFix,
@@ -160,15 +182,59 @@ export default class SubjectCodeFixRun {
    * its never-throws wrapper as the backstop.
    */
   @CaptureSpan()
-  public static async enqueueSubjectCodeFixRun(data: {
-    projectId: ObjectID;
-    taskType: CodeFixTaskType;
-    incidentId?: ObjectID | undefined;
-    alertId?: ObjectID | undefined;
-    userId?: ObjectID | undefined;
-    taskContext?: CodeFixTaskContext | undefined;
-  }): Promise<AIRun> {
-    await FixRunBudget.assertWithinBudget(data.projectId);
+  public static async enqueueSubjectCodeFixRun(
+    data: SubjectCodeFixRunInput,
+  ): Promise<AIRun> {
+    if (data.incidentId && data.alertId) {
+      throw new BadDataException(
+        "A fix task cannot belong to both an incident and an alert.",
+      );
+    }
+
+    /*
+     * FixFromIncident has both human and automatic writers, and its source
+     * authorization depends on which Investigation is latest. Serialize the
+     * final dedupe + source revalidation + INSERT with Investigation enqueue
+     * for this subject. Redis is a correctness dependency: lock acquisition
+     * failure fails closed instead of falling back to a racy write.
+     */
+    if (data.taskType === CodeFixTaskType.FixFromIncident) {
+      return InvestigationSubjectLock.runExclusive(
+        {
+          projectId: data.projectId,
+          incidentId: data.incidentId,
+          alertId: data.alertId,
+        },
+        async (): Promise<AIRun> => {
+          const existingRun: AIRun | null =
+            await this.findNonTerminalRunForSubject({
+              taskType: data.taskType,
+              incidentId: data.incidentId,
+              alertId: data.alertId,
+            });
+
+          if (existingRun) {
+            throw new BadDataException(
+              "A fix pull request task is already queued or running for this subject.",
+            );
+          }
+
+          return this.createSubjectCodeFixRun(data, true);
+        },
+      );
+    }
+
+    return this.createSubjectCodeFixRun(data, false);
+  }
+
+  private static async createSubjectCodeFixRun(
+    data: SubjectCodeFixRunInput,
+    revalidateInvestigationSource: boolean,
+  ): Promise<AIRun> {
+    await FixRunBudget.assertWithinBudget(data.projectId, {
+      incidentId: data.incidentId,
+      alertId: data.alertId,
+    });
 
     const run: AIRun = new AIRun();
     run.projectId = data.projectId;
@@ -190,9 +256,65 @@ export default class SubjectCodeFixRun {
       run.taskContext = data.taskContext;
     }
 
+    if (revalidateInvestigationSource) {
+      /* No awaited work may be added between this gate and the INSERT. */
+      await this.assertLatestRecommendedInvestigationSource(data);
+    }
+
     return AIRunService.create({
       data: run,
       props: { isRoot: true },
     });
   }
+
+  private static async assertLatestRecommendedInvestigationSource(
+    data: SubjectCodeFixRunInput,
+  ): Promise<void> {
+    const requestedSnapshot: InvestigationCodeFixTaskSnapshot | null =
+      getInvestigationCodeFixTaskSnapshot(data.taskContext);
+
+    if (!requestedSnapshot) {
+      throw new BadDataException(
+        "A complete Recommended investigation snapshot is required to create a fix task.",
+      );
+    }
+
+    const latestInvestigation: AIRun | null = await AIRunService.findOneBy({
+      query: {
+        projectId: data.projectId,
+        runType: AIRunType.Investigation,
+        ...(data.incidentId
+          ? { triggeredByIncidentId: data.incidentId }
+          : { triggeredByAlertId: data.alertId! }),
+      },
+      select: {
+        _id: true,
+        status: true,
+        codeFixRecommendation: true,
+        taskContext: true,
+      },
+      sort: { createdAt: SortOrder.Descending },
+      props: { isRoot: true },
+    });
+    const persistedSnapshot: InvestigationCodeFixTaskSnapshot | null =
+      getInvestigationCodeFixTaskSnapshot(latestInvestigation?.taskContext);
+
+    if (
+      !latestInvestigation?.id ||
+      latestInvestigation.id.toString() !==
+        requestedSnapshot.investigationRunId ||
+      latestInvestigation.status !== AIRunStatus.Completed ||
+      latestInvestigation.codeFixRecommendation !==
+        AIRunCodeFixRecommendation.Recommended ||
+      !persistedSnapshot ||
+      persistedSnapshot.investigationRunId !==
+        requestedSnapshot.investigationRunId ||
+      persistedSnapshot.investigationAnalysisMarkdown !==
+        requestedSnapshot.investigationAnalysisMarkdown
+    ) {
+      throw new BadDataException(
+        "The source investigation is no longer the latest durably Recommended analysis. Refresh and try again.",
+      );
+    }
+  }
 }

package/Server/Utils/AnalyticsDatabase/ClusterConfig.ts

@@ -27,6 +27,8 @@ import AnalyticsTableEngine from "../../../Types/AnalyticsDatabase/AnalyticsTabl
 export const DEFAULT_CLICKHOUSE_CLUSTER_NAME: string = "oneuptime";
 export const DEFAULT_CLICKHOUSE_SHARDING_KEY: string = "cityHash64(projectId)";
 export const DEFAULT_CLICKHOUSE_DATABASE: string = "oneuptime";
+// Matches the ClickHouse server default for distributed_ddl_task_timeout.
+export const DEFAULT_DISTRIBUTED_DDL_TASK_TIMEOUT_SECONDS: number = 180;
 
 // Suffix appended to a model's tableName to name its local (data) table.
 export const LOCAL_TABLE_SUFFIX: string = "Local";
@@ -49,6 +51,28 @@ export function getClickhouseDatabaseName(): string {
   return process.env["CLICKHOUSE_DATABASE"] || DEFAULT_CLICKHOUSE_DATABASE;
 }
 
+/*
+ * How long the initiator of an ON CLUSTER DDL statement waits for every host
+ * to report completion (CLICKHOUSE_DISTRIBUTED_DDL_TASK_TIMEOUT_SECONDS,
+ * default 180 — the ClickHouse server default). ClickHouse semantics: 0
+ * enqueues the task and returns immediately; a negative value removes the
+ * server-side wait limit, but the confirmation wait streams zero bytes, so
+ * the client still gives up at the migration pool's socket-idle ceiling
+ * (30-minute floor, scaled up with this value — see ClickhouseConfig.ts).
+ * Prefer a finite value. Raising it helps on clusters whose DDL queue drains
+ * slowly (each host's DDLWorker processes the queue sequentially by default,
+ * so one slow task delays all later ones).
+ */
+export function getDistributedDdlTaskTimeoutSeconds(): number {
+  const raw: string = (
+    process.env["CLICKHOUSE_DISTRIBUTED_DDL_TASK_TIMEOUT_SECONDS"] || ""
+  ).trim();
+  const parsed: number = parseInt(raw, 10);
+  return Number.isFinite(parsed)
+    ? parsed
+    : DEFAULT_DISTRIBUTED_DDL_TASK_TIMEOUT_SECONDS;
+}
+
 /*
  * The `ON CLUSTER '<name>'` clause (with a leading space), emitted on every
  * object-lifecycle DDL statement (CREATE TABLE / CREATE MATERIALIZED VIEW /

package/Server/Utils/Monitor/Criteria/APIRequestCriteria.ts

@@ -88,6 +88,35 @@ export default class APIRequestCriteria {
       });
     }
 
+    if (
+      input.criteriaFilter.checkOn === CheckOn.PortDnsLookupTime ||
+      input.criteriaFilter.checkOn === CheckOn.PortTcpConnectTime
+    ) {
+      const portTimings: ProbeMonitorResponse["portTimings"] = (
+        input.dataToProcess as ProbeMonitorResponse
+      ).portTimings;
+
+      const currentValue: number | undefined =
+        input.criteriaFilter.checkOn === CheckOn.PortDnsLookupTime
+          ? portTimings?.dnsLookupInMs
+          : portTimings?.tcpConnectInMs;
+
+      const value: Array<number> | number | undefined =
+        (overTimeValue as Array<number> | number | undefined) ?? currentValue;
+
+      if (value === undefined) {
+        return null;
+      }
+
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: value,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
     // check packet loss (Ping/IP monitors with multi-packet checks)
     if (input.criteriaFilter.checkOn === CheckOn.PacketLossPercent) {
       const packetLossPercent: number | undefined = (

package/Server/Utils/Monitor/Criteria/DomainMonitorCriteria.ts

@@ -8,6 +8,8 @@ import {
 import DomainMonitorResponse from "../../../../Types/Monitor/DomainMonitor/DomainMonitorResponse";
 import ProbeMonitorResponse from "../../../../Types/Probe/ProbeMonitorResponse";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
+import EvaluateOverTime from "./EvaluateOverTime";
+import logger from "../../Logger";
 
 export default class DomainMonitorCriteria {
   @CaptureSpan()
@@ -24,6 +26,60 @@ export default class DomainMonitorCriteria {
     const domainResponse: DomainMonitorResponse | undefined =
       dataToProcess.domainResponse;
 
+    let overTimeValue: Array<number | boolean> | number | boolean | undefined =
+      undefined;
+
+    if (
+      input.criteriaFilter.evaluateOverTime &&
+      input.criteriaFilter.evaluateOverTimeOptions
+    ) {
+      try {
+        overTimeValue = await EvaluateOverTime.getValueOverTime({
+          projectId: dataToProcess.projectId,
+          monitorId: input.dataToProcess.monitorId!,
+          evaluateOverTimeOptions: input.criteriaFilter.evaluateOverTimeOptions,
+          metricType: input.criteriaFilter.checkOn,
+        });
+
+        if (Array.isArray(overTimeValue) && overTimeValue.length === 0) {
+          overTimeValue = undefined;
+        }
+      } catch (err) {
+        logger.error(
+          `Error in getting over time value for ${input.criteriaFilter.checkOn}`,
+        );
+        logger.error(err);
+        overTimeValue = undefined;
+      }
+    }
+
+    /*
+     * Whether the registration lookup itself succeeded. Without this, a
+     * domain whose TLD has no working WHOIS or RDAP service produces a
+     * response with no expiry date, every Domain* filter below returns null,
+     * and the monitor silently keeps its previous status - which is what
+     * made an unsupported TLD look healthy.
+     */
+    if (input.criteriaFilter.checkOn === CheckOn.IsOnline) {
+      const currentIsOnline: boolean | Array<boolean> =
+        (overTimeValue as Array<boolean>) || dataToProcess.isOnline;
+
+      return CompareCriteria.compareCriteriaBoolean({
+        value: currentIsOnline,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    if (input.criteriaFilter.checkOn === CheckOn.IsRequestTimeout) {
+      const currentIsTimeout: boolean | Array<boolean> =
+        (overTimeValue as Array<boolean>) || dataToProcess.isTimeout;
+
+      return CompareCriteria.compareCriteriaBoolean({
+        value: currentIsTimeout,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
     // Check domain expires in days
     if (input.criteriaFilter.checkOn === CheckOn.DomainExpiresDaysIn) {
       threshold = CompareCriteria.convertToNumber(threshold);

package/Server/Utils/Monitor/MonitorMetricUtil.ts

@@ -17,6 +17,7 @@ import CapturedMetric from "../../../Types/Monitor/CustomCodeMonitor/CapturedMet
 import HttpPhaseTimings from "../../../Types/Monitor/HttpPhaseTimings";
 import MonitorMetricType from "../../../Types/Monitor/MonitorMetricType";
 import PingMonitorResponse from "../../../Types/Monitor/PingMonitor/PingMonitorResponse";
+import PortMonitorTimings from "../../../Types/Monitor/PortMonitor/PortMonitorTimings";
 import SnmpInterface from "../../../Types/Monitor/SnmpMonitor/SnmpInterface";
 import { SnmpOidResponse } from "../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
@@ -1093,6 +1094,51 @@ export default class MonitorMetricUtil {
       }
     }
 
+    const portTimings: PortMonitorTimings | undefined = (
+      data.dataToProcess as ProbeMonitorResponse
+    ).portTimings;
+
+    if (portTimings) {
+      const extraAttributes: JSONObject = {
+        probeId: (
+          data.dataToProcess as ProbeMonitorResponse
+        ).probeId.toString(),
+      };
+
+      const phaseMetrics: Array<{
+        metricName: MonitorMetricType;
+        value: number | undefined;
+        description: string;
+      }> = [
+        {
+          metricName: MonitorMetricType.PortDnsLookupTime,
+          value: portTimings.dnsLookupInMs,
+          description: "DNS lookup time for this Port monitor",
+        },
+        {
+          metricName: MonitorMetricType.PortTcpConnectTime,
+          value: portTimings.tcpConnectInMs,
+          description: "TCP connect time for this Port monitor",
+        },
+      ];
+
+      for (const phaseMetric of phaseMetrics) {
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: phaseMetric.metricName,
+          value: phaseMetric.value,
+          description: phaseMetric.description,
+          unit: "ms",
+          extraAttributes: extraAttributes,
+          metricRows: metricRows,
+          metricNameServiceNameMap: metricNameServiceNameMap,
+        });
+      }
+    }
+
     if ((data.dataToProcess as ProbeMonitorResponse).pingResponse) {
       const pingResponse: PingMonitorResponse = (
         data.dataToProcess as ProbeMonitorResponse

package/Server/Utils/Monitor/MonitorTemplateUtil.ts

@@ -424,6 +424,7 @@ export default class MonitorTemplateUtil {
           nameServers: domainResponse?.nameServers,
           domainStatus: domainResponse?.domainStatus,
           dnssec: domainResponse?.dnssec,
+          lookupMethod: domainResponse?.lookupMethod,
         } as JSONObject;
       }
 

package/Server/Utils/PasswordHash.ts

@@ -241,9 +241,12 @@ export default class PasswordHash {
    *   salted SHA   `SHA256("v2:" + salt + ":" + secret + ":" + password)`.
    *   bare SHA     `SHA256(secret + password)`, from before salts existed.
    *
-   * The last two are told apart by whether the row has a salt, which is the
-   * same rule they were written under. Anything not carrying a recognised
-   * scheme prefix is one of them.
+   * Normally the last two are told apart by whether the row has a salt,
+   * which is the rule they were written under. During a rolling deployment,
+   * however, an old pod can write a bare SHA digest while leaving a new pod's
+   * salt behind. Anything without a recognised scheme prefix therefore tries
+   * the selected legacy scheme first and the bare scheme as a compatibility
+   * fallback.
    */
   public static async verify(data: {
     plainValue: string;
@@ -259,11 +262,32 @@ export default class PasswordHash {
     );
 
     if (!parsed) {
+      const matchesSelectedLegacyScheme: boolean =
+        await HashedString.verifyValue({
+          plainValue: data.plainValue,
+          hashedValue: data.storedValue,
+          encryptionSecret: EncryptionSecret,
+          salt: data.salt || null,
+        });
+
+      if (matchesSelectedLegacyScheme || !data.salt) {
+        return matchesSelectedLegacyScheme;
+      }
+
+      /*
+       * Rolling-deploy compatibility. A pod running the pre-salt write path
+       * can replace a password with the original bare SHA-256 digest while a
+       * salt written by a newer pod remains on the row. A bare digest carries
+       * no scheme marker, so trying only the salted-SHA construction would
+       * lock that resource out permanently. Fall back to the original scheme
+       * when a supplied salt does not match; the successful caller upgrades
+       * the row to scrypt immediately afterwards.
+       */
       return await HashedString.verifyValue({
         plainValue: data.plainValue,
         hashedValue: data.storedValue,
         encryptionSecret: EncryptionSecret,
-        salt: data.salt || null,
+        salt: null,
       });
     }