@oneuptime/common 11.3.26 → 11.3.28

package/Server/Utils/AI/Toolbox/RecentChangesTools.ts

@@ -0,0 +1,208 @@
+import Monitor from "../../../../Models/DatabaseModels/Monitor";
+import MonitorStatusTimeline from "../../../../Models/DatabaseModels/MonitorStatusTimeline";
+import ScheduledMaintenance from "../../../../Models/DatabaseModels/ScheduledMaintenance";
+import TelemetryException from "../../../../Models/DatabaseModels/TelemetryException";
+import { JSONObject } from "../../../../Types/JSON";
+import Permission from "../../../../Types/Permission";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import MonitorStatusTimelineService from "../../../Services/MonitorStatusTimelineService";
+import ScheduledMaintenanceService from "../../../Services/ScheduledMaintenanceService";
+import TelemetryExceptionService from "../../../Services/TelemetryExceptionService";
+import QueryHelper from "../../../Types/Database/QueryHelper";
+import logger from "../../Logger";
+import ToolResultSerializer, { SerializedResult } from "./Serializer";
+import {
+  ObservabilityTool,
+  ToolArgs,
+  ToolContext,
+  ToolExecutionResult,
+  TimeRangeSchemaProperties,
+} from "./ToolTypes";
+
+/*
+ * Gate on Monitor read permissions. This is defense-in-depth only: every
+ * source query below runs under the requesting user's props, so a user who
+ * cannot see monitors / exceptions / maintenance simply gets no rows from that
+ * source — never another user's data. Resolved lazily to avoid the
+ * import-time circular-dependency TypeError other tools document.
+ */
+let cachedReadPermissions: Array<Permission> | null = null;
+const resolveReadPermissions: () => Array<Permission> =
+  (): Array<Permission> => {
+    if (!cachedReadPermissions) {
+      cachedReadPermissions = new Monitor().getReadPermissions();
+    }
+    return cachedReadPermissions;
+  };
+
+// One normalized entry in the merged "what changed" feed.
+interface ChangeEvent {
+  at: Date;
+  change: string;
+  detail: string;
+}
+
+export const RecentChangesTool: ObservabilityTool = {
+  name: "recent_changes",
+  description:
+    "Get a single chronological feed of what changed in a time window: newly-seen exceptions, monitor status changes (up/down), and scheduled maintenance overlapping the window. This is the highest-yield way to answer 'what changed right before this started?' during root-cause analysis — call it with the window around when a problem began.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      ...TimeRangeSchemaProperties,
+      limitPerSource: {
+        type: "number",
+        description:
+          "Maximum items to pull from each source (exceptions, monitor changes, maintenance) before merging. Default 15, max 30.",
+      },
+    },
+  },
+  get requiredPermissions(): Array<Permission> {
+    return resolveReadPermissions();
+  },
+  execute: async (
+    args: JSONObject,
+    ctx: ToolContext,
+  ): Promise<ToolExecutionResult> => {
+    const timeRange: { startTime: Date; endTime: Date } = ToolArgs.getTimeRange(
+      args,
+      { defaultHours: 24, maxDays: 30 },
+    );
+    const startTime: Date = timeRange.startTime;
+    const endTime: Date = timeRange.endTime;
+
+    const limitPerSource: number = ToolArgs.getNumber(args, "limitPerSource", {
+      defaultValue: 15,
+      min: 1,
+      max: 30,
+    });
+
+    const events: Array<ChangeEvent> = [];
+
+    // 1) Newly-seen exceptions — their first occurrence falls inside the window.
+    try {
+      const exceptions: Array<TelemetryException> =
+        await TelemetryExceptionService.findBy({
+          query: {
+            firstSeenAt: QueryHelper.inBetween(startTime, endTime),
+          } as never,
+          select: {
+            message: true,
+            exceptionType: true,
+            firstSeenAt: true,
+            occuranceCount: true,
+          },
+          sort: { firstSeenAt: SortOrder.Descending },
+          limit: limitPerSource,
+          skip: 0,
+          props: ctx.props,
+        });
+
+      for (const exception of exceptions) {
+        if (!exception.firstSeenAt) {
+          continue;
+        }
+        events.push({
+          at: exception.firstSeenAt,
+          change: "new_exception",
+          detail: `${exception.exceptionType || "Exception"}: ${exception.message || ""} (x${exception.occuranceCount ?? 0})`,
+        });
+      }
+    } catch (error) {
+      logger.debug(`recent_changes: exceptions source skipped: ${error}`);
+    }
+
+    // 2) Monitor status changes (up/down) inside the window.
+    try {
+      const statusChanges: Array<MonitorStatusTimeline> =
+        await MonitorStatusTimelineService.findBy({
+          query: {
+            createdAt: QueryHelper.inBetween(startTime, endTime),
+          },
+          select: {
+            createdAt: true,
+            monitor: { name: true },
+            monitorStatus: { name: true },
+          },
+          sort: { createdAt: SortOrder.Descending },
+          limit: limitPerSource,
+          skip: 0,
+          props: ctx.props,
+        });
+
+      for (const statusChange of statusChanges) {
+        if (!statusChange.createdAt) {
+          continue;
+        }
+        events.push({
+          at: statusChange.createdAt,
+          change: "monitor_status_change",
+          detail: `${statusChange.monitor?.name || "Monitor"} → ${statusChange.monitorStatus?.name || "unknown"}`,
+        });
+      }
+    } catch (error) {
+      logger.debug(`recent_changes: monitor source skipped: ${error}`);
+    }
+
+    /*
+     * 3) Scheduled maintenance overlapping the window (started before it ended,
+     *    ends after it started).
+     */
+    try {
+      const maintenanceEvents: Array<ScheduledMaintenance> =
+        await ScheduledMaintenanceService.findBy({
+          query: {
+            startsAt: QueryHelper.lessThanEqualTo(endTime),
+            endsAt: QueryHelper.greaterThanEqualTo(startTime),
+          },
+          select: {
+            title: true,
+            startsAt: true,
+            endsAt: true,
+            currentScheduledMaintenanceState: { name: true },
+          },
+          sort: { startsAt: SortOrder.Descending },
+          limit: limitPerSource,
+          skip: 0,
+          props: ctx.props,
+        });
+
+      for (const maintenance of maintenanceEvents) {
+        if (!maintenance.startsAt) {
+          continue;
+        }
+        events.push({
+          at: maintenance.startsAt,
+          change: "scheduled_maintenance",
+          detail: `${maintenance.title || "Maintenance"} (${maintenance.currentScheduledMaintenanceState?.name || "scheduled"})`,
+        });
+      }
+    } catch (error) {
+      logger.debug(`recent_changes: maintenance source skipped: ${error}`);
+    }
+
+    // Merge into one chronological feed, most recent first.
+    events.sort((a: ChangeEvent, b: ChangeEvent) => {
+      return b.at.getTime() - a.at.getTime();
+    });
+
+    const rows: Array<JSONObject> = events.map((event: ChangeEvent) => {
+      return {
+        at: event.at,
+        change: event.change,
+        detail: event.detail,
+      };
+    });
+
+    const serialized: SerializedResult =
+      ToolResultSerializer.serializeRows(rows);
+
+    return {
+      dataForLlm: serialized.text,
+      rowCount: serialized.rowCount,
+      citationLabel: `Changes ${startTime.toISOString()} → ${endTime.toISOString()} (${serialized.rowCount} events)`,
+      redactionCount: serialized.redactionCount,
+      isTruncated: serialized.isTruncated,
+    };
+  },
+};

package/Server/Utils/AI/Toolbox/Serializer.ts

@@ -32,6 +32,37 @@ const REDACTION_RULES: Array<RedactionRule> = [
     regex: /Bearer\s+[A-Za-z0-9._~+/-]{16,}=*/gi,
     replacement: "Bearer [redacted-token]",
   },
+  {
+    /*
+     * PEM private key blocks (RSA/EC/OPENSSH/PGP…). Match the whole
+     * BEGIN…END block, including newlines, so no key material leaks.
+     */
+    name: "private-key",
+    regex:
+      /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9]+ )*PRIVATE KEY-----/g,
+    replacement: "[redacted-private-key]",
+  },
+  {
+    name: "aws-access-key-id",
+    regex: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASCA)[0-9A-Z]{16}\b/g,
+    replacement: "[redacted-aws-key]",
+  },
+  {
+    name: "github-token",
+    regex:
+      /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b|\bgithub_pat_[A-Za-z0-9_]{22,}\b/g,
+    replacement: "[redacted-github-token]",
+  },
+  {
+    name: "slack-token",
+    regex: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g,
+    replacement: "[redacted-slack-token]",
+  },
+  {
+    name: "google-api-key",
+    regex: /\bAIza[0-9A-Za-z_-]{35}\b/g,
+    replacement: "[redacted-google-api-key]",
+  },
   {
     name: "email",
     regex: /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g,
@@ -42,6 +73,17 @@ const REDACTION_RULES: Array<RedactionRule> = [
     regex: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g,
     replacement: "[redacted-ip]",
   },
+  {
+    /*
+     * Payment card numbers in the classic space/dash-separated grouping
+     * (e.g. "4111 1111 1111 1111"). Deliberately NOT matching bare 16-digit
+     * runs: an all-numeric 16-char OTel span id would collide with those and
+     * break the search_logs → get_trace pivot.
+     */
+    name: "credit-card",
+    regex: /\b\d{4}[ -]\d{4}[ -]\d{4}[ -]\d{4}\b/g,
+    replacement: "[redacted-card]",
+  },
   {
     /*
      * Long hex blobs are usually secrets (SHA-256 digests, 40+ char API

package/Server/Utils/Database/MigrationFailureLog.ts

@@ -0,0 +1,240 @@
+import os from "os";
+import { DataSource, DataSourceOptions } from "typeorm";
+import logger from "../Logger";
+import { AppVersion } from "../../EnvironmentConfig";
+
+/*
+ * Records migrations that FAILED to apply into the "MigrationFailure" table so
+ * the admin health page can explain WHY a migration is pending (what error,
+ * when it ran) instead of only showing that it is behind. Every write here is
+ * best-effort: a diagnostic log must never itself crash a boot / migrate Job or
+ * mask the real migration error, so all failures inside this module are
+ * swallowed (logged, not thrown).
+ */
+
+// Which runner a failed migration belongs to. Stored verbatim in the table.
+export enum MigrationFailureType {
+  PostgresSchema = "PostgresSchema",
+  DataMigration = "DataMigration",
+}
+
+// Keep a single row bounded so a pathological error/stack can't bloat the table.
+const MAX_ERROR_MESSAGE_LENGTH: number = 10000;
+const MAX_ERROR_STACK_LENGTH: number = 20000;
+
+/*
+ * Best-effort credential scrub for the free-form error text we persist. DB /
+ * migration errors seldom carry secrets, but a connection error can echo a
+ * `scheme://user:pass@host` DSN and a failing DDL can quote a literal default,
+ * so we mask the obvious vectors before the text is stored and later shown.
+ */
+const CONNECTION_STRING_SECRET_PATTERN: RegExp =
+  /([a-z][a-z0-9+.-]*:\/\/[^:@\s/]+):[^\s/]+@/gi;
+const KEYWORD_SECRET_PATTERN: RegExp =
+  /(password|secret|token|api[_-]?key|apikey|credential|private[_-]?key|auth[_-]?token|access[_-]?key)(\s*[:=]\s*)('(?:[^']|'')*'|"[^"]*"|[^\s,)]+)/gi;
+
+export function scrubMigrationErrorText(text: string): string {
+  if (!text) {
+    return text;
+  }
+
+  return text
+    .replace(
+      CONNECTION_STRING_SECRET_PATTERN,
+      (_match: string, prefix: string): string => {
+        return `${prefix}:***REDACTED***@`;
+      },
+    )
+    .replace(
+      KEYWORD_SECRET_PATTERN,
+      (_match: string, keyword: string, separator: string): string => {
+        return `${keyword}${separator}***REDACTED***`;
+      },
+    );
+}
+
+function toErrorMessage(error: unknown): string {
+  if (error instanceof Error) {
+    return error.message || error.name || "Unknown error";
+  }
+  return String(error);
+}
+
+function toErrorStack(error: unknown): string | null {
+  return error instanceof Error && error.stack ? error.stack : null;
+}
+
+interface MigrationFailureInput {
+  migrationName: string;
+  migrationType: MigrationFailureType;
+  error: unknown;
+}
+
+/*
+ * Append one failed-attempt row using the provided (already-connected)
+ * DataSource. Raw SQL — not the Service layer — so it also works on a
+ * throwaway diagnostic connection and never depends on realtime/event wiring.
+ * Returns true if the row was written. If the "MigrationFailure" table does not
+ * exist yet (e.g. its own migration is still pending on a first rollout) the
+ * INSERT throws and is swallowed — the caller's real error is unaffected.
+ */
+export async function recordMigrationFailure(
+  dataSource: DataSource,
+  input: MigrationFailureInput,
+): Promise<boolean> {
+  try {
+    const message: string = scrubMigrationErrorText(
+      toErrorMessage(input.error),
+    ).substring(0, MAX_ERROR_MESSAGE_LENGTH);
+
+    const rawStack: string | null = toErrorStack(input.error);
+    const stack: string | null = rawStack
+      ? scrubMigrationErrorText(rawStack).substring(0, MAX_ERROR_STACK_LENGTH)
+      : null;
+
+    await dataSource.query(
+      `INSERT INTO "MigrationFailure"
+         ("migrationName", "migrationType", "errorMessage", "errorStack", "attemptedAt", "hostName", "appVersion", "version", "createdAt", "updatedAt")
+       VALUES ($1, $2, $3, $4, $5, $6, $7, 1, now(), now())`,
+      [
+        input.migrationName,
+        input.migrationType,
+        message,
+        stack,
+        new Date(),
+        os.hostname(),
+        AppVersion,
+      ],
+    );
+
+    return true;
+  } catch (err) {
+    // Diagnostic write must never mask the real failure — log and move on.
+    logger.warn(
+      `Could not record migration failure for "${input.migrationName}" (${input.migrationType}); continuing.`,
+    );
+    logger.warn(err);
+    return false;
+  }
+}
+
+// Trailing epoch in a migration class name — how TypeORM orders/identifies them.
+function parseMigrationTimestamp(name: string): number {
+  const match: RegExpMatchArray | null = name.match(/(\d{10,})$/);
+  return match ? Number(match[1]) : 0;
+}
+
+/*
+ * The TypeORM schema runner (migrationsRun) applies migrations in ascending
+ * timestamp order and halts at the first failure, so the FIRST defined
+ * migration not yet in the `migrations` table is exactly the one that failed.
+ * We recompute it here from a throwaway connection because the app's own
+ * connect() has already failed (its DataSource is not usable) at the point a
+ * schema migration throws.
+ */
+export async function recordSchemaMigrationFailureBestEffort(
+  dataSourceOptions: DataSourceOptions,
+  originalError: unknown,
+): Promise<void> {
+  /*
+   * Only THIS process's own boot runs schema migrations (migrationsRun=true —
+   * the migrate Job and single-pod / dev boots). On runtime pods where a
+   * dedicated migrate Job owns migrations (migrationsRun=false), a connect()
+   * failure is a connectivity problem, never a failed migration — recording
+   * one here would misattribute an outage to a (legitimately) pending
+   * migration, so skip.
+   */
+  if (
+    (dataSourceOptions as { migrationsRun?: boolean }).migrationsRun !== true
+  ) {
+    return;
+  }
+
+  let diagnosticDataSource: DataSource | null = null;
+
+  try {
+    diagnosticDataSource = new DataSource({
+      ...dataSourceOptions,
+      // Never re-run migrations or sync schema on the diagnostic connection.
+      migrationsRun: false,
+      synchronize: false,
+      // Keep the probe short so a genuinely-down DB doesn't slow the failing boot.
+      extra: {
+        ...((dataSourceOptions as { extra?: Record<string, unknown> }).extra ||
+          {}),
+        connectionTimeoutMillis: 5000,
+      },
+    } as DataSourceOptions);
+
+    /*
+     * If this fails, the original error was a connectivity problem (not a bad
+     * migration) — there is nothing meaningful to attribute, so we skip.
+     */
+    await diagnosticDataSource.initialize();
+
+    const definedMigrationClasses: Array<new () => { name: string }> =
+      Array.isArray(dataSourceOptions.migrations)
+        ? (dataSourceOptions.migrations as unknown as Array<
+            new () => { name: string }
+          >)
+        : [];
+
+    const definedNames: Array<string> = Array.from(
+      new Set(
+        definedMigrationClasses.map(
+          (MigrationClass: new () => { name: string }): string => {
+            return new MigrationClass().name;
+          },
+        ),
+      ),
+    ).sort((a: string, b: string): number => {
+      return parseMigrationTimestamp(a) - parseMigrationTimestamp(b);
+    });
+
+    let appliedNames: Set<string> = new Set();
+    try {
+      const rows: Array<{ name: string }> = await diagnosticDataSource.query(
+        "SELECT name FROM migrations",
+      );
+      appliedNames = new Set(
+        rows.map((row: { name: string }): string => {
+          return row.name;
+        }),
+      );
+    } catch {
+      // migrations table may not exist on a brand-new DB — treat as none applied.
+    }
+
+    const firstPending: string | undefined = definedNames.find(
+      (name: string): boolean => {
+        return !appliedNames.has(name);
+      },
+    );
+
+    /*
+     * No pending schema migration means the connect() failure was not a pending
+     * migration failing (e.g. an entity-metadata error) — don't misattribute it.
+     */
+    if (!firstPending) {
+      return;
+    }
+
+    await recordMigrationFailure(diagnosticDataSource, {
+      migrationName: firstPending,
+      migrationType: MigrationFailureType.PostgresSchema,
+      error: originalError,
+    });
+  } catch (err) {
+    // Best-effort diagnostics only — swallow everything.
+    logger.warn("Could not record Postgres schema migration failure.");
+    logger.warn(err);
+  } finally {
+    try {
+      if (diagnosticDataSource && diagnosticDataSource.isInitialized) {
+        await diagnosticDataSource.destroy();
+      }
+    } catch (destroyErr) {
+      logger.warn(destroyErr);
+    }
+  }
+}

package/Server/Utils/IncomingCallPhoneNumber.ts

@@ -0,0 +1,41 @@
+import { AppApiHostname } from "../EnvironmentConfig";
+import ClusterKeyAuthorization from "../Middleware/ClusterKeyAuthorization";
+import logger from "./Logger";
+import Protocol from "../../Types/API/Protocol";
+import Route from "../../Types/API/Route";
+import URL from "../../Types/API/URL";
+import API from "../../Utils/API";
+import ObjectID from "../../Types/ObjectID";
+
+/*
+ * Releases a provisioned incoming-call number on the call provider by delegating
+ * to the notification app's internal endpoint (the Common layer cannot reach the
+ * provider directly). Best-effort: failures are logged but never thrown, so a
+ * provider/network issue during cleanup does not block the parent delete.
+ */
+export default async function releaseIncomingCallPhoneNumber(data: {
+  projectCallSMSConfigId: ObjectID;
+  callProviderPhoneNumberId: string;
+}): Promise<void> {
+  try {
+    await API.post({
+      url: new URL(
+        Protocol.HTTP,
+        AppApiHostname,
+        new Route("/api/notification/phone-number/internal/release"),
+      ),
+      data: {
+        projectCallSMSConfigId: data.projectCallSMSConfigId.toString(),
+        callProviderPhoneNumberId: data.callProviderPhoneNumberId,
+      },
+      headers: {
+        ...ClusterKeyAuthorization.getClusterKeyHeaders(),
+      },
+    });
+  } catch (err) {
+    logger.error(
+      "Failed to release incoming-call phone number on the provider during cleanup:",
+    );
+    logger.error(err);
+  }
+}