@oneuptime/common 11.3.26 → 11.4.0

package/Server/Utils/Database/MigrationFailureLog.ts

@@ -0,0 +1,240 @@
+import os from "os";
+import { DataSource, DataSourceOptions } from "typeorm";
+import logger from "../Logger";
+import { AppVersion } from "../../EnvironmentConfig";
+
+/*
+ * Records migrations that FAILED to apply into the "MigrationFailure" table so
+ * the admin health page can explain WHY a migration is pending (what error,
+ * when it ran) instead of only showing that it is behind. Every write here is
+ * best-effort: a diagnostic log must never itself crash a boot / migrate Job or
+ * mask the real migration error, so all failures inside this module are
+ * swallowed (logged, not thrown).
+ */
+
+// Which runner a failed migration belongs to. Stored verbatim in the table.
+export enum MigrationFailureType {
+  PostgresSchema = "PostgresSchema",
+  DataMigration = "DataMigration",
+}
+
+// Keep a single row bounded so a pathological error/stack can't bloat the table.
+const MAX_ERROR_MESSAGE_LENGTH: number = 10000;
+const MAX_ERROR_STACK_LENGTH: number = 20000;
+
+/*
+ * Best-effort credential scrub for the free-form error text we persist. DB /
+ * migration errors seldom carry secrets, but a connection error can echo a
+ * `scheme://user:pass@host` DSN and a failing DDL can quote a literal default,
+ * so we mask the obvious vectors before the text is stored and later shown.
+ */
+const CONNECTION_STRING_SECRET_PATTERN: RegExp =
+  /([a-z][a-z0-9+.-]*:\/\/[^:@\s/]+):[^\s/]+@/gi;
+const KEYWORD_SECRET_PATTERN: RegExp =
+  /(password|secret|token|api[_-]?key|apikey|credential|private[_-]?key|auth[_-]?token|access[_-]?key)(\s*[:=]\s*)('(?:[^']|'')*'|"[^"]*"|[^\s,)]+)/gi;
+
+export function scrubMigrationErrorText(text: string): string {
+  if (!text) {
+    return text;
+  }
+
+  return text
+    .replace(
+      CONNECTION_STRING_SECRET_PATTERN,
+      (_match: string, prefix: string): string => {
+        return `${prefix}:***REDACTED***@`;
+      },
+    )
+    .replace(
+      KEYWORD_SECRET_PATTERN,
+      (_match: string, keyword: string, separator: string): string => {
+        return `${keyword}${separator}***REDACTED***`;
+      },
+    );
+}
+
+function toErrorMessage(error: unknown): string {
+  if (error instanceof Error) {
+    return error.message || error.name || "Unknown error";
+  }
+  return String(error);
+}
+
+function toErrorStack(error: unknown): string | null {
+  return error instanceof Error && error.stack ? error.stack : null;
+}
+
+interface MigrationFailureInput {
+  migrationName: string;
+  migrationType: MigrationFailureType;
+  error: unknown;
+}
+
+/*
+ * Append one failed-attempt row using the provided (already-connected)
+ * DataSource. Raw SQL — not the Service layer — so it also works on a
+ * throwaway diagnostic connection and never depends on realtime/event wiring.
+ * Returns true if the row was written. If the "MigrationFailure" table does not
+ * exist yet (e.g. its own migration is still pending on a first rollout) the
+ * INSERT throws and is swallowed — the caller's real error is unaffected.
+ */
+export async function recordMigrationFailure(
+  dataSource: DataSource,
+  input: MigrationFailureInput,
+): Promise<boolean> {
+  try {
+    const message: string = scrubMigrationErrorText(
+      toErrorMessage(input.error),
+    ).substring(0, MAX_ERROR_MESSAGE_LENGTH);
+
+    const rawStack: string | null = toErrorStack(input.error);
+    const stack: string | null = rawStack
+      ? scrubMigrationErrorText(rawStack).substring(0, MAX_ERROR_STACK_LENGTH)
+      : null;
+
+    await dataSource.query(
+      `INSERT INTO "MigrationFailure"
+         ("migrationName", "migrationType", "errorMessage", "errorStack", "attemptedAt", "hostName", "appVersion", "version", "createdAt", "updatedAt")
+       VALUES ($1, $2, $3, $4, $5, $6, $7, 1, now(), now())`,
+      [
+        input.migrationName,
+        input.migrationType,
+        message,
+        stack,
+        new Date(),
+        os.hostname(),
+        AppVersion,
+      ],
+    );
+
+    return true;
+  } catch (err) {
+    // Diagnostic write must never mask the real failure — log and move on.
+    logger.warn(
+      `Could not record migration failure for "${input.migrationName}" (${input.migrationType}); continuing.`,
+    );
+    logger.warn(err);
+    return false;
+  }
+}
+
+// Trailing epoch in a migration class name — how TypeORM orders/identifies them.
+function parseMigrationTimestamp(name: string): number {
+  const match: RegExpMatchArray | null = name.match(/(\d{10,})$/);
+  return match ? Number(match[1]) : 0;
+}
+
+/*
+ * The TypeORM schema runner (migrationsRun) applies migrations in ascending
+ * timestamp order and halts at the first failure, so the FIRST defined
+ * migration not yet in the `migrations` table is exactly the one that failed.
+ * We recompute it here from a throwaway connection because the app's own
+ * connect() has already failed (its DataSource is not usable) at the point a
+ * schema migration throws.
+ */
+export async function recordSchemaMigrationFailureBestEffort(
+  dataSourceOptions: DataSourceOptions,
+  originalError: unknown,
+): Promise<void> {
+  /*
+   * Only THIS process's own boot runs schema migrations (migrationsRun=true —
+   * the migrate Job and single-pod / dev boots). On runtime pods where a
+   * dedicated migrate Job owns migrations (migrationsRun=false), a connect()
+   * failure is a connectivity problem, never a failed migration — recording
+   * one here would misattribute an outage to a (legitimately) pending
+   * migration, so skip.
+   */
+  if (
+    (dataSourceOptions as { migrationsRun?: boolean }).migrationsRun !== true
+  ) {
+    return;
+  }
+
+  let diagnosticDataSource: DataSource | null = null;
+
+  try {
+    diagnosticDataSource = new DataSource({
+      ...dataSourceOptions,
+      // Never re-run migrations or sync schema on the diagnostic connection.
+      migrationsRun: false,
+      synchronize: false,
+      // Keep the probe short so a genuinely-down DB doesn't slow the failing boot.
+      extra: {
+        ...((dataSourceOptions as { extra?: Record<string, unknown> }).extra ||
+          {}),
+        connectionTimeoutMillis: 5000,
+      },
+    } as DataSourceOptions);
+
+    /*
+     * If this fails, the original error was a connectivity problem (not a bad
+     * migration) — there is nothing meaningful to attribute, so we skip.
+     */
+    await diagnosticDataSource.initialize();
+
+    const definedMigrationClasses: Array<new () => { name: string }> =
+      Array.isArray(dataSourceOptions.migrations)
+        ? (dataSourceOptions.migrations as unknown as Array<
+            new () => { name: string }
+          >)
+        : [];
+
+    const definedNames: Array<string> = Array.from(
+      new Set(
+        definedMigrationClasses.map(
+          (MigrationClass: new () => { name: string }): string => {
+            return new MigrationClass().name;
+          },
+        ),
+      ),
+    ).sort((a: string, b: string): number => {
+      return parseMigrationTimestamp(a) - parseMigrationTimestamp(b);
+    });
+
+    let appliedNames: Set<string> = new Set();
+    try {
+      const rows: Array<{ name: string }> = await diagnosticDataSource.query(
+        "SELECT name FROM migrations",
+      );
+      appliedNames = new Set(
+        rows.map((row: { name: string }): string => {
+          return row.name;
+        }),
+      );
+    } catch {
+      // migrations table may not exist on a brand-new DB — treat as none applied.
+    }
+
+    const firstPending: string | undefined = definedNames.find(
+      (name: string): boolean => {
+        return !appliedNames.has(name);
+      },
+    );
+
+    /*
+     * No pending schema migration means the connect() failure was not a pending
+     * migration failing (e.g. an entity-metadata error) — don't misattribute it.
+     */
+    if (!firstPending) {
+      return;
+    }
+
+    await recordMigrationFailure(diagnosticDataSource, {
+      migrationName: firstPending,
+      migrationType: MigrationFailureType.PostgresSchema,
+      error: originalError,
+    });
+  } catch (err) {
+    // Best-effort diagnostics only — swallow everything.
+    logger.warn("Could not record Postgres schema migration failure.");
+    logger.warn(err);
+  } finally {
+    try {
+      if (diagnosticDataSource && diagnosticDataSource.isInitialized) {
+        await diagnosticDataSource.destroy();
+      }
+    } catch (destroyErr) {
+      logger.warn(destroyErr);
+    }
+  }
+}

package/Server/Utils/IncomingCallPhoneNumber.ts

@@ -0,0 +1,41 @@
+import { AppApiHostname } from "../EnvironmentConfig";
+import ClusterKeyAuthorization from "../Middleware/ClusterKeyAuthorization";
+import logger from "./Logger";
+import Protocol from "../../Types/API/Protocol";
+import Route from "../../Types/API/Route";
+import URL from "../../Types/API/URL";
+import API from "../../Utils/API";
+import ObjectID from "../../Types/ObjectID";
+
+/*
+ * Releases a provisioned incoming-call number on the call provider by delegating
+ * to the notification app's internal endpoint (the Common layer cannot reach the
+ * provider directly). Best-effort: failures are logged but never thrown, so a
+ * provider/network issue during cleanup does not block the parent delete.
+ */
+export default async function releaseIncomingCallPhoneNumber(data: {
+  projectCallSMSConfigId: ObjectID;
+  callProviderPhoneNumberId: string;
+}): Promise<void> {
+  try {
+    await API.post({
+      url: new URL(
+        Protocol.HTTP,
+        AppApiHostname,
+        new Route("/api/notification/phone-number/internal/release"),
+      ),
+      data: {
+        projectCallSMSConfigId: data.projectCallSMSConfigId.toString(),
+        callProviderPhoneNumberId: data.callProviderPhoneNumberId,
+      },
+      headers: {
+        ...ClusterKeyAuthorization.getClusterKeyHeaders(),
+      },
+    });
+  } catch (err) {
+    logger.error(
+      "Failed to release incoming-call phone number on the provider during cleanup:",
+    );
+    logger.error(err);
+  }
+}

package/Server/Utils/LLM/LLMService.ts

@@ -49,6 +49,13 @@ export interface LLMUsage {
   promptTokens: number;
   completionTokens: number;
   totalTokens: number;
+  /*
+   * Prompt-caching breakdown, when the provider reports it. cachedInputTokens
+   * are input tokens served from cache (billed at a large discount);
+   * cacheCreationTokens are input tokens written to the cache on this call.
+   */
+  cachedInputTokens?: number | undefined;
+  cacheCreationTokens?: number | undefined;
 }
 
 export interface LLMCompletionResponse {
@@ -237,6 +244,17 @@ export default class LLMService {
     const toolCalls: Array<LLMToolCall> | undefined =
       this.parseOpenAIToolCalls(message);
 
+    /*
+     * OpenAI (and Azure OpenAI) automatically cache a stable prompt prefix
+     * once it is long enough and report the cache hit under
+     * prompt_tokens_details.cached_tokens — surface it for cost visibility.
+     */
+    const cachedTokens: number | undefined = usage
+      ? ((usage["prompt_tokens_details"] as JSONObject | undefined)?.[
+          "cached_tokens"
+        ] as number | undefined)
+      : undefined;
+
     return {
       content: (message["content"] as string) || "",
       toolCalls: toolCalls,
@@ -246,6 +264,7 @@ export default class LLMService {
             promptTokens: usage["prompt_tokens"] as number,
             completionTokens: usage["completion_tokens"] as number,
             totalTokens: usage["total_tokens"] as number,
+            cachedInputTokens: cachedTokens || undefined,
           }
         : undefined,
     };
@@ -565,6 +584,32 @@ export default class LLMService {
         continue;
       }
 
+      /*
+       * Anthropic requires strictly alternating user/assistant turns and
+       * returns a 400 on two consecutive same-role messages. The agent loop
+       * can legitimately emit back-to-back user turns (e.g. a tool_result
+       * user message immediately followed by the "budget exhausted, answer
+       * now" nudge), so coalesce a run of same-role messages into one instead
+       * of failing the whole request.
+       */
+      const previousMessage: JSONObject | undefined =
+        anthropicMessages[anthropicMessages.length - 1];
+
+      if (previousMessage && previousMessage["role"] === msg.role) {
+        if (typeof previousMessage["content"] === "string") {
+          previousMessage["content"] =
+            `${previousMessage["content"] as string}\n\n${msg.content}`;
+          continue;
+        }
+        if (Array.isArray(previousMessage["content"])) {
+          (previousMessage["content"] as Array<JSONObject>).push({
+            type: "text",
+            text: msg.content,
+          });
+          continue;
+        }
+      }
+
       anthropicMessages.push({
         role: msg.role,
         content: msg.content,
@@ -602,18 +647,43 @@ export default class LLMService {
       max_tokens: request.maxTokens || LLMService.ANTHROPIC_DEFAULT_MAX_TOKENS,
     };
 
+    /*
+     * Prompt caching. The system prompt and tool definitions are the large,
+     * stable prefix re-sent on every turn of the agent loop, so caching them is
+     * the biggest single cost + latency lever (cached input is billed at ~10%).
+     * An ephemeral cache_control breakpoint on the system block and on the LAST
+     * tool caches the whole system + tools prefix. cache_control is GA under
+     * anthropic-version 2023-06-01, so no beta header is required.
+     */
     if (systemMessage) {
-      requestData["system"] = systemMessage;
+      requestData["system"] = [
+        {
+          type: "text",
+          text: systemMessage,
+          cache_control: { type: "ephemeral" },
+        },
+      ];
     }
 
     if (request.tools && request.tools.length > 0) {
-      requestData["tools"] = request.tools.map((tool: LLMToolDefinition) => {
-        return {
-          name: tool.name,
-          description: tool.description,
-          input_schema: tool.inputSchema,
-        };
-      });
+      const anthropicTools: Array<JSONObject> = request.tools.map(
+        (tool: LLMToolDefinition) => {
+          return {
+            name: tool.name,
+            description: tool.description,
+            input_schema: tool.inputSchema,
+          };
+        },
+      );
+
+      // A breakpoint on the last tool caches the entire tools block before it.
+      const lastTool: JSONObject | undefined =
+        anthropicTools[anthropicTools.length - 1];
+      if (lastTool) {
+        lastTool["cache_control"] = { type: "ephemeral" };
+      }
+
+      requestData["tools"] = anthropicTools;
     }
 
     const response: HTTPErrorResponse | HTTPResponse<JSONObject> =
@@ -685,10 +755,21 @@ export default class LLMService {
       stopReason: jsonData["stop_reason"] === "tool_use" ? "tool_use" : "stop",
       usage: usage
         ? {
-            promptTokens: usage["input_tokens"] as number,
-            completionTokens: usage["output_tokens"] as number,
+            promptTokens: (usage["input_tokens"] as number) || 0,
+            completionTokens: (usage["output_tokens"] as number) || 0,
+            /*
+             * Anthropic reports cache-read and cache-creation input tokens
+             * separately from input_tokens. Surface them and fold them into
+             * totalTokens so the logged total reflects the full billable input.
+             */
+            cachedInputTokens:
+              (usage["cache_read_input_tokens"] as number) || undefined,
+            cacheCreationTokens:
+              (usage["cache_creation_input_tokens"] as number) || undefined,
             totalTokens:
               ((usage["input_tokens"] as number) || 0) +
+              ((usage["cache_read_input_tokens"] as number) || 0) +
+              ((usage["cache_creation_input_tokens"] as number) || 0) +
               ((usage["output_tokens"] as number) || 0),
           }
         : undefined,
@@ -812,11 +893,29 @@ export default class LLMService {
       });
     }
 
+    /*
+     * Ollama reports token counts on the final /api/chat response as
+     * prompt_eval_count (input) and eval_count (output). Populate usage from
+     * them so LlmLog, the AI dashboards and (costed self-hosted Ollama)
+     * billing are not silently blind to token spend.
+     */
+    const ollamaPromptTokens: number =
+      (jsonData["prompt_eval_count"] as number) || 0;
+    const ollamaCompletionTokens: number =
+      (jsonData["eval_count"] as number) || 0;
+
     return {
       content: (message["content"] as string) || "",
       toolCalls: toolCalls,
       stopReason: toolCalls && toolCalls.length > 0 ? "tool_use" : "stop",
-      usage: undefined, // Ollama doesn't provide token usage in the same way
+      usage:
+        ollamaPromptTokens || ollamaCompletionTokens
+          ? {
+              promptTokens: ollamaPromptTokens,
+              completionTokens: ollamaCompletionTokens,
+              totalTokens: ollamaPromptTokens + ollamaCompletionTokens,
+            }
+          : undefined,
     };
   }
 }

package/Server/Utils/Telemetry/IoTSnapshotScan.ts

@@ -141,31 +141,34 @@ function parseUnixNanoToDate(
   value: string | number | undefined,
   context: string,
 ): Date {
-  let numericValue: number = OneUptimeDate.getCurrentDateAsUnixNano();
-
   if (value !== undefined && value !== null) {
     try {
       if (typeof value === "string") {
-        const parsed: number = Number.parseFloat(value);
-        if (isNaN(parsed)) {
+        const trimmed: string = value.trim();
+        if (trimmed === "" || isNaN(Number(trimmed))) {
           throw new Error(`Invalid timestamp string: ${value}`);
         }
-        numericValue = parsed;
-      } else if (typeof value === "number") {
+        /*
+         * Hand the raw nanosecond string to fromUnixNano so it can divide
+         * with BigInt — nanosecond epochs exceed Number.MAX_SAFE_INTEGER, so
+         * parsing to a float here would drop up to a full millisecond.
+         */
+        return OneUptimeDate.fromUnixNano(trimmed);
+      }
+      if (typeof value === "number") {
         if (!Number.isFinite(value)) {
           throw new Error(`Invalid timestamp number: ${value}`);
         }
-        numericValue = value;
+        return OneUptimeDate.fromUnixNano(value);
       }
     } catch (error) {
       logger.warn(
         `Error processing ${context}: ${error instanceof Error ? error.message : String(error)}, using current time`,
       );
-      numericValue = OneUptimeDate.getCurrentDateAsUnixNano();
     }
   }
 
-  return OneUptimeDate.fromUnixNano(numericValue);
+  return OneUptimeDate.getCurrentDate();
 }
 
 // Same trim-or-null read contract as OtelIngestBaseService.getStringAttribute.

package/Server/Utils/Telemetry/ProxmoxCephSnapshotScan.ts

@@ -242,31 +242,34 @@ function parseUnixNanoToDate(
   value: string | number | undefined,
   context: string,
 ): Date {
-  let numericValue: number = OneUptimeDate.getCurrentDateAsUnixNano();
-
   if (value !== undefined && value !== null) {
     try {
       if (typeof value === "string") {
-        const parsed: number = Number.parseFloat(value);
-        if (isNaN(parsed)) {
+        const trimmed: string = value.trim();
+        if (trimmed === "" || isNaN(Number(trimmed))) {
           throw new Error(`Invalid timestamp string: ${value}`);
         }
-        numericValue = parsed;
-      } else if (typeof value === "number") {
+        /*
+         * Hand the raw nanosecond string to fromUnixNano so it can divide
+         * with BigInt — nanosecond epochs exceed Number.MAX_SAFE_INTEGER, so
+         * parsing to a float here would drop up to a full millisecond.
+         */
+        return OneUptimeDate.fromUnixNano(trimmed);
+      }
+      if (typeof value === "number") {
         if (!Number.isFinite(value)) {
           throw new Error(`Invalid timestamp number: ${value}`);
         }
-        numericValue = value;
+        return OneUptimeDate.fromUnixNano(value);
       }
     } catch (error) {
       logger.warn(
         `Error processing ${context}: ${error instanceof Error ? error.message : String(error)}, using current time`,
       );
-      numericValue = OneUptimeDate.getCurrentDateAsUnixNano();
     }
   }
 
-  return OneUptimeDate.fromUnixNano(numericValue);
+  return OneUptimeDate.getCurrentDate();
 }
 
 // Same trim-or-null read contract as OtelIngestBaseService.getStringAttribute.

package/Tests/Server/API/BaseAPI.test.ts

@@ -103,6 +103,7 @@ jest.mock("../../../Server/Services/ProjectService", () => {
 
 jest.mock("../../../Server/EnvironmentConfig", () => {
   return {
+    ...jest.requireActual("../../../Server/EnvironmentConfig"),
     IsBillingEnabled: true,
     LogLevel: "INFO", // Use string literal instead of ConfigLogLevel.INFO
     DisableTelemetry: true,

package/Tests/Server/Services/AnalyticsDatabaseService.test.ts

@@ -234,14 +234,6 @@ describe("AnalyticsDatabaseService", () => {
       jest.spyOn(logger, "error").mockImplementation(() => {
         return undefined!;
       });
-
-      /*
-       * The base getException method is async and throws without await in the
-       * catch block, creating unhandled rejections. Override to throw synchronously.
-       */
-      (service as any).getException = (error: Error): never => {
-        throw error;
-      };
     });
 
     afterEach(() => {