@oneuptime/common 10.5.17 → 10.5.19

package/Server/API/TelemetryAPI.ts

@@ -236,12 +236,18 @@ const getAttributeValues: GetAttributeValuesFunction = async (
         ? (req.body["metricName"] as string)
         : undefined;
 
+    const searchText: string | undefined =
+      req.body["searchText"] && typeof req.body["searchText"] === "string"
+        ? (req.body["searchText"] as string)
+        : undefined;
+
     const values: string[] =
       await TelemetryAttributeService.fetchAttributeValues({
         projectId: databaseProps.tenantId,
         telemetryType,
         metricName,
         attributeKey,
+        searchText,
       });
 
     return Response.sendJsonObjectResponse(req, res, {

package/Server/EnvironmentConfig.ts

@@ -204,6 +204,33 @@ export const PostgresIdleTimeoutMs: number = parseInt(
   10,
 );
 
+/*
+ * TCP keepalive initial delay (ms) for Postgres sockets. When the client
+ * process dies ungracefully (SIGKILL, OOM, crash) or a network partition cuts
+ * the link, Postgres has no way to know the client is gone and the backend
+ * lingers as an orphaned connection — by default up to the OS
+ * tcp_keepalive_time (~2h on Linux). Enabling socket keepalive makes
+ * node-postgres probe the peer so dead connections are detected and torn down
+ * promptly.
+ */
+export const PostgresKeepAliveInitialDelayMs: number = parseInt(
+  process.env["DATABASE_KEEPALIVE_INITIAL_DELAY_MS"] || "10000",
+  10,
+);
+
+/*
+ * Postgres-side idle-session timeout (ms). Server-side backstop for orphaned
+ * connections: the server terminates any session that sits idle (outside a
+ * transaction) longer than this. MUST be larger than the pool's
+ * idleTimeoutMillis (PostgresIdleTimeoutMs) so the pool reaps its own healthy
+ * idle connections first and only truly-orphaned sessions (client gone) ever
+ * hit this. Set to 0 to disable. Requires Postgres 14+.
+ */
+export const PostgresIdleSessionTimeoutMs: number = parseInt(
+  process.env["DATABASE_IDLE_SESSION_TIMEOUT_MS"] || "300000",
+  10,
+);
+
 /*
  * TypeORM slow-query log threshold (ms). Any query exceeding this is
  * logged so we can find offenders in production without per-query

package/Server/Infrastructure/ClickhouseDatabase.ts

@@ -14,6 +14,7 @@ import HTTPErrorResponse from "../../Types/API/HTTPErrorResponse";
 import HTTPResponse from "../../Types/API/HTTPResponse";
 import { JSONObject } from "../../Types/JSON";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import GracefulShutdown, { ShutdownPriority } from "../Utils/GracefulShutdown";
 
 export type ClickhouseClient = ClickHouseClient;
 
@@ -21,6 +22,14 @@ export default class ClickhouseDatabase {
   private dataSource!: ClickhouseClient | null;
   private options: ClickHouseClientConfigOptions;
 
+  /*
+   * Each instance owns its own pool (App vs. Ingest), so each needs a
+   * distinct shutdown-handler name. The two instances share a database name,
+   * so a per-instance counter is what makes the names unique.
+   */
+  private static instanceCounter: number = 0;
+  private readonly instanceId: number = ++ClickhouseDatabase.instanceCounter;
+
   public constructor(
     options: ClickHouseClientConfigOptions = dataSourceOptions,
   ) {
@@ -97,7 +106,18 @@ export default class ClickhouseDatabase {
           }
         };
 
-      return await connectToDatabase();
+      const client: ClickhouseClient = await connectToDatabase();
+
+      // Close this Clickhouse pool on shutdown.
+      GracefulShutdown.registerHandler(
+        `ClickhouseDatabase#${this.instanceId}`,
+        ShutdownPriority.DataStores,
+        () => {
+          return this.disconnect();
+        },
+      );
+
+      return client;
     } catch (err) {
       logger.error("Clickhouse Database Connection Failed");
       logger.error(err);

package/Server/Infrastructure/Postgres/DataSourceOptions.ts

@@ -11,7 +11,9 @@ import {
   MaxPostgresConnections,
   PostgresConnectionAcquireTimeoutMs,
   PostgresIdleInTransactionTimeoutMs,
+  PostgresIdleSessionTimeoutMs,
   PostgresIdleTimeoutMs,
+  PostgresKeepAliveInitialDelayMs,
   PostgresQueryTimeoutMs,
   PostgresSlowQueryLogThresholdMs,
   PostgresStatementTimeoutMs,
@@ -54,6 +56,23 @@ const dataSourceOptions: DataSourceOptions = {
     statement_timeout: PostgresStatementTimeoutMs,
     query_timeout: PostgresQueryTimeoutMs,
     idle_in_transaction_session_timeout: PostgresIdleInTransactionTimeoutMs,
+    /*
+     * Detect dead TCP peers (ungraceful client exit / network partition) so
+     * orphaned server-side connections get torn down instead of lingering
+     * until the OS keepalive default (~2h).
+     */
+    keepAlive: true,
+    keepAliveInitialDelayMillis: PostgresKeepAliveInitialDelayMs,
+    /*
+     * Server-side backstop for orphaned idle sessions. node-postgres has no
+     * first-class option for this GUC, so pass it via the libpq `options`
+     * startup parameter. Unitless values are milliseconds. Only applied when
+     * > 0, and must exceed idleTimeoutMillis (see EnvironmentConfig) so the
+     * pool reaps healthy idle connections before the server force-closes them.
+     */
+    ...(PostgresIdleSessionTimeoutMs > 0
+      ? { options: `-c idle_session_timeout=${PostgresIdleSessionTimeoutMs}` }
+      : {}),
   },
   /*
    * Log any query slower than the configured threshold so we can find

package/Server/Infrastructure/PostgresDatabase.ts

@@ -4,6 +4,7 @@ import Sleep from "../../Types/Sleep";
 import { DataSource, DataSourceOptions } from "typeorm";
 import { createDatabase, dropDatabase } from "typeorm-extension";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import GracefulShutdown, { ShutdownPriority } from "../Utils/GracefulShutdown";
 
 export type DatabaseSourceOptions = DataSourceOptions;
 export type DatabaseSource = DataSource;
@@ -30,6 +31,15 @@ export default class Database {
 
   @CaptureSpan()
   public static async connect(): Promise<DataSource> {
+    /*
+     * Idempotent: a second connect() must not overwrite (and thereby orphan)
+     * the existing pool. Return the live DataSource instead of building a new
+     * one.
+     */
+    if (this.dataSource) {
+      return this.dataSource;
+    }
+
     let retry: number = 0;
 
     const dataSourceOptions: DataSourceOptions = this.getDatasourceOptions();
@@ -64,7 +74,23 @@ export default class Database {
           }
         };
 
-      return await connectToDatabase();
+      const dataSource: DataSource = await connectToDatabase();
+
+      /*
+       * Drain the pool on shutdown. Registered here (after a successful
+       * connect) so we never register cleanup for a pool that was never
+       * created, and — thanks to GracefulShutdown deduping by name — exactly
+       * once even if connect() is somehow reached twice.
+       */
+      GracefulShutdown.registerHandler(
+        "PostgresDatabase",
+        ShutdownPriority.DataStores,
+        () => {
+          return this.disconnect();
+        },
+      );
+
+      return dataSource;
     } catch (err) {
       logger.error("Postgres Database Connection Failed");
       logger.error(err);

package/Server/Infrastructure/QueueWorker.ts

@@ -15,6 +15,7 @@ import Telemetry, {
   SpanStatusCode,
 } from "../Utils/Telemetry";
 import Redis from "./Redis";
+import GracefulShutdown, { ShutdownPriority } from "../Utils/GracefulShutdown";
 
 export default class QueueWorker {
   @CaptureSpan()
@@ -116,9 +117,19 @@ export default class QueueWorker {
         : {}),
     });
 
-    process.on("SIGINT", async () => {
-      await worker.close();
-    });
+    /*
+     * Stop pulling new jobs and let in-flight ones finish on shutdown. Runs in
+     * the Workers tier — before datastores are drained — so jobs mid-flight can
+     * still reach Postgres / Redis. Replaces a SIGINT-only handler that never
+     * fired in containers (Kubernetes / docker stop send SIGTERM).
+     */
+    GracefulShutdown.registerHandler(
+      `QueueWorker:${queueName}`,
+      ShutdownPriority.Workers,
+      () => {
+        return worker.close();
+      },
+    );
 
     return worker;
   }

package/Server/Infrastructure/Redis.ts

@@ -15,6 +15,7 @@ import logger from "../Utils/Logger";
 import Sleep from "../../Types/Sleep";
 import { Redis as RedisClient, RedisOptions } from "ioredis";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import GracefulShutdown, { ShutdownPriority } from "../Utils/GracefulShutdown";
 
 export type ClientType = RedisClient;
 export type RedisOptionsType = RedisOptions;
@@ -122,6 +123,16 @@ export default abstract class Redis {
       logger.debug(
         `Redis connected on ${RedisHostname}:${RedisPort.toNumber()}`,
       );
+
+      // Close the Redis connection on shutdown.
+      GracefulShutdown.registerHandler(
+        "Redis",
+        ShutdownPriority.DataStores,
+        () => {
+          return this.disconnect();
+        },
+      );
+
       return this.client;
     } catch (err) {
       logger.error("Redis Connection Failed");

package/Server/Services/TelemetryAttributeService.ts

@@ -364,6 +364,7 @@ export class TelemetryAttributeService {
     telemetryType: TelemetryType;
     metricName?: string | undefined;
     attributeKey: string;
+    searchText?: string | undefined;
   }): Promise<string[]> {
     const source: TelemetrySource | null = this.getTelemetrySource(
       data.telemetryType,
@@ -378,15 +379,17 @@ export class TelemetryAttributeService {
       source,
       metricName: data.metricName,
       attributeKey: data.attributeKey,
+      searchText: data.searchText,
     });
   }
 
-  private static async fetchAttributeValuesFromDatabase(data: {
+  private static buildAttributeValuesStatement(data: {
     projectId: ObjectID;
     source: TelemetrySource;
     metricName?: string | undefined;
     attributeKey: string;
-  }): Promise<Array<string>> {
+    searchText?: string | undefined;
+  }): Statement {
     const lookbackStartDate: Date =
       TelemetryAttributeService.getLookbackStartDate();
 
@@ -419,6 +422,26 @@ export class TelemetryAttributeService {
       );
     }
 
+    /*
+     * Case-insensitive substring filter so the value autocomplete keeps
+     * narrowing server-side as the user types. Without it only the first
+     * ATTRIBUTE_VALUES_LIMIT values (alphabetically) are ever reachable,
+     * which hides matches on high-cardinality keys (host.name, url, ...).
+     * Mirrors the ILIKE idiom used for bodySearchText / nameSearchText.
+     */
+    if (data.searchText && data.searchText.trim().length > 0) {
+      statement.append(
+        SQL`
+        AND ${data.source.attributesColumn}[${{
+          type: TableColumnType.Text,
+          value: data.attributeKey,
+        }}] ILIKE ${{
+          type: TableColumnType.Text,
+          value: `%${data.searchText.trim()}%`,
+        }}`,
+      );
+    }
+
     statement.append(
       SQL`
       ORDER BY attributeValue ASC
@@ -428,6 +451,19 @@ export class TelemetryAttributeService {
       }}`,
     );
 
+    return statement;
+  }
+
+  private static async fetchAttributeValuesFromDatabase(data: {
+    projectId: ObjectID;
+    source: TelemetrySource;
+    metricName?: string | undefined;
+    attributeKey: string;
+    searchText?: string | undefined;
+  }): Promise<Array<string>> {
+    const statement: Statement =
+      TelemetryAttributeService.buildAttributeValuesStatement(data);
+
     const dbResult: Results = await data.source.service.executeQuery(statement);
     const response: DbJSONResponse = await dbResult.json<{
       data?: Array<JSONObject>;

package/Server/Utils/Express.ts

@@ -1,4 +1,5 @@
 import logger from "./Logger";
+import GracefulShutdown, { ShutdownPriority } from "./GracefulShutdown";
 import Dictionary from "../../Types/Dictionary";
 import GenericFunction from "../../Types/GenericFunction";
 import { JSONObject, JSONObjectOrArray } from "../../Types/JSON";
@@ -104,6 +105,37 @@ class Express {
       this.httpServer = createServer(this.app);
     }
 
+    /*
+     * On shutdown, stop accepting new connections first (before datastores are
+     * drained) so in-flight requests can finish but new ones don't acquire
+     * resources we're about to tear down. closeIdleConnections() drops idle
+     * keep-alive sockets so server.close() doesn't block waiting on them; the
+     * GracefulShutdown per-handler timeout bounds anything still in flight.
+     */
+    GracefulShutdown.registerHandler(
+      "HttpServer",
+      ShutdownPriority.HttpServer,
+      () => {
+        return new Promise<void>((resolve: () => void) => {
+          if (!this.httpServer || !this.httpServer.listening) {
+            resolve();
+            return;
+          }
+
+          const server: Server & { closeIdleConnections?: () => void } =
+            this.httpServer;
+
+          if (typeof server.closeIdleConnections === "function") {
+            server.closeIdleConnections();
+          }
+
+          server.close(() => {
+            resolve();
+          });
+        });
+      },
+    );
+
     type ResolveFunction = (app: express.Application) => void;
 
     return new Promise<express.Application>((resolve: ResolveFunction) => {

package/Server/Utils/GracefulShutdown.ts

@@ -0,0 +1,194 @@
+import logger from "./Logger";
+
+/*
+ * Centralized graceful-shutdown coordinator.
+ *
+ * Before this existed, each subsystem registered its own
+ * process.on("SIGTERM" | "SIGINT") handler independently. The telemetry
+ * handler in particular called process.exit(0) as soon as the OTEL SDK
+ * flushed, which raced every other handler and — crucially — meant the
+ * Postgres / Redis / Clickhouse pools were never drained. Connections were
+ * left for the OS socket teardown to reap (and leaked outright on SIGKILL or
+ * a network partition).
+ *
+ * Now every subsystem registers an async cleanup callback here, and this class
+ * is the single owner of the signal handlers and of process.exit. Handlers run
+ * in ascending priority order (lower first) so we stop accepting new work
+ * before tearing down the resources that work depends on:
+ *
+ *   HttpServer (10) -> stop accepting new HTTP requests
+ *   Workers    (20) -> stop pulling new queue jobs, finish in-flight jobs
+ *   Buffers    (30) -> flush in-memory write buffers to their datastore
+ *   DataStores (40) -> drain Postgres / Redis / Clickhouse pools
+ *   Telemetry  (50) -> flush traces / metrics / logs / profiles last
+ *
+ * Handlers in the same tier run concurrently. Each handler is bounded by a
+ * per-handler timeout, and the whole sequence by an overall deadline, so a
+ * single hung handler can never wedge the shutdown.
+ */
+export enum ShutdownPriority {
+  HttpServer = 10,
+  Workers = 20,
+  Buffers = 30,
+  DataStores = 40,
+  Telemetry = 50,
+}
+
+export type ShutdownCallback = () => Promise<void> | void;
+
+interface RegisteredShutdownHandler {
+  name: string;
+  priority: ShutdownPriority;
+  callback: ShutdownCallback;
+}
+
+export default class GracefulShutdown {
+  private static handlers: Array<RegisteredShutdownHandler> = [];
+  private static signalListenersInstalled: boolean = false;
+  private static isShuttingDown: boolean = false;
+
+  /*
+   * How long a single handler may run before we give up on it and move on.
+   * Kept comfortably under the orchestrator (Kubernetes) default
+   * terminationGracePeriodSeconds of 30s.
+   */
+  private static readonly perHandlerTimeoutMs: number = 10_000;
+
+  /*
+   * Hard ceiling for the entire shutdown. If we blow past this we force-exit
+   * rather than risk being SIGKILLed mid-cleanup.
+   */
+  private static readonly overallTimeoutMs: number = 25_000;
+
+  /*
+   * Register a cleanup callback to run on SIGTERM / SIGINT. Registering by a
+   * stable name is idempotent: a repeat registration (e.g. a second connect())
+   * replaces the previous callback instead of stacking a duplicate. Callers
+   * that own multiple independent resources (e.g. two Clickhouse pools) must
+   * therefore pass distinct names.
+   */
+  public static registerHandler(
+    name: string,
+    priority: ShutdownPriority,
+    callback: ShutdownCallback,
+  ): void {
+    const existingIndex: number = this.handlers.findIndex(
+      (handler: RegisteredShutdownHandler) => {
+        return handler.name === name;
+      },
+    );
+
+    if (existingIndex >= 0) {
+      this.handlers[existingIndex] = { name, priority, callback };
+    } else {
+      this.handlers.push({ name, priority, callback });
+    }
+
+    this.installSignalListeners();
+  }
+
+  private static installSignalListeners(): void {
+    if (this.signalListenersInstalled) {
+      return;
+    }
+    this.signalListenersInstalled = true;
+
+    process.on("SIGTERM", () => {
+      void this.shutdown("SIGTERM");
+    });
+    process.on("SIGINT", () => {
+      void this.shutdown("SIGINT");
+    });
+  }
+
+  public static async shutdown(signal: string): Promise<void> {
+    if (this.isShuttingDown) {
+      /*
+       * A second signal while we're already draining means the operator (or
+       * orchestrator) is impatient. Bail out immediately.
+       */
+      logger.warn(
+        `GracefulShutdown: received ${signal} while already shutting down. Forcing exit.`,
+      );
+      return process.exit(1);
+    }
+
+    this.isShuttingDown = true;
+    logger.info(
+      `GracefulShutdown: received ${signal}. Draining ${this.handlers.length} handler(s)...`,
+    );
+
+    const forceExitTimer: ReturnType<typeof setTimeout> = setTimeout(() => {
+      logger.error(
+        `GracefulShutdown: exceeded ${this.overallTimeoutMs}ms overall deadline. Forcing exit.`,
+      );
+      return process.exit(1);
+    }, this.overallTimeoutMs);
+
+    // Don't let this timer keep the event loop alive on its own.
+    forceExitTimer.unref();
+
+    // Run handlers tier by tier; lower priority tiers complete before the next.
+    const tiers: Array<number> = Array.from(
+      new Set(
+        this.handlers.map((handler: RegisteredShutdownHandler) => {
+          return handler.priority;
+        }),
+      ),
+    ).sort((a: number, b: number) => {
+      return a - b;
+    });
+
+    for (const tier of tiers) {
+      const handlersInTier: Array<RegisteredShutdownHandler> =
+        this.handlers.filter((handler: RegisteredShutdownHandler) => {
+          return handler.priority === tier;
+        });
+
+      // Handlers within a tier are independent, so run them concurrently.
+      await Promise.all(
+        handlersInTier.map((handler: RegisteredShutdownHandler) => {
+          return this.runHandlerWithTimeout(handler);
+        }),
+      );
+    }
+
+    clearTimeout(forceExitTimer);
+    logger.info("GracefulShutdown: all handlers complete. Exiting cleanly.");
+    return process.exit(0);
+  }
+
+  private static async runHandlerWithTimeout(
+    handler: RegisteredShutdownHandler,
+  ): Promise<void> {
+    logger.debug(`GracefulShutdown: running handler "${handler.name}"...`);
+
+    let timer: ReturnType<typeof setTimeout> | null = null;
+
+    const timeout: Promise<void> = new Promise<void>((resolve: () => void) => {
+      timer = setTimeout(() => {
+        logger.warn(
+          `GracefulShutdown: handler "${handler.name}" exceeded ${this.perHandlerTimeoutMs}ms. Moving on.`,
+        );
+        return resolve();
+      }, this.perHandlerTimeoutMs);
+      timer.unref();
+    });
+
+    const run: Promise<void> = (async (): Promise<void> => {
+      try {
+        await handler.callback();
+        logger.debug(`GracefulShutdown: handler "${handler.name}" done.`);
+      } catch (err) {
+        logger.error(`GracefulShutdown: handler "${handler.name}" failed:`);
+        logger.error(err);
+      }
+    })();
+
+    await Promise.race([run, timeout]);
+
+    if (timer) {
+      clearTimeout(timer);
+    }
+  }
+}

package/Server/Utils/Monitor/MonitorLogUtil.ts

@@ -2,6 +2,7 @@ import MonitorLogService from "../../Services/MonitorLogService";
 import GlobalConfigService from "../../Services/GlobalConfigService";
 import GlobalConfig from "../../../Models/DatabaseModels/GlobalConfig";
 import logger from "../Logger";
+import GracefulShutdown, { ShutdownPriority } from "../GracefulShutdown";
 import OneUptimeDate from "../../../Types/Date";
 import ObjectID from "../../../Types/ObjectID";
 import { JSONObject } from "../../../Types/JSON";
@@ -44,8 +45,9 @@ export default class MonitorLogUtil {
    * here until either MONITOR_LOG_FLUSH_BATCH_SIZE rows arrive
    * (size trigger) or MONITOR_LOG_FLUSH_INTERVAL_MS elapses since
    * the first row entered an empty buffer (time trigger),
-   * whichever comes first. On graceful shutdown the SIGTERM /
-   * SIGINT hook below drains the buffer before the process exits.
+   * whichever comes first. On graceful shutdown the registered
+   * GracefulShutdown handler below drains the buffer (in the
+   * Buffers tier, before the datastores are torn down).
    */
   private static buffer: Array<JSONObject> = [];
   private static flushTimer: NodeJS.Timeout | null = null;
@@ -233,10 +235,12 @@ export default class MonitorLogUtil {
   }
 
   /*
-   * Register SIGTERM / SIGINT handlers exactly once, lazily on
-   * first ingest. We avoid registering at module-load time so
-   * tooling that imports this file (e.g. migration runners,
-   * CLI scripts) doesn't end up with stray process listeners.
+   * Register the shutdown flush exactly once, lazily on first ingest. We avoid
+   * registering at module-load time so tooling that imports this file (e.g.
+   * migration runners, CLI scripts) doesn't end up holding a stray handler.
+   *
+   * Runs in the Buffers tier — ahead of the DataStores tier — so the buffer is
+   * drained to Clickhouse before the datastore pools are torn down.
    */
   private static ensureShutdownHooks(): void {
     if (this.shutdownHooksRegistered) {
@@ -244,16 +248,17 @@ export default class MonitorLogUtil {
     }
     this.shutdownHooksRegistered = true;
 
-    const flushOnShutdown: () => Promise<void> = async (): Promise<void> => {
-      try {
-        await this.flushAndWait();
-      } catch (err) {
-        logger.error("Error flushing MonitorLog buffer on shutdown:");
-        logger.error(err);
-      }
-    };
-
-    process.on("SIGTERM", flushOnShutdown);
-    process.on("SIGINT", flushOnShutdown);
+    GracefulShutdown.registerHandler(
+      "MonitorLogUtil",
+      ShutdownPriority.Buffers,
+      async (): Promise<void> => {
+        try {
+          await this.flushAndWait();
+        } catch (err) {
+          logger.error("Error flushing MonitorLog buffer on shutdown:");
+          logger.error(err);
+        }
+      },
+    );
   }
 }

package/Server/Utils/Profiling.ts

@@ -1,6 +1,7 @@
 import Pyroscope from "@pyroscope/nodejs";
 import { EnableProfiling } from "../EnvironmentConfig";
 import logger, { LogAttributes } from "./Logger";
+import GracefulShutdown, { ShutdownPriority } from "./GracefulShutdown";
 
 export default class Profiling {
   public static init(data: { serviceName: string }): void {
@@ -44,12 +45,19 @@ export default class Profiling {
       logger.error(err, profilingLogAttributes);
     }
 
-    process.on("SIGTERM", () => {
-      Pyroscope.stop().catch((err: unknown) => {
-        logger.error("Error stopping profiler:", profilingLogAttributes);
-        logger.error(err, profilingLogAttributes);
-      });
-    });
+    // Stop the profiler last (Telemetry tier), alongside the OTEL flush.
+    GracefulShutdown.registerHandler(
+      "Profiling",
+      ShutdownPriority.Telemetry,
+      async (): Promise<void> => {
+        try {
+          await Pyroscope.stop();
+        } catch (err) {
+          logger.error("Error stopping profiler:", profilingLogAttributes);
+          logger.error(err, profilingLogAttributes);
+        }
+      },
+    );
   }
 
   private static getServerAddress(): string | undefined {