@oneuptime/common 12.0.4 → 12.0.6

package/Server/Utils/DataSource/SqlResultShaper.ts

@@ -0,0 +1,372 @@
+import AggregatedResult from "../../../Types/BaseDatabase/AggregatedResult";
+import AggregatedModel from "../../../Types/BaseDatabase/AggregatedModel";
+import DataSourceTableResult, {
+  DataSourceTableColumn,
+  DataSourceTableColumnType,
+} from "../../../Types/DataSource/DataSourceTableResult";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import { JSONValue } from "../../../Types/JSON";
+
+/*
+ * Shared row → chart/table shaping for every SQL-family connector
+ * (PostgreSQL, MySQL, SQL Server, ClickHouse). One implementation means one
+ * behavior: the same query against any engine renders the same chart.
+ *
+ * Time-series convention (Grafana-compatible):
+ *  - a TIME column named time / timestamp / date / ts (case-insensitive);
+ *  - a numeric VALUE column named value (or the first numeric non-time
+ *    column);
+ *  - every OTHER column becomes a series label — rows sharing the same
+ *    label values merge into one series. The chart layer groups on the
+ *    `attributes` object of each data point.
+ */
+
+const TIME_COLUMN_NAMES: Array<string> = ["time", "timestamp", "date", "ts"];
+const VALUE_COLUMN_NAME: string = "value";
+
+export interface ShapedRows {
+  rows: Array<Record<string, unknown>>;
+  truncated: boolean;
+}
+
+/*
+ * A 9-19 digit unsigned integer string — an epoch in seconds through
+ * nanoseconds. Shorter numeric strings (bare years, YYYYMMDD) stay on the
+ * Date-parse path.
+ */
+const NUMERIC_EPOCH_STRING_REGEX: RegExp = /^\d{9,19}$/;
+
+export default class SqlResultShaper {
+  /*
+   * Clamp raw driver rows to the cap. Connectors fetch cap+1 so the shaper
+   * can tell "exactly cap" apart from "more than cap".
+   */
+  public static clampRows(
+    rows: Array<Record<string, unknown>>,
+    maxRows: number,
+  ): ShapedRows {
+    if (rows.length > maxRows) {
+      return { rows: rows.slice(0, maxRows), truncated: true };
+    }
+    return { rows: rows, truncated: false };
+  }
+
+  /*
+   * Copy of the probe SqlMonitor's cell coercion semantics: keep JSON-safe
+   * primitives, ISO-ify dates, never ship raw binary.
+   */
+  public static coerceCell(value: unknown): JSONValue {
+    if (value === null || value === undefined) {
+      return null;
+    }
+    if (
+      typeof value === "number" ||
+      typeof value === "boolean" ||
+      typeof value === "string"
+    ) {
+      return value;
+    }
+    if (value instanceof Date) {
+      return value.toISOString();
+    }
+    if (Buffer.isBuffer(value)) {
+      return "[binary]";
+    }
+    if (typeof value === "bigint") {
+      return value.toString();
+    }
+    try {
+      return JSON.stringify(value);
+    } catch {
+      return String(value);
+    }
+  }
+
+  private static findTimeColumn(
+    columnNames: Array<string>,
+  ): string | undefined {
+    return columnNames.find((name: string) => {
+      return TIME_COLUMN_NAMES.includes(name.toLowerCase());
+    });
+  }
+
+  private static findValueColumn(
+    columnNames: Array<string>,
+    timeColumn: string,
+    firstRow: Record<string, unknown>,
+  ): string | undefined {
+    const explicit: string | undefined = columnNames.find((name: string) => {
+      return name.toLowerCase() === VALUE_COLUMN_NAME;
+    });
+    if (explicit) {
+      return explicit;
+    }
+    // First numeric non-time column.
+    return columnNames.find((name: string) => {
+      if (name === timeColumn) {
+        return false;
+      }
+      const cell: unknown = firstRow[name];
+      return (
+        typeof cell === "number" ||
+        typeof cell === "bigint" ||
+        (typeof cell === "string" &&
+          cell.trim() !== "" &&
+          Number.isFinite(Number(cell)))
+      );
+    });
+  }
+
+  /*
+   * Parse a SQL time cell: Date object, ISO-ish string, or epoch number
+   * (ms when >= 1e12, seconds when >= 1e9). Returns null when
+   * unparseable.
+   */
+  public static coerceTimestamp(value: unknown): Date | null {
+    if (value instanceof Date) {
+      return isNaN(value.getTime()) ? null : value;
+    }
+    if (typeof value === "number" || typeof value === "bigint") {
+      const numeric: number = Number(value);
+      if (!Number.isFinite(numeric) || numeric <= 0) {
+        return null;
+      }
+      if (numeric >= 1e12) {
+        return new Date(numeric);
+      }
+      if (numeric >= 1e9) {
+        return new Date(numeric * 1000);
+      }
+      return null;
+    }
+    if (typeof value === "string" && value.trim() !== "") {
+      const trimmed: string = value.trim();
+      /*
+       * Numeric-string epochs: ClickHouse's JSON format serializes
+       * 64-bit integers as strings, and REST payloads routinely quote
+       * epoch numbers — route them through the same epoch heuristic as
+       * native numbers instead of letting new Date('1700000000000')
+       * produce Invalid Date and silently drop the row. 9+ digits only,
+       * so short numeric strings (bare years) keep the Date-parse path.
+       */
+      if (NUMERIC_EPOCH_STRING_REGEX.test(trimmed)) {
+        return this.coerceTimestamp(Number(trimmed));
+      }
+      /*
+       * Engines commonly return 'YYYY-MM-DD HH:MM:SS' with no zone —
+       * treat it as UTC (the macros are UTC, and Date.parse would
+       * otherwise apply the server's local zone).
+       */
+      const spaceSeparated: RegExpMatchArray | null = trimmed.match(
+        /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}(?:\.\d+)?)$/,
+      );
+      const normalized: string = spaceSeparated
+        ? `${spaceSeparated[1]}T${spaceSeparated[2]}Z`
+        : trimmed;
+      const parsed: Date = new Date(normalized);
+      return isNaN(parsed.getTime()) ? null : parsed;
+    }
+    return null;
+  }
+
+  private static coerceValue(value: unknown): number | null {
+    if (typeof value === "number") {
+      return Number.isFinite(value) ? value : null;
+    }
+    if (typeof value === "bigint") {
+      return Number(value);
+    }
+    if (typeof value === "string" && value.trim() !== "") {
+      const numeric: number = Number(value);
+      return Number.isFinite(numeric) ? numeric : null;
+    }
+    return null;
+  }
+
+  public static rowsToTimeSeries(
+    rows: Array<Record<string, unknown>>,
+    options: { maxRows: number },
+  ): AggregatedResult {
+    const clamped: ShapedRows = this.clampRows(rows, options.maxRows);
+
+    if (clamped.rows.length === 0) {
+      return { data: [], truncated: false };
+    }
+
+    const firstRow: Record<string, unknown> = clamped.rows[0]!;
+    const columnNames: Array<string> = Object.keys(firstRow);
+
+    const timeColumn: string | undefined = this.findTimeColumn(columnNames);
+    if (!timeColumn) {
+      throw new BadDataException(
+        `Time-series queries must return a time column named one of: ${TIME_COLUMN_NAMES.join(
+          ", ",
+        )}. Got columns: ${columnNames.join(", ")}. Use the Table widget for non-time-series results.`,
+      );
+    }
+
+    const valueColumn: string | undefined = this.findValueColumn(
+      columnNames,
+      timeColumn,
+      firstRow,
+    );
+    if (!valueColumn) {
+      throw new BadDataException(
+        `Time-series queries must return a numeric column (name it "value"). Got columns: ${columnNames.join(
+          ", ",
+        )}.`,
+      );
+    }
+
+    const labelColumns: Array<string> = columnNames.filter((name: string) => {
+      return name !== timeColumn && name !== valueColumn;
+    });
+
+    const data: Array<AggregatedModel> = [];
+    for (const row of clamped.rows) {
+      const timestamp: Date | null = this.coerceTimestamp(row[timeColumn]);
+      const value: number | null = this.coerceValue(row[valueColumn]);
+      if (timestamp === null || value === null) {
+        continue; // skip unplottable rows rather than failing the chart.
+      }
+
+      const point: AggregatedModel = {
+        timestamp: timestamp,
+        value: value,
+      } as AggregatedModel;
+
+      if (labelColumns.length > 0) {
+        const attributes: Record<string, string> = {};
+        for (const labelColumn of labelColumns) {
+          const cell: JSONValue = this.coerceCell(row[labelColumn]);
+          attributes[labelColumn] = cell === null ? "(unset)" : String(cell);
+        }
+        (point as Record<string, unknown>)["attributes"] = attributes;
+      }
+
+      data.push(point);
+    }
+
+    return { data: data, truncated: clamped.truncated };
+  }
+
+  private static sniffColumnType(value: unknown): DataSourceTableColumnType {
+    if (typeof value === "number" || typeof value === "bigint") {
+      return DataSourceTableColumnType.Number;
+    }
+    if (typeof value === "boolean") {
+      return DataSourceTableColumnType.Boolean;
+    }
+    if (value instanceof Date) {
+      return DataSourceTableColumnType.Date;
+    }
+    if (value !== null && typeof value === "object") {
+      return DataSourceTableColumnType.Json;
+    }
+    return DataSourceTableColumnType.Text;
+  }
+
+  public static rowsToTable(
+    rows: Array<Record<string, unknown>>,
+    options: { maxRows: number },
+  ): DataSourceTableResult {
+    const clamped: ShapedRows = this.clampRows(rows, options.maxRows);
+
+    if (clamped.rows.length === 0) {
+      return { columns: [], rows: [], truncated: false };
+    }
+
+    /*
+     * Column set = union across rows (drivers can omit NULL columns in
+     * some row formats); type sniffed from the first non-null cell.
+     */
+    const columnNames: Array<string> = [];
+    for (const row of clamped.rows) {
+      for (const name of Object.keys(row)) {
+        if (!columnNames.includes(name)) {
+          columnNames.push(name);
+        }
+      }
+    }
+
+    const columns: Array<DataSourceTableColumn> = columnNames.map(
+      (name: string) => {
+        const sampleRow: Record<string, unknown> | undefined =
+          clamped.rows.find((row: Record<string, unknown>) => {
+            return row[name] !== null && row[name] !== undefined;
+          });
+        return {
+          key: name,
+          title: name,
+          type: this.sniffColumnType(sampleRow ? sampleRow[name] : null),
+        };
+      },
+    );
+
+    const shapedRows: Array<{ [key: string]: JSONValue }> = clamped.rows.map(
+      (row: Record<string, unknown>) => {
+        const shaped: { [key: string]: JSONValue } = {};
+        for (const name of columnNames) {
+          shaped[name] = this.coerceCell(row[name]);
+        }
+        return shaped;
+      },
+    );
+
+    return {
+      columns: columns,
+      rows: shapedRows,
+      truncated: clamped.truncated,
+    };
+  }
+
+  /*
+   * Redact credentials and connection internals from driver errors before
+   * they reach API responses (probe SqlMonitor semantics).
+   */
+  /*
+   * Every user-supplied credential on a connection-settings object,
+   * including custom header VALUES (write-only in the model, so they can
+   * carry tokens) — the standard secret list for sanitizeError so no
+   * connector forgets one.
+   */
+  public static collectSecrets(settings: {
+    password?: string | undefined;
+    apiToken?: string | undefined;
+    username?: string | undefined;
+    databaseHost?: string | undefined;
+    customHeaders?: Record<string, string> | undefined;
+  }): Array<string | undefined> {
+    return [
+      settings.password,
+      settings.apiToken,
+      settings.username,
+      settings.databaseHost,
+      ...Object.values(settings.customHeaders || {}),
+    ];
+  }
+
+  public static sanitizeError(
+    error: unknown,
+    secrets: Array<string | undefined>,
+  ): string {
+    let message: string =
+      error instanceof Error ? error.message : String(error);
+
+    for (const secret of secrets) {
+      if (secret && secret.length >= 4) {
+        message = message.split(secret).join("***");
+      }
+    }
+
+    // DSNs with embedded credentials.
+    message = message.replace(/[a-zA-Z]+:\/\/[^\s]*@[^\s]*/g, "[redacted-dsn]");
+    // Connection-string password pairs.
+    message = message.replace(
+      /(password|pwd)\s*[=:]\s*(\{[^}]*\}|"[^"]*"|'[^']*'|[^;\r\n]+)/gi,
+      "$1=***",
+    );
+
+    return message;
+  }
+}

package/Server/Utils/DataSource/Types.ts

@@ -0,0 +1,93 @@
+import AggregatedResult from "../../../Types/BaseDatabase/AggregatedResult";
+import DataSourceTableResult from "../../../Types/DataSource/DataSourceTableResult";
+import DataSourceType from "../../../Types/DataSource/DataSourceType";
+import Dictionary from "../../../Types/Dictionary";
+import { JSONObject } from "../../../Types/JSON";
+
+/*
+ * Decrypted, server-side-only connection settings for one Data Source.
+ * Built by DataSourceService from the DataSource record (secret columns
+ * decrypted with root props) and handed straight to a connector — this
+ * object must NEVER be serialized into an API response or a log line.
+ */
+export interface DataSourceConnectionSettings {
+  dataSourceType: DataSourceType;
+
+  // HTTP-based sources (Prometheus, Loki, Elasticsearch, REST API).
+  url?: string | undefined;
+
+  // Database sources (PostgreSQL, MySQL, SQL Server, ClickHouse).
+  databaseHost?: string | undefined;
+  databasePort?: number | undefined;
+  databaseName?: string | undefined;
+
+  /*
+   * Credentials. `username`+`password` double as HTTP basic auth for
+   * HTTP-based sources; `apiToken` is sent as a Bearer token (Prometheus /
+   * Loki / Elasticsearch behind auth proxies, REST APIs).
+   */
+  username?: string | undefined;
+  password?: string | undefined;
+  apiToken?: string | undefined;
+
+  // Extra HTTP headers (REST APIs, auth proxies). May contain secrets.
+  customHeaders?: Dictionary<string> | undefined;
+
+  /*
+   * Per-type extras that don't warrant their own column, e.g.
+   * { sslEnabled: true, elasticsearchIndex: "logs-*" }.
+   */
+  additionalOptions?: JSONObject | undefined;
+}
+
+export interface DataSourceQueryWindow {
+  startDate: Date;
+  endDate: Date;
+  /*
+   * Range-query resolution. Absent ⇒ derived server-side from the window
+   * via DataSourceLimitsUtil.getStepInSeconds.
+   */
+  stepInSeconds?: number | undefined;
+}
+
+/*
+ * One connector per DataSourceType. Implementations MUST:
+ *  - validate their egress target through DataSourceEgressGuard before
+ *    every connection (test and query alike);
+ *  - enforce read-only semantics where the protocol allows writes (SQL
+ *    engines via SqlQueryGuard + engine-level read-only sessions);
+ *  - apply the DataSourceLimits timeouts and row/series/response caps;
+ *  - normalize failures into thrown Exceptions with operator-actionable
+ *    messages (bad credentials vs unreachable host vs bad query), never
+ *    raw driver errors that could leak connection internals.
+ */
+export interface DataSourceConnector {
+  dataSourceType: DataSourceType;
+
+  /*
+   * Cheap connectivity + credential probe used by the "Test Connection"
+   * button. Resolves on success; throws a friendly Exception on failure.
+   */
+  testConnection(settings: DataSourceConnectionSettings): Promise<void>;
+
+  /*
+   * Run `query` and normalize to the AggregatedResult shape chart widgets
+   * already consume: data points of { timestamp, value, ...groupKeys },
+   * where non-timestamp/value keys (e.g. Prometheus labels or a SQL
+   * `series` column) become the series grouping.
+   */
+  queryTimeSeries(
+    settings: DataSourceConnectionSettings,
+    query: string,
+    window: DataSourceQueryWindow,
+  ): Promise<AggregatedResult>;
+
+  /*
+   * Run `query` and return rows for the table widget.
+   */
+  queryTable(
+    settings: DataSourceConnectionSettings,
+    query: string,
+    window: DataSourceQueryWindow,
+  ): Promise<DataSourceTableResult>;
+}

package/Server/Utils/Database/PostgresErrorTranslator.ts

@@ -6,6 +6,7 @@ import Exception from "../../../Types/Exception/Exception";
  * https://www.postgresql.org/docs/current/errcodes-appendix.html
  */
 const FOREIGN_KEY_VIOLATION: string = "23503";
+const UNIQUE_VIOLATION: string = "23505";
 
 /*
  * TypeORM reports driver failures as QueryFailedError, which does not extend
@@ -25,15 +26,75 @@ interface PostgresDriverError {
   detail?: string | undefined;
 }
 
+/*
+ * The SQLSTATE that produced a translated exception, recorded on the exception
+ * itself. `Exception.code` is the OneUptime ExceptionCode (a number), so the
+ * driver code needs a field of its own — without it, translation is one-way and
+ * a caller that recovers from one specific failure can no longer tell which
+ * failure it got. RumSessionPinService is exactly that caller: it turns a
+ * unique violation on (projectId, rumApplicationId, sessionId) into an
+ * idempotent "already pinned" instead of an error the user has to interpret.
+ */
+export interface TranslatedPostgresException extends Exception {
+  postgresErrorCode?: string | undefined;
+}
+
 export default class PostgresErrorTranslator {
   public static translate(error: unknown): unknown {
     const driverError: PostgresDriverError | null =
       this.getPostgresDriverError(error);
 
-    if (!driverError || driverError.code !== FOREIGN_KEY_VIOLATION) {
+    if (!driverError) {
       return error;
     }
 
+    if (driverError.code === FOREIGN_KEY_VIOLATION) {
+      return this.translateForeignKeyViolation(error, driverError);
+    }
+
+    if (driverError.code === UNIQUE_VIOLATION) {
+      return this.translateUniqueViolation(error, driverError);
+    }
+
+    return error;
+  }
+
+  /*
+   * `throw`-friendly wrapper. Kept synchronous so callers can keep using
+   * `throw translator(...)` on the synchronous throw path.
+   */
+  public static translateException(error: Exception): Exception {
+    return this.translate(error) as Exception;
+  }
+
+  /*
+   * True for a unique violation whether or not it has been through translate().
+   * Callers that recover from a duplicate must use this rather than checking
+   * `code === "23505"` themselves: by the time a service's own catch block sees
+   * an error thrown out of DatabaseService, it has already been translated.
+   */
+  public static isUniqueViolation(error: unknown): boolean {
+    if (!error || typeof error !== "object") {
+      return false;
+    }
+
+    if (
+      (error as TranslatedPostgresException).postgresErrorCode ===
+      UNIQUE_VIOLATION
+    ) {
+      return true;
+    }
+
+    const driverError: PostgresDriverError | null =
+      this.getPostgresDriverError(error);
+
+    return driverError?.code === UNIQUE_VIOLATION;
+  }
+
+  private static translateForeignKeyViolation(
+    error: unknown,
+    driverError: PostgresDriverError,
+  ): unknown {
     const detail: string = driverError.detail || "";
 
     /*
@@ -44,12 +105,15 @@ export default class PostgresErrorTranslator {
       const referencingTable: string =
         this.getTableNameFromDetail(detail) || driverError.table || "";
 
-      return new BadDataException(
-        referencingTable
-          ? `This item cannot be deleted because ${this.humanizeTableName(
-              referencingTable,
-            )} records still reference it. Please delete those records first, and then try again.`
-          : "This item cannot be deleted because other records still reference it. Please delete those records first, and then try again.",
+      return this.tag(
+        new BadDataException(
+          referencingTable
+            ? `This item cannot be deleted because ${this.humanizeTableName(
+                referencingTable,
+              )} records still reference it. Please delete those records first, and then try again.`
+            : "This item cannot be deleted because other records still reference it. Please delete those records first, and then try again.",
+        ),
+        driverError,
       );
     }
 
@@ -60,12 +124,15 @@ export default class PostgresErrorTranslator {
     if (detail.includes("is not present in table")) {
       const referencedTable: string = this.getTableNameFromDetail(detail) || "";
 
-      return new BadDataException(
-        referencedTable
-          ? `This request references ${this.humanizeTableName(
-              referencedTable,
-            )} that does not exist. Please check the request and try again.`
-          : "This request references an item that does not exist. Please check the request and try again.",
+      return this.tag(
+        new BadDataException(
+          referencedTable
+            ? `This request references ${this.humanizeTableName(
+                referencedTable,
+              )} that does not exist. Please check the request and try again.`
+            : "This request references an item that does not exist. Please check the request and try again.",
+        ),
+        driverError,
       );
     }
 
@@ -73,11 +140,63 @@ export default class PostgresErrorTranslator {
   }
 
   /*
-   * `throw`-friendly wrapper. Kept synchronous so callers can keep using
-   * `throw translator(...)` on the synchronous throw path.
+   * INSERT/UPDATE colliding with an existing row:
+   *   Key ("twilioAccountSID")=(AC123...) already exists.
+   *
+   * The column names go into the message because they tell the user which
+   * field to change. The values do not: they are another row's data, and that
+   * row may well belong to a different project (issue #3020 was reported
+   * against exactly such a cross-tenant collision).
    */
-  public static translateException(error: Exception): Exception {
-    return this.translate(error) as Exception;
+  private static translateUniqueViolation(
+    error: unknown,
+    driverError: PostgresDriverError,
+  ): unknown {
+    const detail: string = driverError.detail || "";
+
+    if (!detail.includes("already exists")) {
+      return error;
+    }
+
+    const columns: Array<string> = this.getColumnNamesFromDetail(detail);
+    const subject: string = driverError.table
+      ? `A ${this.humanizeTableName(driverError.table)}`
+      : "A record";
+
+    if (columns.length === 0) {
+      return this.tag(
+        new BadDataException(
+          `${subject} with these values already exists. Please use different values and try again.`,
+        ),
+        driverError,
+      );
+    }
+
+    const fields: string = columns
+      .map((column: string): string => {
+        return this.humanizeColumnName(column);
+      })
+      .join(", ");
+
+    return this.tag(
+      new BadDataException(
+        columns.length === 1
+          ? `${subject} with this ${fields} already exists. Please use a different value and try again.`
+          : `${subject} with the same ${fields} already exists. Please use different values and try again.`,
+      ),
+      driverError,
+    );
+  }
+
+  /* Records the SQLSTATE on the translated exception. See the interface above. */
+  private static tag(
+    exception: BadDataException,
+    driverError: PostgresDriverError,
+  ): BadDataException {
+    (exception as TranslatedPostgresException).postgresErrorCode =
+      driverError.code;
+
+    return exception;
   }
 
   private static getPostgresDriverError(
@@ -117,6 +236,32 @@ export default class PostgresErrorTranslator {
     return match && match[1] ? match[1] : null;
   }
 
+  /*
+   * `Key ("twilioAccountSID")=(AC1)...` -> ["twilioAccountSID"]
+   * `Key (slug)=(acme)...`              -> ["slug"]
+   * `Key (a, "b")=(1, 2)...`            -> ["a", "b"]
+   *
+   * Postgres quotes an identifier only when it needs to, so both forms turn up
+   * in the same database. Anchored on the leading `Key (` so the value list
+   * that follows cannot be mistaken for the column list.
+   */
+  private static getColumnNamesFromDetail(detail: string): Array<string> {
+    const match: RegExpMatchArray | null = detail.match(/^Key \(([^)]*)\)=/);
+
+    if (!match || !match[1]) {
+      return [];
+    }
+
+    return match[1]
+      .split(",")
+      .map((column: string): string => {
+        return column.trim().replace(/^"(.*)"$/, "$1");
+      })
+      .filter((column: string): boolean => {
+        return column.length > 0;
+      });
+  }
+
   /*
    * "MonitorStatusTimeline" -> "Monitor Status Timeline". Table names are the
    * PascalCase model names, so this reads well enough to put in front of a
@@ -128,4 +273,14 @@ export default class PostgresErrorTranslator {
       .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
       .trim();
   }
+
+  /*
+   * "twilioAccountSID" -> "Twilio Account SID". Same split as table names, but
+   * columns are camelCase so the first letter needs capitalising too.
+   */
+  private static humanizeColumnName(columnName: string): string {
+    const humanized: string = this.humanizeTableName(columnName);
+
+    return humanized.charAt(0).toUpperCase() + humanized.slice(1);
+  }
 }