@oneuptime/common 11.3.0 → 11.3.2

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -1327,10 +1327,26 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       );
     }
 
+    /*
+     * ON CLUSTER is appended as RAW SQL right after the table reference. The
+     * analytics schema is ALWAYS a sharded + replicated cluster (see
+     * ClusterConfig: a single node is just a "cluster of one"). A bare
+     * `ALTER … ADD COLUMN` reaches only the shard the client is connected to —
+     * Keeper replicates within a shard but never across shards — so the column
+     * lands on one shard and a later scatter-gather read through the Distributed
+     * wrapper hits a shard that lacks it and fails with
+     * "Missing columns: '<col>'" (Code 47 UNKNOWN_IDENTIFIER). ON CLUSTER also
+     * makes this ADD COLUMN wait for cluster-wide completion, so the separate
+     * `ADD INDEX` that addColumnInDatabase issues next never races a column that
+     * has not yet propagated to the node the index DDL lands on.
+     */
     const statement: Statement = SQL`
             ALTER TABLE ${this.database.getDatasourceOptions().database!}.${getStorageTableName(
               this.model.tableName,
-            )} ADD COLUMN IF NOT EXISTS `.append(columnDef);
+            )}`
+      .append(onClusterClause())
+      .append(" ADD COLUMN IF NOT EXISTS ")
+      .append(columnDef);
 
     logger.debug(`${this.model.tableName} Add Column Statement`);
     logger.debug(statement);
@@ -1359,8 +1375,14 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
     const databaseName: string = this.database.getDatasourceOptions().database!;
     const statement: Statement = new Statement();
+    /*
+     * ON CLUSTER (raw SQL) so the skip index is added on every shard, matching
+     * toAddColumnStatement. A bare ADD INDEX only reaches the connected shard,
+     * and if the column it references has not propagated to that node yet it
+     * fails with "Missing columns: '<col>'" (Code 47).
+     */
     statement.append(
-      `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
+      `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)}${onClusterClause()} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
     );
 
     logger.debug(`${this.model.tableName} Add Skip Index Statement`);
@@ -1371,7 +1393,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
   public toDropSkipIndexStatement(indexName: string): string {
     const databaseName: string = this.database.getDatasourceOptions().database!;
-    const statement: string = `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)} DROP INDEX IF EXISTS ${indexName}`;
+    const statement: string = `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)}${onClusterClause()} DROP INDEX IF EXISTS ${indexName}`;
 
     logger.debug(`${this.model.tableName} Drop Skip Index Statement`);
     logger.debug(statement);
@@ -1383,7 +1405,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     const statement: string = `ALTER TABLE ${this.database.getDatasourceOptions()
       .database!}.${getStorageTableName(
       this.model.tableName,
-    )} DROP COLUMN IF EXISTS ${columnName}`;
+    )}${onClusterClause()} DROP COLUMN IF EXISTS ${columnName}`;
 
     logger.debug(`${this.model.tableName} Drop Column Statement`);
     logger.debug(statement);

package/Server/Utils/Telemetry.ts

@@ -524,30 +524,45 @@ export default class Telemetry {
   }): void {
     const { span, exception } = data;
 
-    const exceptionAttributes: Attributes =
-      this.getExceptionAttributes(exception);
-
-    // log the exception as well
-    logger.error(exception);
-
     /*
-     * Span *events* (from recordException) are not reliably surfaced when the
-     * span is read back, and setStatus on its own only records the error CODE,
-     * not the message. So we also attach the exception details as queryable
-     * span attributes — including DB driver fields like the failing constraint
-     * and table — so the actual cause is visible in the trace UI instead of an
-     * empty "Error" status.
+     * This runs on the universal catch path of every @CaptureSpan-decorated
+     * function, so it must NEVER throw — a throw here would mask the original
+     * error and skip marking the span. The whole body is wrapped defensively
+     * and the span is always marked-as-error and ended (in the finally).
      */
-    span.setAttributes(exceptionAttributes);
-    span.recordException(exception as SpanException);
-    span.setStatus({
-      code: SpanStatusCode.ERROR,
-      message:
-        (exceptionAttributes["exception.message"] as string | undefined) ||
-        "Error",
-    });
+    try {
+      const exceptionAttributes: Attributes =
+        this.getExceptionAttributes(exception);
+
+      // log the exception as well
+      logger.error(exception);
 
-    this.endSpan(span);
+      /*
+       * Span *events* (from recordException) are not reliably surfaced when the
+       * span is read back, and setStatus on its own only records the error CODE,
+       * not the message. So we also attach the exception details as queryable
+       * span attributes — including DB driver fields like the failing constraint
+       * and table — so the actual cause is visible in the trace UI instead of an
+       * empty "Error" status.
+       */
+      span.setAttributes(exceptionAttributes);
+      span.recordException(exception as SpanException);
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message:
+          (exceptionAttributes["exception.message"] as string | undefined) ||
+          "Error",
+      });
+    } catch {
+      // Enrichment failed on some exotic thrown value — still flag the span.
+      try {
+        span.setStatus({ code: SpanStatusCode.ERROR });
+      } catch {
+        // span may already be ended; nothing more we can do.
+      }
+    } finally {
+      this.endSpan(span);
+    }
   }
 
   /*
@@ -557,72 +572,113 @@ export default class Telemetry {
    * the Postgres fields (SQLSTATE code, detail, constraint, table, column) live
    * either on the error itself or on `driverError`; these are what tell us which
    * constraint failed during e.g. a cascade delete.
+   *
+   * The thrown value is `unknown`, so anything (a Proxy, an object with a
+   * throwing getter or a throwing toString) could be passed — every field read
+   * and string coercion is guarded so this can never throw on the error path.
    */
   private static getExceptionAttributes(exception: unknown): Attributes {
     const attributes: Attributes = {};
 
-    if (exception === null || exception === undefined) {
-      attributes["exception.message"] =
-        "Unknown error: null or undefined was thrown";
-      return attributes;
-    }
+    try {
+      if (exception === null || exception === undefined) {
+        attributes["exception.message"] =
+          "Unknown error: null or undefined was thrown";
+        return attributes;
+      }
 
-    if (exception instanceof Error) {
-      attributes["exception.type"] =
-        exception.name || exception.constructor?.name || "Error";
-      attributes["exception.message"] = exception.message || "";
-      if (exception.stack) {
-        attributes["exception.stacktrace"] = exception.stack.substring(0, 8000);
+      if (exception instanceof Error) {
+        attributes["exception.type"] =
+          exception.name || exception.constructor?.name || "Error";
+        attributes["exception.message"] = this.truncate(
+          exception.message || "",
+          4000,
+        );
+        if (exception.stack) {
+          attributes["exception.stacktrace"] = this.truncate(
+            exception.stack,
+            8000,
+          );
+        }
+      } else if (typeof exception === "string") {
+        attributes["exception.message"] = this.truncate(exception, 4000);
+      } else {
+        attributes["exception.message"] = this.truncate(
+          this.safeStringify(exception),
+          4000,
+        );
       }
-    } else if (typeof exception === "string") {
-      attributes["exception.message"] = exception;
-    } else {
-      attributes["exception.message"] = this.safeStringify(exception);
-    }
 
-    type PotentialDatabaseError = {
-      code?: unknown;
-      detail?: unknown;
-      constraint?: unknown;
-      table?: unknown;
-      column?: unknown;
-      schema?: unknown;
-      query?: unknown;
-      driverError?: PotentialDatabaseError;
-    };
+      type PotentialDatabaseError = {
+        code?: unknown;
+        detail?: unknown;
+        constraint?: unknown;
+        table?: unknown;
+        column?: unknown;
+        schema?: unknown;
+        query?: unknown;
+        driverError?: PotentialDatabaseError;
+      };
 
-    const error: PotentialDatabaseError = exception as PotentialDatabaseError;
-    const databaseError: PotentialDatabaseError = error.driverError || error;
+      const error: PotentialDatabaseError = exception as PotentialDatabaseError;
+      const databaseError: PotentialDatabaseError = error.driverError || error;
 
-    const setStringAttribute: (key: string, value: unknown) => void = (
-      key: string,
-      value: unknown,
-    ): void => {
-      if (value !== undefined && value !== null && value !== "") {
-        attributes[key] = String(value);
-      }
-    };
+      const setStringAttribute: (key: string, value: unknown) => void = (
+        key: string,
+        value: unknown,
+      ): void => {
+        try {
+          if (value !== undefined && value !== null && value !== "") {
+            attributes[key] = this.truncate(String(value), 2000);
+          }
+        } catch {
+          // A single unserializable field must not abort the whole enrichment.
+        }
+      };
 
-    // SQLSTATE (e.g. "23503" = foreign key violation) or a Node error code.
-    setStringAttribute("exception.code", error.code ?? databaseError.code);
-    setStringAttribute("db.error.detail", databaseError.detail);
-    setStringAttribute("db.error.constraint", databaseError.constraint);
-    setStringAttribute("db.error.table", databaseError.table);
-    setStringAttribute("db.error.column", databaseError.column);
-    setStringAttribute("db.error.schema", databaseError.schema);
+      // SQLSTATE (e.g. "23503" = foreign key violation) or a Node error code.
+      setStringAttribute("exception.code", error.code ?? databaseError.code);
+      setStringAttribute("db.error.detail", databaseError.detail);
+      setStringAttribute("db.error.constraint", databaseError.constraint);
+      setStringAttribute("db.error.table", databaseError.table);
+      setStringAttribute("db.error.column", databaseError.column);
+      setStringAttribute("db.error.schema", databaseError.schema);
 
-    if (typeof error.query === "string" && error.query.length > 0) {
-      attributes["db.statement"] = error.query.substring(0, 2000);
+      if (typeof error.query === "string" && error.query.length > 0) {
+        attributes["db.statement"] = this.truncate(error.query, 2000);
+      }
+    } catch {
+      /*
+       * Reading exotic thrown values (throwing getters, Proxies) must never
+       * crash the error-reporting path. Keep whatever was collected so far.
+       */
+      if (!attributes["exception.message"]) {
+        attributes["exception.message"] =
+          "Error (exception details could not be extracted)";
+      }
     }
 
     return attributes;
   }
 
+  private static truncate(value: string, maxLength: number): string {
+    return value.length > maxLength ? value.substring(0, maxLength) : value;
+  }
+
   private static safeStringify(value: unknown): string {
     try {
-      return JSON.stringify(value) || String(value);
+      const serialized: string | undefined = JSON.stringify(value);
+      if (serialized) {
+        return serialized;
+      }
     } catch {
+      // fall through to String() below
+    }
+
+    try {
       return String(value);
+    } catch {
+      return "[unserializable error value]";
     }
   }
 

package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts

@@ -232,20 +232,33 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
       );
     });
 
-    test("schema ALTERs target the local table", () => {
+    test("schema ALTERs target the local table ON CLUSTER", () => {
       const column: AnalyticsTableColumn = new SpanModel().tableColumns[1]!;
-      expect(fullText(spanGen.toAddColumnStatement(column))).toContain(
-        "SpanItemV3Local",
-      );
-      expect(fullText(spanGen.toAddSkipIndexStatement(column)!)).toContain(
-        "SpanItemV3Local",
-      );
-      expect(spanGen.toDropColumnStatement("traceId")).toContain(
-        "SpanItemV3Local",
-      );
-      expect(spanGen.toDropSkipIndexStatement("idx_trace_id")).toContain(
-        "SpanItemV3Local",
+
+      /*
+       * Column / skip-index ADD + DROP must carry ON CLUSTER: the local storage
+       * table is Replicated per shard, and Keeper only replicates within a
+       * shard. Without ON CLUSTER a reconciled column/index lands on a single
+       * shard, so a scatter-gather read through the Distributed wrapper hits a
+       * shard that lacks it and fails with "Missing columns" (Code 47).
+       */
+      const addColumn: string = fullText(spanGen.toAddColumnStatement(column));
+      expect(addColumn).toContain("SpanItemV3Local");
+      expect(addColumn).toContain("ON CLUSTER 'oneuptime'");
+
+      const addIndex: string = fullText(
+        spanGen.toAddSkipIndexStatement(column)!,
       );
+      expect(addIndex).toContain("SpanItemV3Local");
+      expect(addIndex).toContain("ON CLUSTER 'oneuptime'");
+
+      const dropColumn: string = spanGen.toDropColumnStatement("traceId");
+      expect(dropColumn).toContain("SpanItemV3Local");
+      expect(dropColumn).toContain("ON CLUSTER 'oneuptime'");
+
+      const dropIndex: string = spanGen.toDropSkipIndexStatement("idx_trace_id");
+      expect(dropIndex).toContain("SpanItemV3Local");
+      expect(dropIndex).toContain("ON CLUSTER 'oneuptime'");
     });
 
     test("ALTER UPDATE mutates the local table with ON CLUSTER", () => {

package/Tests/Server/Utils/Telemetry.test.ts

@@ -0,0 +1,251 @@
+import Telemetry, {
+  Span,
+  SpanStatusCode,
+} from "../../../Server/Utils/Telemetry";
+import {
+  afterAll,
+  beforeAll,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+
+type ExceptionAttributes = Record<string, string | undefined>;
+
+// getExceptionAttributes is a private static; reach it through a narrow cast.
+function getAttributes(exception: unknown): ExceptionAttributes {
+  return (
+    Telemetry as unknown as {
+      getExceptionAttributes: (e: unknown) => ExceptionAttributes;
+    }
+  ).getExceptionAttributes(exception);
+}
+
+describe("Telemetry.getExceptionAttributes", () => {
+  test("extracts Postgres fields from a TypeORM QueryFailedError-shaped error", () => {
+    const error: Error = Object.assign(
+      new Error('delete on "Project" violates foreign key constraint'),
+      {
+        driverError: {
+          code: "23503",
+          detail: 'Key (id)=(abc) is still referenced from table "Monitor".',
+          constraint: "FK_monitor_project",
+          table: "Monitor",
+          column: "projectId",
+          schema: "public",
+        },
+        query: 'DELETE FROM "Project" WHERE "_id" IN ($1)',
+      },
+    );
+
+    const attributes: ExceptionAttributes = getAttributes(error);
+
+    expect(attributes["exception.code"]).toBe("23503");
+    expect(attributes["db.error.constraint"]).toBe("FK_monitor_project");
+    expect(attributes["db.error.table"]).toBe("Monitor");
+    expect(attributes["db.error.column"]).toBe("projectId");
+    expect(attributes["db.error.schema"]).toBe("public");
+    expect(attributes["db.error.detail"]).toContain("still referenced");
+    expect(attributes["db.statement"]).toBe(
+      'DELETE FROM "Project" WHERE "_id" IN ($1)',
+    );
+    expect(attributes["exception.type"]).toBe("Error");
+    expect(attributes["exception.message"]).toContain("violates foreign key");
+  });
+
+  test("reads a top-level pg error code when there is no driverError", () => {
+    const error: Error = Object.assign(
+      new Error("duplicate key value violates unique constraint"),
+      { code: "23505", constraint: "uniq_email" },
+    );
+
+    const attributes: ExceptionAttributes = getAttributes(error);
+
+    expect(attributes["exception.code"]).toBe("23505");
+    expect(attributes["db.error.constraint"]).toBe("uniq_email");
+  });
+
+  test("captures type, message and stacktrace for a plain Error", () => {
+    const attributes: ExceptionAttributes = getAttributes(new Error("boom"));
+
+    expect(attributes["exception.type"]).toBe("Error");
+    expect(attributes["exception.message"]).toBe("boom");
+    expect(typeof attributes["exception.stacktrace"]).toBe("string");
+  });
+
+  test("handles a thrown string", () => {
+    const attributes: ExceptionAttributes = getAttributes("kaboom");
+
+    expect(attributes["exception.message"]).toBe("kaboom");
+    expect(attributes["exception.type"]).toBeUndefined();
+    expect(attributes["db.error.constraint"]).toBeUndefined();
+  });
+
+  test("handles null and undefined throws", () => {
+    expect(getAttributes(null)["exception.message"]).toBe(
+      "Unknown error: null or undefined was thrown",
+    );
+    expect(getAttributes(undefined)["exception.message"]).toBe(
+      "Unknown error: null or undefined was thrown",
+    );
+  });
+
+  test("serializes a non-Error object throw into the message", () => {
+    const attributes: ExceptionAttributes = getAttributes({ a: 1, b: "x" });
+
+    expect(attributes["exception.message"]).toBe('{"a":1,"b":"x"}');
+  });
+
+  test("truncates an oversized message and SQL statement", () => {
+    const error: Error = Object.assign(new Error("x".repeat(9000)), {
+      query: `SELECT ${"a".repeat(9000)}`,
+    });
+
+    const attributes: ExceptionAttributes = getAttributes(error);
+
+    expect(attributes["exception.message"]?.length).toBe(4000);
+    expect(attributes["db.statement"]?.length).toBe(2000);
+  });
+
+  // --- crash-safety: hostile thrown values must never make this throw. ---
+
+  test("does not throw on an object with throwing getters", () => {
+    const hostile: Record<string, unknown> = {};
+    Object.defineProperty(hostile, "code", {
+      enumerable: true,
+      get: (): never => {
+        throw new Error("getter blew up");
+      },
+    });
+    Object.defineProperty(hostile, "driverError", {
+      enumerable: true,
+      get: (): never => {
+        throw new Error("driverError blew up");
+      },
+    });
+
+    let attributes: ExceptionAttributes = {};
+    expect((): void => {
+      attributes = getAttributes(hostile);
+    }).not.toThrow();
+    expect(typeof attributes["exception.message"]).toBe("string");
+  });
+
+  test("does not throw on a field whose toString throws (good fields survive)", () => {
+    const error: Error = Object.assign(new Error("db fail"), {
+      driverError: {
+        detail: {
+          toString: (): never => {
+            throw new Error("toString blew up");
+          },
+        },
+      },
+    });
+
+    expect((): ExceptionAttributes => {
+      return getAttributes(error);
+    }).not.toThrow();
+    expect(getAttributes(error)["exception.message"]).toBe("db fail");
+  });
+
+  test("does not throw on a Proxy that traps every property read", () => {
+    const hostile: unknown = new Proxy(
+      {},
+      {
+        get: (): never => {
+          throw new Error("proxy trap");
+        },
+      },
+    );
+
+    expect((): ExceptionAttributes => {
+      return getAttributes(hostile);
+    }).not.toThrow();
+  });
+});
+
+describe("Telemetry.recordExceptionMarkSpanAsErrorAndEndSpan", () => {
+  type FakeSpanState = {
+    attributes: Record<string, unknown> | null;
+    status: { code: number; message?: string } | null;
+    recorded: unknown;
+    ended: number;
+  };
+
+  // Silence the logger.error console output these tests intentionally trigger.
+  beforeAll((): void => {
+    jest.spyOn(console, "error").mockImplementation((): void => {
+      return undefined;
+    });
+  });
+
+  afterAll((): void => {
+    jest.restoreAllMocks();
+  });
+
+  function makeFakeSpan(
+    state: FakeSpanState,
+    opts?: { throwOnSetAttributes?: boolean },
+  ): Span {
+    return {
+      setAttributes: (a: Record<string, unknown>): unknown => {
+        if (opts?.throwOnSetAttributes) {
+          throw new Error("setAttributes blew up");
+        }
+        state.attributes = a;
+        return undefined;
+      },
+      recordException: (e: unknown): unknown => {
+        state.recorded = e;
+        return undefined;
+      },
+      setStatus: (s: { code: number; message?: string }): unknown => {
+        state.status = s;
+        return undefined;
+      },
+      end: (): void => {
+        state.ended += 1;
+      },
+    } as unknown as Span;
+  }
+
+  test("marks the span as error with a message and ends it", () => {
+    const state: FakeSpanState = {
+      attributes: null,
+      status: null,
+      recorded: null,
+      ended: 0,
+    };
+
+    Telemetry.recordExceptionMarkSpanAsErrorAndEndSpan({
+      span: makeFakeSpan(state),
+      exception: new Error("kaboom"),
+    });
+
+    expect(state.ended).toBe(1);
+    expect(state.status?.code).toBe(SpanStatusCode.ERROR);
+    expect(state.status?.message).toBe("kaboom");
+    expect((state.attributes || {})["exception.message"]).toBe("kaboom");
+  });
+
+  test("still ends and flags the span even if a span write throws", () => {
+    const state: FakeSpanState = {
+      attributes: null,
+      status: null,
+      recorded: null,
+      ended: 0,
+    };
+    const span: Span = makeFakeSpan(state, { throwOnSetAttributes: true });
+
+    expect((): void => {
+      Telemetry.recordExceptionMarkSpanAsErrorAndEndSpan({
+        span,
+        exception: new Error("kaboom"),
+      });
+    }).not.toThrow();
+
+    expect(state.ended).toBe(1);
+    expect(state.status?.code).toBe(SpanStatusCode.ERROR);
+  });
+});