@oneuptime/common 11.5.13 → 11.6.1

package/Server/Utils/Telemetry/TelemetryWriterClient.ts

@@ -0,0 +1,263 @@
+import ClusterKeyAuthorization from "../../Middleware/ClusterKeyAuthorization";
+import HTTPErrorResponse from "../../../Types/API/HTTPErrorResponse";
+import HTTPResponse from "../../../Types/API/HTTPResponse";
+import URL from "../../../Types/API/URL";
+import { JSONObject } from "../../../Types/JSON";
+import API from "../../../Utils/API";
+import {
+  FanInInsertTarget,
+  FanInInsertTransport,
+  TransientInsertError,
+} from "./TelemetryFanInWriter";
+import { type ClickHouseSettings } from "@clickhouse/client";
+
+/*
+ * Worker-side client for the telemetry-writer tier.
+ *
+ * When TELEMETRY_WRITER_URL is set on a pod, its fan-in writer delivers
+ * every token group as one HTTP POST to the writer tier instead of
+ * inserting into ClickHouse itself. This decouples worker replica count
+ * from ClickHouse INSERT concurrency: telemetry insert load on ClickHouse
+ * is (writerReplicas × TELEMETRY_FANIN_MAX_CONCURRENT_INSERTS) regardless
+ * of how far the worker fleet scales out. (Reads and low-volume direct
+ * inserts that never go through the fan-in writer still use this pod's own
+ * ClickHouse pools.)
+ *
+ * Failure classification maps onto the fan-in writer's retry semantics:
+ * every payload carries a dedup token that is byte-identical across
+ * retries, so any AMBIGUOUS outcome (network error, timeout, writer pod
+ * dying mid-request) is safe to classify transient — a retry of a payload
+ * that actually landed is dropped server-side by ClickHouse. Only a
+ * definitive verdict from the writer (2xx success, 4xx/500 permanent
+ * failure) is treated as final.
+ */
+
+export const TELEMETRY_WRITER_INSERT_ROUTE: string = "/telemetry-writer/insert";
+
+export function getTelemetryWriterUrl(): string | null {
+  const raw: string | undefined = process.env["TELEMETRY_WRITER_URL"];
+  const trimmed: string = (raw || "").trim();
+  if (!trimmed) {
+    return null;
+  }
+  // Normalize away a trailing slash so route concatenation is predictable.
+  return trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed;
+}
+
+export function isTelemetryWriterForwardingEnabled(): boolean {
+  return getTelemetryWriterUrl() !== null;
+}
+
+export function getTelemetryWriterRequestTimeoutMs(): number {
+  const raw: string | undefined =
+    process.env["TELEMETRY_WRITER_REQUEST_TIMEOUT_MS"];
+  const parsed: number = parseInt(raw || "", 10);
+  /*
+   * Generous (the writer holds the request until ClickHouse accepts the
+   * rows, which under load includes its batching wait and retry loop) but
+   * deliberately under the BullMQ job lock: the fan-in writer makes up to
+   * 6 attempts, and 6 x 90s + ~8s backoff ≈ 9.2 min must stay inside
+   * TELEMETRY_LOCK_DURATION_MS (10 min) — otherwise a saturated writer
+   * tier stalls jobs into redelivery churn instead of failing them
+   * cleanly for retry.
+   */
+  return !isNaN(parsed) && parsed > 0 ? parsed : 90_000;
+}
+
+export function getTelemetryWriterMaxBodyBytes(): number {
+  const raw: string | undefined =
+    process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"];
+  const parsed: number = parseInt(raw || "", 10);
+  /*
+   * Stay well under the writer route's 50 MB express.json limit (plus
+   * headroom for headers/encoding). Bodies above this are split before
+   * sending — a 413 would be classified permanent and drop rows.
+   */
+  return !isNaN(parsed) && parsed > 0 ? parsed : 30_000_000;
+}
+
+export interface WriterPostResult {
+  statusCode: number;
+  errorMessage?: string | undefined;
+}
+
+export type WriterPostFn = (args: {
+  url: string;
+  body: JSONObject;
+  timeoutMs: number;
+}) => Promise<WriterPostResult>;
+
+const defaultPost: WriterPostFn = async (args: {
+  url: string;
+  body: JSONObject;
+  timeoutMs: number;
+}): Promise<WriterPostResult> => {
+  let response: HTTPResponse<JSONObject> | HTTPErrorResponse;
+  try {
+    response = await API.post<JSONObject>({
+      url: URL.fromString(args.url),
+      data: args.body,
+      headers: {
+        ...ClusterKeyAuthorization.getClusterKeyHeaders(),
+      },
+      options: {
+        timeout: args.timeoutMs,
+        skipAuthRefresh: true,
+      },
+    });
+  } catch (err) {
+    /*
+     * API.post throws (APIException) when no HTTP response exists at all —
+     * connection refused, reset, DNS, timeout. Outcome unknown; statusCode 0
+     * classifies it transient.
+     */
+    return {
+      statusCode: 0,
+      errorMessage: err instanceof Error ? err.message : String(err),
+    };
+  }
+
+  if (response instanceof HTTPErrorResponse) {
+    return {
+      statusCode: response.statusCode,
+      errorMessage: response.message,
+    };
+  }
+
+  return { statusCode: response.statusCode };
+};
+
+/*
+ * Writer-tier verdicts that mean "retry with the same token": overload
+ * shedding (429) and gateway/unavailability statuses where the request may
+ * or may not have been processed.
+ */
+const TRANSIENT_HTTP_STATUSES: Set<number> = new Set([429, 502, 503, 504]);
+
+function isDefinitiveStatus(statusCode: number): boolean {
+  /*
+   * Anything outside real HTTP semantics (0, -1, undefined coerced) is a
+   * network-level failure with an unknown outcome — transient by the dedup
+   * token argument above.
+   */
+  return statusCode >= 100 && statusCode < 600;
+}
+
+export function createTelemetryWriterTransport(
+  postFn?: WriterPostFn,
+): FanInInsertTransport {
+  const post: WriterPostFn = postFn ?? defaultPost;
+
+  const postGroup: (
+    tableName: string,
+    rows: Array<JSONObject>,
+    dedupToken: string,
+    clickhouseSettings: ClickHouseSettings | undefined,
+    baseUrl: string,
+    maxBodyBytes: number,
+  ) => Promise<void> = async (
+    tableName: string,
+    rows: Array<JSONObject>,
+    dedupToken: string,
+    clickhouseSettings: ClickHouseSettings | undefined,
+    baseUrl: string,
+    maxBodyBytes: number,
+  ): Promise<void> => {
+    const body: JSONObject = {
+      tableName: tableName,
+      rows: rows,
+      dedupToken: dedupToken,
+      ...(clickhouseSettings
+        ? { clickhouseSettings: clickhouseSettings as JSONObject }
+        : {}),
+    };
+
+    /*
+     * Byte-aware splitting: the writer route sits behind a 50 MB JSON
+     * parser, and a 413 is (correctly) classified permanent — so an
+     * oversized group would be dropped after retries, a failure mode the
+     * direct ClickHouse path never had. Halve deterministically instead:
+     * a BullMQ retry re-produces byte-identical rows, hence the identical
+     * split and identical derived tokens ("<token>#0"/"<token>#1"), so
+     * idempotence is preserved (async-insert dedup is a content hash per
+     * insert body, and the distinct suffixes keep tokens collision-free
+     * for any future sync-insert mode). Halves post sequentially — this
+     * all happens under one fan-in insert slot, and splitting must not
+     * multiply the pod's concurrent load.
+     */
+    if (rows.length > 1) {
+      const bodyBytes: number = Buffer.byteLength(JSON.stringify(body), "utf8");
+      if (bodyBytes > maxBodyBytes) {
+        const mid: number = Math.floor(rows.length / 2);
+        await postGroup(
+          tableName,
+          rows.slice(0, mid),
+          `${dedupToken}#0`,
+          clickhouseSettings,
+          baseUrl,
+          maxBodyBytes,
+        );
+        await postGroup(
+          tableName,
+          rows.slice(mid),
+          `${dedupToken}#1`,
+          clickhouseSettings,
+          baseUrl,
+          maxBodyBytes,
+        );
+        return;
+      }
+    }
+
+    const result: WriterPostResult = await post({
+      url: `${baseUrl}${TELEMETRY_WRITER_INSERT_ROUTE}`,
+      body: body,
+      timeoutMs: getTelemetryWriterRequestTimeoutMs(),
+    });
+
+    if (result.statusCode >= 200 && result.statusCode < 300) {
+      return;
+    }
+
+    const detail: string = `telemetry-writer insert into ${tableName} (${rows.length} rows) returned status ${result.statusCode}${result.errorMessage ? `: ${result.errorMessage}` : ""}`;
+
+    if (
+      TRANSIENT_HTTP_STATUSES.has(result.statusCode) ||
+      !isDefinitiveStatus(result.statusCode)
+    ) {
+      throw new TransientInsertError(detail);
+    }
+
+    throw new Error(detail);
+  };
+
+  return async (
+    target: FanInInsertTarget,
+    rows: Array<JSONObject>,
+    options: {
+      dedupToken: string;
+      clickhouseSettings: ClickHouseSettings | undefined;
+    },
+  ): Promise<void> => {
+    const baseUrl: string | null = getTelemetryWriterUrl();
+    if (!baseUrl) {
+      /*
+       * The transport is only installed when the URL is configured; losing
+       * it mid-flight (env mutation in tests) is a permanent config error,
+       * not something a same-token retry can fix.
+       */
+      throw new Error(
+        "TelemetryWriterClient: TELEMETRY_WRITER_URL is not configured.",
+      );
+    }
+
+    await postGroup(
+      target.model.tableName,
+      rows,
+      options.dedupToken,
+      options.clickhouseSettings,
+      baseUrl,
+      getTelemetryWriterMaxBodyBytes(),
+    );
+  };
+}

package/Server/Utils/Telemetry/TelemetryWriterServer.ts

@@ -0,0 +1,240 @@
+import { JSONObject } from "../../../Types/JSON";
+import {
+  FanInInsertError,
+  FanInInsertTarget,
+  FanInSubmitResult,
+  isRetryableInsertError,
+} from "./TelemetryFanInWriter";
+import { type ClickHouseSettings } from "@clickhouse/client";
+
+/*
+ * Writer-tier request handling for the telemetry fan-in path, kept free of
+ * Express so the admission, validation, and status mapping are unit-testable.
+ * The App router (App/FeatureSet/Telemetry/API/TelemetryWriterAPI.ts) is a
+ * thin shell: cluster-key auth → inflight gate → handleWriterInsert.
+ *
+ * Status contract (mirrored by TelemetryWriterClient's classification):
+ * - 200: ClickHouse accepted the rows — into its async-insert buffer by
+ *   default (ClickHouse owns flushing), or durably flushed when
+ *   TELEMETRY_WAIT_FOR_ASYNC_INSERT=true on the writer pods.
+ * - 400: malformed request — never retried.
+ * - 429: this pod is at its inflight-request cap — retry (load shedding is
+ *   the writer tier's flow control at arbitrary worker-fleet sizes; without
+ *   it, parked request bodies would grow pod memory without bound).
+ * - 500: insert failed definitively after the local writer's retries.
+ * - 503: retry — a still-transient insert failure after the local writer
+ *   exhausted its retries, a misconfigured (forwarding) pod, or an unknown
+ *   table. Unknown tables are 503 rather than 400 because of rolling
+ *   upgrades: a new-image worker can submit rows for a just-added
+ *   analytics table to an old-image writer that does not know it yet;
+ *   retries outlive the skew window and the rows land once the writer
+ *   rolls, whereas a 400 would permanently drop them.
+ */
+
+export interface TelemetryWriterInsertRequest {
+  tableName: string;
+  rows: Array<JSONObject>;
+  dedupToken: string;
+  clickhouseSettings: ClickHouseSettings | undefined;
+}
+
+export interface WriterInsertOutcome {
+  statusCode: number;
+  body: JSONObject;
+}
+
+/*
+ * Counting gate for concurrently-served insert requests. Rows a parked
+ * request holds are invisible to the fan-in writer's maxPendingRows until
+ * buffered, so this cap — not maxPendingRows — is what bounds writer-pod
+ * memory when the worker fleet dwarfs the writer tier.
+ */
+export class InflightGate {
+  private max: number;
+  private current: number = 0;
+
+  public constructor(max: number) {
+    this.max = max;
+  }
+
+  public tryAcquire(): boolean {
+    if (this.current >= this.max) {
+      return false;
+    }
+    this.current++;
+    return true;
+  }
+
+  public release(): void {
+    this.current--;
+  }
+
+  public get inflight(): number {
+    return this.current;
+  }
+}
+
+export function readWriterMaxInflightRequestsFromEnv(): number {
+  const raw: string | undefined =
+    process.env["TELEMETRY_WRITER_MAX_INFLIGHT_REQUESTS"];
+  const parsed: number = parseInt(raw || "", 10);
+  return !isNaN(parsed) && parsed > 0 ? parsed : 100;
+}
+
+export type TableTargetResolver = (
+  tableName: string,
+) => FanInInsertTarget | undefined;
+
+/*
+ * Build a tableName → insert-target resolver from the analytics-service
+ * singletons (Common/Server/Services/Index.ts exports AnalyticsServices).
+ * The map is built lazily on first call so import order at boot does not
+ * matter.
+ */
+export function createTableTargetResolver(
+  targets: Array<FanInInsertTarget>,
+): TableTargetResolver {
+  let byTable: Map<string, FanInInsertTarget> | null = null;
+
+  return (tableName: string): FanInInsertTarget | undefined => {
+    if (!byTable) {
+      byTable = new Map<string, FanInInsertTarget>();
+      for (const target of targets) {
+        byTable.set(target.model.tableName, target);
+      }
+    }
+    return byTable.get(tableName);
+  };
+}
+
+export function parseWriterInsertRequest(
+  body: unknown,
+): TelemetryWriterInsertRequest | { error: string } {
+  if (!body || typeof body !== "object" || Array.isArray(body)) {
+    return { error: "Request body must be a JSON object." };
+  }
+
+  const json: JSONObject = body as JSONObject;
+
+  const tableName: unknown = json["tableName"];
+  if (typeof tableName !== "string" || tableName.trim() === "") {
+    return { error: "tableName must be a non-empty string." };
+  }
+
+  const rows: unknown = json["rows"];
+  if (!Array.isArray(rows)) {
+    return { error: "rows must be an array." };
+  }
+  for (const row of rows) {
+    if (!row || typeof row !== "object" || Array.isArray(row)) {
+      return { error: "rows must contain only JSON objects." };
+    }
+  }
+
+  /*
+   * The worker-side writer sends a token on EVERY payload (per-job captured
+   * or per-batch minted). Requiring it here keeps writer-side retries and
+   * ambiguous-network-failure retries duplicate-free by construction.
+   */
+  const dedupToken: unknown = json["dedupToken"];
+  if (typeof dedupToken !== "string" || dedupToken.trim() === "") {
+    return { error: "dedupToken must be a non-empty string." };
+  }
+
+  const clickhouseSettings: unknown = json["clickhouseSettings"];
+  if (
+    clickhouseSettings !== undefined &&
+    (typeof clickhouseSettings !== "object" ||
+      clickhouseSettings === null ||
+      Array.isArray(clickhouseSettings))
+  ) {
+    return { error: "clickhouseSettings must be an object when present." };
+  }
+
+  return {
+    tableName: tableName,
+    rows: rows as Array<JSONObject>,
+    dedupToken: dedupToken,
+    clickhouseSettings: clickhouseSettings as ClickHouseSettings | undefined,
+  };
+}
+
+export interface WriterInsertDeps {
+  resolveTarget: TableTargetResolver;
+  submit: (
+    target: FanInInsertTarget,
+    rows: Array<JSONObject>,
+    options: {
+      dedupToken: string;
+      clickhouseSettings?: ClickHouseSettings | undefined;
+    },
+  ) => Promise<FanInSubmitResult>;
+  /*
+   * True when THIS pod is itself configured to forward inserts to a remote
+   * writer (TELEMETRY_WRITER_URL set). Serving inserts in that state would
+   * bounce payloads between pods indefinitely, so refuse loudly instead.
+   */
+  forwardingEnabled: boolean;
+}
+
+export async function handleWriterInsert(
+  body: unknown,
+  deps: WriterInsertDeps,
+): Promise<WriterInsertOutcome> {
+  if (deps.forwardingEnabled) {
+    return {
+      statusCode: 503,
+      body: {
+        message:
+          "This pod forwards telemetry inserts to a remote writer (TELEMETRY_WRITER_URL is set); it cannot serve writer inserts. Point workers at the telemetry-writer deployment instead.",
+      },
+    };
+  }
+
+  const parsed: TelemetryWriterInsertRequest | { error: string } =
+    parseWriterInsertRequest(body);
+  if ("error" in parsed) {
+    return { statusCode: 400, body: { message: parsed.error } };
+  }
+
+  if (parsed.rows.length === 0) {
+    return { statusCode: 200, body: { rowsAccepted: 0 } };
+  }
+
+  const target: FanInInsertTarget | undefined = deps.resolveTarget(
+    parsed.tableName,
+  );
+  if (!target) {
+    // 503, not 400: see the status contract — version-skew self-healing.
+    return {
+      statusCode: 503,
+      body: {
+        message: `Unknown analytics table: ${parsed.tableName}. If this writer pod is mid-rolling-upgrade, retries will succeed once it runs the newer image.`,
+      },
+    };
+  }
+
+  try {
+    const submission: FanInSubmitResult = await deps.submit(
+      target,
+      parsed.rows,
+      {
+        dedupToken: parsed.dedupToken,
+        clickhouseSettings: parsed.clickhouseSettings,
+      },
+    );
+    await submission.flushed;
+  } catch (err) {
+    const message: string = err instanceof Error ? err.message : String(err);
+    const stillTransient: boolean =
+      err instanceof FanInInsertError
+        ? isRetryableInsertError(err.causeError)
+        : isRetryableInsertError(err);
+    return {
+      statusCode: stillTransient ? 503 : 500,
+      body: { message: message },
+    };
+  }
+
+  return { statusCode: 200, body: { rowsAccepted: parsed.rows.length } };
+}

package/Server/Utils/Telemetry/TelemetryWriterShedMetrics.ts

@@ -0,0 +1,93 @@
+import Redis from "../../Infrastructure/Redis";
+import logger from "../Logger";
+
+/*
+ * Fleet-wide load-shed counter for the telemetry-writer tier, used as the
+ * KEDA autoscaling signal: sustained 429 shedding while ClickHouse is
+ * healthy is the one honest "the writer tier is too small" indicator
+ * (queue depth is the WORKER fleet's signal and must never scale this
+ * tier).
+ *
+ * Every shed increments a per-minute Redis bucket shared by all writer
+ * pods; the metric endpoint sums the current and previous buckets, so any
+ * single pod KEDA happens to poll reports the whole tier's shed count for
+ * roughly the last one-to-two minutes. Buckets expire after five minutes.
+ *
+ * Metrics must never break serving: both functions swallow Redis errors
+ * (recordShed silently, the reader by returning 0 — which biases KEDA
+ * toward NOT scaling on missing data, the safe direction for a tier whose
+ * ceiling is a ClickHouse capacity budget).
+ */
+
+const KEY_PREFIX: string = "telemetry-writer-shed-count";
+const BUCKET_TTL_SECONDS: number = 300;
+
+/** Minimal slice of the ioredis client the counter needs; injectable for tests. */
+export interface ShedCounterClient {
+  incr(key: string): Promise<number>;
+  expire(key: string, seconds: number): Promise<unknown>;
+  get(key: string): Promise<string | null>;
+}
+
+function getDefaultClient(): ShedCounterClient | null {
+  return Redis.getClient();
+}
+
+export function shedBucketKey(epochMinute: number): string {
+  return `${KEY_PREFIX}:${epochMinute}`;
+}
+
+function epochMinute(now: Date): number {
+  return Math.floor(now.getTime() / 60_000);
+}
+
+export async function recordShed(data?: {
+  now?: Date;
+  client?: ShedCounterClient | null;
+}): Promise<void> {
+  try {
+    const client: ShedCounterClient | null =
+      data?.client !== undefined ? data.client : getDefaultClient();
+    if (!client) {
+      return;
+    }
+    const key: string = shedBucketKey(epochMinute(data?.now ?? new Date()));
+    await client.incr(key);
+    await client.expire(key, BUCKET_TTL_SECONDS);
+  } catch (err) {
+    logger.debug("TelemetryWriterShedMetrics: failed to record shed:");
+    logger.debug(err as Error);
+  }
+}
+
+/*
+ * Sheds across the whole writer tier in the current + previous minute
+ * buckets (~ the last one-to-two minutes). Returns 0 when Redis is
+ * unavailable or the buckets are empty/garbled.
+ */
+export async function getRecentShedCount(data?: {
+  now?: Date;
+  client?: ShedCounterClient | null;
+}): Promise<number> {
+  try {
+    const client: ShedCounterClient | null =
+      data?.client !== undefined ? data.client : getDefaultClient();
+    if (!client) {
+      return 0;
+    }
+    const currentMinute: number = epochMinute(data?.now ?? new Date());
+    let total: number = 0;
+    for (const minute of [currentMinute, currentMinute - 1]) {
+      const raw: string | null = await client.get(shedBucketKey(minute));
+      const parsed: number = parseInt(raw ?? "", 10);
+      if (!isNaN(parsed) && parsed > 0) {
+        total += parsed;
+      }
+    }
+    return total;
+  } catch (err) {
+    logger.debug("TelemetryWriterShedMetrics: failed to read shed count:");
+    logger.debug(err as Error);
+    return 0;
+  }
+}