@oneuptime/common 11.5.12 → 11.6.0

package/build/dist/Server/Utils/Telemetry/TelemetryFanInWriter.js

@@ -0,0 +1,496 @@
+import logger from "../Logger";
+import Sleep from "../../../Types/Sleep";
+import UUID from "../../../Utils/UUID";
+import GracefulShutdown, { ShutdownPriority } from "../GracefulShutdown";
+import { nextInsertDedupToken } from "../AnalyticsDatabase/InsertDedupContext";
+import { ClickHouseError } from "@clickhouse/client";
+/*
+ * Marker for transient failures raised by a pluggable insert transport
+ * (e.g. the remote telemetry-writer client): overload shedding (HTTP 429),
+ * writer-tier unavailability (503), or an ambiguous network failure. The
+ * writer retries these with the SAME dedup token, so an ambiguous failure
+ * whose rows actually landed never double-writes.
+ */
+export class TransientInsertError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "TransientInsertError";
+    }
+}
+export class FanInInsertError extends Error {
+    constructor(data) {
+        const causeMessage = data.cause instanceof Error ? data.cause.message : String(data.cause);
+        super(`Fan-in insert into ${data.tableName} failed after ${data.attempts} attempt(s): ${causeMessage}`);
+        this.name = "FanInInsertError";
+        this.causeError = data.cause;
+        if (data.cause instanceof Error && data.cause.stack) {
+            this.stack = `${this.stack}\nCaused by: ${data.cause.stack}`;
+        }
+    }
+}
+/*
+ * ClickHouse server error codes that indicate transient overload or an
+ * ambiguous outcome. All are safe to retry here because every insert
+ * carries a dedup token that is stable across the writer's retries: a
+ * retry of a block that actually landed is dropped server-side.
+ *   202 TOO_MANY_SIMULTANEOUS_QUERIES, 209 SOCKET_TIMEOUT,
+ *   210 NETWORK_ERROR, 241 MEMORY_LIMIT_EXCEEDED, 252 TOO_MANY_PARTS,
+ *   319 UNKNOWN_STATUS_OF_INSERT
+ * NOTE: the client exposes `code` as a STRING (e.g. "202").
+ */
+const RETRYABLE_CLICKHOUSE_CODES = new Set([
+    "202",
+    "209",
+    "210",
+    "241",
+    "252",
+    "319",
+]);
+const NUMERIC_ERROR_CODE_REGEX = /^\d+$/;
+const RETRYABLE_SOCKET_CODES = new Set([
+    "ECONNRESET",
+    "ECONNREFUSED",
+    "ECONNABORTED",
+    "ETIMEDOUT",
+    "EPIPE",
+    "EAI_AGAIN",
+]);
+export function isRetryableInsertError(err) {
+    if (err instanceof ClickHouseError) {
+        return RETRYABLE_CLICKHOUSE_CODES.has(err.code);
+    }
+    if (err instanceof TransientInsertError) {
+        return true;
+    }
+    if (err instanceof Error) {
+        // Duck-typed TransientInsertError (duplicated module instances).
+        if (err.name === "TransientInsertError") {
+            return true;
+        }
+        const code = err["code"];
+        if (typeof code === "string") {
+            if (RETRYABLE_SOCKET_CODES.has(code)) {
+                return true;
+            }
+            // Duck-typed ClickHouseError (in case of duplicated package instances).
+            if (NUMERIC_ERROR_CODE_REGEX.test(code)) {
+                return RETRYABLE_CLICKHOUSE_CODES.has(code);
+            }
+        }
+        /*
+         * @clickhouse/client surfaces its socket-idle request_timeout as a plain
+         * Error("Timeout error."), with no code. The outcome is ambiguous (the
+         * body may have been received), which is exactly what the per-batch dedup
+         * token exists for — retry.
+         */
+        if (err.message.includes("Timeout error") ||
+            err.message.includes("socket hang up")) {
+            return true;
+        }
+    }
+    return false;
+}
+const readPositiveInt = (envKey, defaultValue) => {
+    const raw = process.env[envKey];
+    if (!raw) {
+        return defaultValue;
+    }
+    const parsed = parseInt(raw, 10);
+    return !isNaN(parsed) && parsed > 0 ? parsed : defaultValue;
+};
+export function readFanInWriterOptionsFromEnv() {
+    return {
+        maxBatchRows: readPositiveInt("TELEMETRY_FANIN_MAX_BATCH_ROWS", 100000),
+        maxWaitMs: readPositiveInt("TELEMETRY_FANIN_MAX_WAIT_MS", 5000),
+        maxConcurrentInserts: readPositiveInt("TELEMETRY_FANIN_MAX_CONCURRENT_INSERTS", 4),
+        /*
+         * Must stay comfortably above maxBatchRows: if the two are equal, the
+         * instant a full batch is cut every submitter blocks until that whole
+         * batch finishes inserting — no pipelining. 2x allows one batch in
+         * flight while the next fills.
+         */
+        maxPendingRows: readPositiveInt("TELEMETRY_FANIN_MAX_PENDING_ROWS", 200000),
+        retryMaxAttempts: readPositiveInt("TELEMETRY_FANIN_RETRY_MAX_ATTEMPTS", 6),
+        retryBaseDelayMs: readPositiveInt("TELEMETRY_FANIN_RETRY_BASE_DELAY_MS", 250),
+        retryMaxDelayMs: readPositiveInt("TELEMETRY_FANIN_RETRY_MAX_DELAY_MS", 10000),
+    };
+}
+export class TelemetryFanInWriter {
+    constructor(options) {
+        this.buffers = new Map();
+        this.pendingRows = 0;
+        this.capacityWaiters = [];
+        this.activeInserts = 0;
+        this.insertSlotWaiters = [];
+        this.inFlightDispatches = new Set();
+        /*
+         * Submits that have been called but have not yet buffered their rows
+         * (parked at the waitForCapacity await, or merely yielding its one
+         * microtask). flushAll() must not conclude the drain while any exist —
+         * their rows would land just after the final cut and strand.
+         */
+        this.acceptingSubmits = 0;
+        this.shutdownHandlerRegistered = false;
+        this.insertTransport = null;
+        this.options = options;
+    }
+    /*
+     * Install (or clear, with null) a custom delivery transport for token
+     * groups. Set once at boot when TELEMETRY_WRITER_URL is configured;
+     * batching, dedup-token capture, the per-pod semaphore, retry/backoff,
+     * and backpressure all stay in front of it unchanged.
+     */
+    setInsertTransport(transport) {
+        this.insertTransport = transport;
+    }
+    hasInsertTransport() {
+        return this.insertTransport !== null;
+    }
+    /** Test hook: override options on the shared instance (e.g. shrink maxWaitMs). */
+    configure(partial) {
+        this.options = Object.assign(Object.assign({}, this.options), partial);
+    }
+    getStats() {
+        let bufferedRows = 0;
+        for (const buffer of this.buffers.values()) {
+            bufferedRows += buffer.rowCount;
+        }
+        return {
+            bufferedRows,
+            pendingRows: this.pendingRows,
+            activeInserts: this.activeInserts,
+        };
+    }
+    /*
+     * Hand a batch of rows to the writer. Takes ownership of the `rows` array —
+     * callers must not mutate it afterwards.
+     *
+     * Awaiting submit() itself is the backpressure point: it resolves once the
+     * rows are accepted into the buffer (immediately under normal load, later
+     * when the pod is at maxPendingRows). The returned `flushed` promise is the
+     * acceptance ack — resolve it into the job's pending-ack list and await
+     * before completing the job.
+     *
+     * All submitters of one table must pass identical clickhouseSettings: a
+     * batch mixes submissions and applies the first submission's settings.
+     */
+    async submit(target, rows, options) {
+        var _a, _b, _c;
+        if (!rows || rows.length === 0) {
+            return { flushed: Promise.resolve() };
+        }
+        this.registerShutdownHandlerOnce();
+        const tableName = target.model.tableName;
+        /*
+         * Capture the deterministic per-job dedup token NOW, synchronously in
+         * the caller's flow: submit() runs inside the job's runWithInsertDedup
+         * scope, while the dispatch that eventually inserts these rows runs on
+         * a timer with no ambient context. Capturing before any await also
+         * keeps token order deterministic across retries of the same payload.
+         */
+        const dedupToken = (_a = options === null || options === void 0 ? void 0 : options.dedupToken) !== null && _a !== void 0 ? _a : nextInsertDedupToken(tableName);
+        this.acceptingSubmits++;
+        try {
+            await this.waitForCapacity();
+            let buffer = this.buffers.get(tableName);
+            if (!buffer) {
+                buffer = {
+                    target,
+                    submissions: [],
+                    rowCount: 0,
+                    timer: null,
+                };
+                this.buffers.set(tableName, buffer);
+            }
+            let resolveAck = () => { };
+            let rejectAck = () => { };
+            const flushed = new Promise((resolve, reject) => {
+                resolveAck = resolve;
+                rejectAck = reject;
+            });
+            buffer.submissions.push({
+                rows,
+                clickhouseSettings: options === null || options === void 0 ? void 0 : options.clickhouseSettings,
+                dedupToken,
+                resolveAck,
+                rejectAck,
+            });
+            buffer.rowCount += rows.length;
+            this.pendingRows += rows.length;
+            if (buffer.rowCount >= this.options.maxBatchRows) {
+                this.cutAndDispatch(tableName, false);
+            }
+            else if (!buffer.timer) {
+                buffer.timer = setTimeout(() => {
+                    const timedBuffer = this.buffers.get(tableName);
+                    if (timedBuffer) {
+                        timedBuffer.timer = null;
+                    }
+                    this.cutAndDispatch(tableName, true);
+                }, this.options.maxWaitMs);
+                /*
+                 * Never keep the process alive just for a pending flush timer —
+                 * graceful shutdown drains buffers explicitly.
+                 */
+                (_c = (_b = buffer.timer).unref) === null || _c === void 0 ? void 0 : _c.call(_b);
+            }
+            return { flushed };
+        }
+        finally {
+            this.acceptingSubmits--;
+        }
+    }
+    /** Force-flush everything buffered and wait for all in-flight inserts to settle. */
+    async flushAll() {
+        /*
+         * Dispatch promises never reject (failures are routed to submission
+         * acks), so a plain all() is safe. Re-cut on EVERY pass: a submit
+         * parked in waitForCapacity() resumes when a completing dispatch
+         * releases capacity — scheduled BEFORE the dispatch promise's own
+         * reactions — and buffers its rows WITHOUT creating a dispatch when
+         * they are below maxBatchRows (its flush timer is unref'd and will
+         * never fire before process exit). Waiting on inFlightDispatches alone
+         * would therefore return with those rows stranded. Exit only when a
+         * fresh cut pass leaves nothing in flight and no submit is still
+         * mid-acceptance; the microtask yield lets a not-yet-buffered submit
+         * (parked only on waitForCapacity's fast-path await) land its rows so
+         * the next pass picks them up.
+         */
+        for (;;) {
+            for (const tableName of Array.from(this.buffers.keys())) {
+                this.cutAndDispatch(tableName, true);
+            }
+            if (this.inFlightDispatches.size > 0) {
+                await Promise.all(Array.from(this.inFlightDispatches));
+                continue;
+            }
+            if (this.acceptingSubmits === 0) {
+                return;
+            }
+            await Promise.resolve();
+        }
+    }
+    registerShutdownHandlerOnce() {
+        if (this.shutdownHandlerRegistered) {
+            return;
+        }
+        this.shutdownHandlerRegistered = true;
+        GracefulShutdown.registerHandler("TelemetryFanInWriter", ShutdownPriority.Buffers, async () => {
+            await this.flushAll();
+        });
+    }
+    async waitForCapacity() {
+        while (this.pendingRows >= this.options.maxPendingRows) {
+            await new Promise((resolve) => {
+                this.capacityWaiters.push(resolve);
+            });
+        }
+    }
+    releaseCapacity() {
+        /*
+         * Wake all waiters; each re-checks the high-water mark in its
+         * waitForCapacity loop, so overshoot stays bounded to one submission
+         * per waiter.
+         */
+        const waiters = this.capacityWaiters;
+        this.capacityWaiters = [];
+        for (const wake of waiters) {
+            wake();
+        }
+    }
+    async acquireInsertSlot() {
+        while (this.activeInserts >= this.options.maxConcurrentInserts) {
+            await new Promise((resolve) => {
+                this.insertSlotWaiters.push(resolve);
+            });
+        }
+        this.activeInserts++;
+    }
+    releaseInsertSlot() {
+        this.activeInserts--;
+        const next = this.insertSlotWaiters.shift();
+        if (next) {
+            next();
+        }
+    }
+    /*
+     * Cut batches from a table buffer and dispatch them. When `drain` is false,
+     * only full batches (>= maxBatchRows) are cut — the remainder keeps waiting
+     * for more rows or the timer. When true, everything goes.
+     *
+     * A batch is a FIFO run of WHOLE submissions packed up to maxBatchRows
+     * (a single oversized submission forms its own batch), so an insert failure
+     * maps 1:1 onto the submissions it contained.
+     */
+    cutAndDispatch(tableName, drain) {
+        const buffer = this.buffers.get(tableName);
+        if (!buffer) {
+            return;
+        }
+        while (buffer.rowCount >= this.options.maxBatchRows ||
+            (drain && buffer.rowCount > 0)) {
+            const batch = [];
+            let batchRows = 0;
+            while (buffer.submissions.length > 0) {
+                const next = buffer.submissions[0];
+                if (batch.length > 0 &&
+                    batchRows + next.rows.length > this.options.maxBatchRows) {
+                    break;
+                }
+                buffer.submissions.shift();
+                batch.push(next);
+                batchRows += next.rows.length;
+                if (batchRows >= this.options.maxBatchRows) {
+                    break;
+                }
+            }
+            if (batch.length === 0) {
+                break;
+            }
+            buffer.rowCount -= batchRows;
+            const dispatch = this.dispatchInsert(buffer.target, tableName, batch, batchRows);
+            this.inFlightDispatches.add(dispatch);
+            dispatch.finally(() => {
+                this.inFlightDispatches.delete(dispatch);
+            });
+        }
+        if (buffer.rowCount === 0 && buffer.timer) {
+            clearTimeout(buffer.timer);
+            buffer.timer = null;
+        }
+    }
+    /*
+     * Insert one batch while holding a single insert slot. Submissions that
+     * carry a deterministic per-job dedup token are inserted INDIVIDUALLY
+     * under their own token — this is what keeps BullMQ job retries
+     * duplicate-free: the retry re-derives byte-identical tokens and
+     * ClickHouse drops blocks a prior attempt already landed. Merging them
+     * under one shared cross-job token would break that (a token dedups by
+     * token, not content, so a differently-composed block under a reused
+     * token would silently drop other jobs' rows — the same hazard that kept
+     * the probe paths off dedup tokens historically). Untokened submissions
+     * are merged into one insert under a minted per-batch token.
+     *
+     * The tokened inserts run sequentially within this one slot, so the
+     * per-pod concurrency cap — the property that fixes the
+     * max_concurrent_queries incident — is unaffected; only the statement
+     * count returns to master's per-chunk granularity, and ClickHouse's
+     * async-insert coalescing (async_insert=1) still assembles fat parts
+     * server-side.
+     *
+     * Never rejects — outcomes are routed to the submissions' acks.
+     */
+    async dispatchInsert(target, tableName, batch, batchRows) {
+        await this.acquireInsertSlot();
+        try {
+            const untokened = [];
+            for (const submission of batch) {
+                if (submission.dedupToken) {
+                    await this.insertGroupWithRetry(target, tableName, [submission], submission.dedupToken);
+                }
+                else {
+                    untokened.push(submission);
+                }
+            }
+            if (untokened.length > 0) {
+                await this.insertGroupWithRetry(target, tableName, untokened, `fanin:${tableName}:${UUID.generateTimeOrdered()}`);
+            }
+        }
+        finally {
+            this.pendingRows -= batchRows;
+            this.releaseCapacity();
+            this.releaseInsertSlot();
+        }
+    }
+    /*
+     * Insert one token group, retrying transient failures with full-jitter
+     * exponential backoff and the SAME dedup token — an ambiguous failure
+     * (timeout, connection reset) whose rows actually landed is dropped
+     * server-side on the retry. The insert slot is held across backoff sleeps
+     * on purpose: when ClickHouse is saturated, offering it MORE concurrent
+     * load from this pod is the failure mode this component exists to
+     * prevent. Never rejects — outcomes are routed to the group's acks, so
+     * one group's definitive failure does not fail the batch's other groups.
+     */
+    async insertGroupWithRetry(target, tableName, group, dedupToken) {
+        var _a, _b;
+        const rows = [];
+        for (const submission of group) {
+            for (const row of submission.rows) {
+                rows.push(row);
+            }
+        }
+        const clickhouseSettings = (_a = group[0]) === null || _a === void 0 ? void 0 : _a.clickhouseSettings;
+        const sleep = (_b = this.options.sleep) !== null && _b !== void 0 ? _b : ((ms) => {
+            return Sleep.sleep(ms);
+        });
+        let lastError = null;
+        let attemptsMade = 0;
+        for (let attempt = 1; attempt <= this.options.retryMaxAttempts; attempt++) {
+            attemptsMade = attempt;
+            try {
+                if (this.insertTransport) {
+                    await this.insertTransport(target, rows, {
+                        dedupToken,
+                        clickhouseSettings,
+                    });
+                }
+                else {
+                    await target.insertJsonRows(rows, {
+                        dedupToken,
+                        clickhouseSettings,
+                    });
+                }
+                for (const submission of group) {
+                    submission.resolveAck();
+                }
+                if (attempt > 1) {
+                    logger.info(`TelemetryFanInWriter: insert into ${tableName} (${rows.length} rows) succeeded on attempt ${attempt}.`);
+                }
+                return;
+            }
+            catch (err) {
+                lastError = err;
+                if (!isRetryableInsertError(err) ||
+                    attempt === this.options.retryMaxAttempts) {
+                    break;
+                }
+                const expDelay = Math.min(this.options.retryMaxDelayMs, this.options.retryBaseDelayMs * Math.pow(2, attempt - 1));
+                const jitteredDelay = Math.ceil(Math.random() * expDelay);
+                logger.warn(`TelemetryFanInWriter: transient insert failure on ${tableName} (${rows.length} rows, attempt ${attempt}/${this.options.retryMaxAttempts}); retrying with the same dedup token in ${jitteredDelay}ms: ${err instanceof Error ? err.message : String(err)}`);
+                await sleep(jitteredDelay);
+            }
+        }
+        const finalError = new FanInInsertError({
+            tableName,
+            attempts: attemptsMade,
+            cause: lastError,
+        });
+        logger.error(`TelemetryFanInWriter: giving up on insert into ${tableName} (${rows.length} rows); failing ${group.length} submission(s).`);
+        logger.error(finalError);
+        for (const submission of group) {
+            submission.rejectAck(finalError);
+        }
+    }
+}
+/*
+ * Register a durability ack on a job's pending-ack list, wrapped in the
+ * service's storage error class and pre-observed: a mid-job batch failure
+ * would otherwise reject the stored promise long before the job's final
+ * `await Promise.all(pendingAcks)` reaches it, firing the process-level
+ * unhandledRejection logger once per ack. The no-op catch marks the promise
+ * handled without swallowing anything — the rejection is still delivered at
+ * the job's await point, failing the job so BullMQ retries the payload.
+ */
+export function pushObservedAck(pendingAcks, flushed, wrapError) {
+    const ack = flushed.catch((error) => {
+        throw wrapError(error);
+    });
+    ack.catch(() => {
+        // Pre-observed; delivered for real at the job's await point.
+    });
+    pendingAcks.push(ack);
+}
+const defaultWriter = new TelemetryFanInWriter(readFanInWriterOptionsFromEnv());
+export default defaultWriter;
+//# sourceMappingURL=TelemetryFanInWriter.js.map

package/build/dist/Server/Utils/Telemetry/TelemetryFanInWriter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TelemetryFanInWriter.js","sourceRoot":"","sources":["../../../../../Server/Utils/Telemetry/TelemetryFanInWriter.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,OAAO,KAAK,MAAM,sBAAsB,CAAC;AACzC,OAAO,IAAI,MAAM,qBAAqB,CAAC;AAEvC,OAAO,gBAAgB,EAAE,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,eAAe,EAA2B,MAAM,oBAAoB,CAAC;AAwH9E;;;;;;GAMG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAmB,OAAe;QAChC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAQzC,YAAmB,IAIlB;QACC,MAAM,YAAY,GAChB,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxE,KAAK,CACH,sBAAsB,IAAI,CAAC,SAAS,iBAAiB,IAAI,CAAC,QAAQ,gBAAgB,YAAY,EAAE,CACjG,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;QAC7B,IAAI,IAAI,CAAC,KAAK,YAAY,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,GAAG,GAAG,IAAI,CAAC,KAAK,gBAAgB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/D,CAAC;IACH,CAAC;CACF;AAyBD;;;;;;;;;GASG;AACH,MAAM,0BAA0B,GAAgB,IAAI,GAAG,CAAC;IACtD,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;CACN,CAAC,CAAC;AAEH,MAAM,wBAAwB,GAAW,OAAO,CAAC;AAEjD,MAAM,sBAAsB,GAAgB,IAAI,GAAG,CAAC;IAClD,YAAY;IACZ,cAAc;IACd,cAAc;IACd,WAAW;IACX,OAAO;IACP,WAAW;CACZ,CAAC,CAAC;AAEH,MAAM,UAAU,sBAAsB,CAAC,GAAY;IACjD,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;QACnC,OAAO,0BAA0B,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,GAAG,YAAY,oBAAoB,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,iEAAiE;QACjE,IAAI,GAAG,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAa,GAA6B,CAAC,MAAM,CAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,IAAI,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YACD,wEAAwE;YACxE,IAAI,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,OAAO,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QACD;;;;;WAKG;QACH,IACE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;YACrC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EACtC,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAID,MAAM,eAAe,GAAiB,CACpC,MAAc,EACd,YAAoB,EACZ,EAAE;IACV,MAAM,GAAG,GAAuB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,MAAM,MAAM,GAAW,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;AAC9D,CAAC,CAAC;AAEF,MAAM,UAAU,6BAA6B;IAC3C,OAAO;QACL,YAAY,EAAE,eAAe,CAAC,gCAAgC,EAAE,MAAO,CAAC;QACxE,SAAS,EAAE,eAAe,CAAC,6BAA6B,EAAE,IAAI,CAAC;QAC/D,oBAAoB,EAAE,eAAe,CACnC,wCAAwC,EACxC,CAAC,CACF;QACD;;;;;WAKG;QACH,cAAc,EAAE,eAAe,CAC7B,kCAAkC,EAClC,MAAO,CACR;QACD,gBAAgB,EAAE,eAAe,CAAC,oCAAoC,EAAE,CAAC,CAAC;QAC1E,gBAAgB,EAAE,eAAe,CAC/B,qCAAqC,EACrC,GAAG,CACJ;QACD,eAAe,EAAE,eAAe,CAC9B,oCAAoC,EACpC,KAAM,CACP;KACF,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,oBAAoB;IAkB/B,YAAmB,OAA2B;QAhBtC,YAAO,GAA6B,IAAI,GAAG,EAAE,CAAC;QAC9C,gBAAW,GAAW,CAAC,CAAC;QACxB,oBAAe,GAAsB,EAAE,CAAC;QACxC,kBAAa,GAAW,CAAC,CAAC;QAC1B,sBAAiB,GAAsB,EAAE,CAAC;QAC1C,uBAAkB,GAAuB,IAAI,GAAG,EAAE,CAAC;QAC3D;;;;;WAKG;QACK,qBAAgB,GAAW,CAAC,CAAC;QAC7B,8BAAyB,GAAY,KAAK,CAAC;QAC3C,oBAAe,GAAgC,IAAI,CAAC;QAG1D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;;;;OAKG;IACI,kBAAkB,CAAC,SAAsC;QAC9D,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;IACnC,CAAC;IAEM,kBAAkB;QACvB,OAAO,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;IACvC,CAAC;IAED,kFAAkF;IAC3E,SAAS,CAAC,OAAoC;QACnD,IAAI,CAAC,OAAO,mCAAQ,IAAI,CAAC,OAAO,GAAK,OAAO,CAAE,CAAC;IACjD,CAAC;IAEM,QAAQ;QAKb,IAAI,YAAY,GAAW,CAAC,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,YAAY,IAAI,MAAM,CAAC,QAAQ,CAAC;QAClC,CAAC;QACD,OAAO;YACL,YAAY;YACZ,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;;OAYG;IACI,KAAK,CAAC,MAAM,CACjB,MAAyB,EACzB,IAAuB,EACvB,OASC;;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAEnC,MAAM,SAAS,GAAW,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;QAEjD;;;;;;WAMG;QACH,MAAM,UAAU,GACd,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,UAAU,mCAAI,oBAAoB,CAAC,SAAS,CAAC,CAAC;QAEzD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAE7B,IAAI,MAAM,GAA4B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG;oBACP,MAAM;oBACN,WAAW,EAAE,EAAE;oBACf,QAAQ,EAAE,CAAC;oBACX,KAAK,EAAE,IAAI;iBACZ,CAAC;gBACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACtC,CAAC;YAED,IAAI,UAAU,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;YACtC,IAAI,SAAS,GAAyB,GAAG,EAAE,GAAE,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAkB,IAAI,OAAO,CACxC,CAAC,OAAmB,EAAE,MAA4B,EAAE,EAAE;gBACpD,UAAU,GAAG,OAAO,CAAC;gBACrB,SAAS,GAAG,MAAM,CAAC;YACrB,CAAC,CACF,CAAC;YAEF,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC;gBACtB,IAAI;gBACJ,kBAAkB,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,kBAAkB;gBAC/C,UAAU;gBACV,UAAU;gBACV,SAAS;aACV,CAAC,CAAC;YACH,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC;YAC/B,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC;YAEhC,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBACjD,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YACxC,CAAC;iBAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACzB,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC7B,MAAM,WAAW,GACf,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;oBAC9B,IAAI,WAAW,EAAE,CAAC;wBAChB,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC;oBAC3B,CAAC;oBACD,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBACvC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC3B;;;mBAGG;gBACH,MAAA,MAAA,MAAM,CAAC,KAAK,EAAC,KAAK,kDAAI,CAAC;YACzB,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,CAAC;QACrB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,oFAAoF;IAC7E,KAAK,CAAC,QAAQ;QACnB;;;;;;;;;;;;;WAaG;QACH,SAAS,CAAC;YACR,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBACxD,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACvC,CAAC;YAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;gBACvD,SAAS;YACX,CAAC;YAED,IAAI,IAAI,CAAC,gBAAgB,KAAK,CAAC,EAAE,CAAC;gBAChC,OAAO;YACT,CAAC;YAED,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,2BAA2B;QACjC,IAAI,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnC,OAAO;QACT,CAAC;QACD,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC;QACtC,gBAAgB,CAAC,eAAe,CAC9B,sBAAsB,EACtB,gBAAgB,CAAC,OAAO,EACxB,KAAK,IAAI,EAAE;YACT,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACxB,CAAC,CACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,OAAO,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACvD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAmB,EAAE,EAAE;gBAC9C,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,eAAe;QACrB;;;;WAIG;QACH,MAAM,OAAO,GAAsB,IAAI,CAAC,eAAe,CAAC;QACxD,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,OAAO,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;YAC/D,MAAM,IAAI,OAAO,CAAO,CAAC,OAAmB,EAAE,EAAE;gBAC9C,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,IAAI,GAA6B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QACtE,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,cAAc,CAAC,SAAiB,EAAE,KAAc;QACtD,MAAM,MAAM,GAA4B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,OACE,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY;YAC5C,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,EAC9B,CAAC;YACD,MAAM,KAAK,GAA6B,EAAE,CAAC;YAC3C,IAAI,SAAS,GAAW,CAAC,CAAC;YAE1B,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAI,GAAsB,MAAM,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;gBACvD,IACE,KAAK,CAAC,MAAM,GAAG,CAAC;oBAChB,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EACxD,CAAC;oBACD,MAAM;gBACR,CAAC;gBACD,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC9B,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;oBAC3C,MAAM;gBACR,CAAC;YACH,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM;YACR,CAAC;YAED,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC;YAE7B,MAAM,QAAQ,GAAkB,IAAI,CAAC,cAAc,CACjD,MAAM,CAAC,MAAM,EACb,SAAS,EACT,KAAK,EACL,SAAS,CACV,CAAC;YACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE;gBACpB,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC1C,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC3B,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACK,KAAK,CAAC,cAAc,CAC1B,MAAyB,EACzB,SAAiB,EACjB,KAA+B,EAC/B,SAAiB;QAEjB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,SAAS,GAA6B,EAAE,CAAC;YAE/C,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;gBAC/B,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;oBAC1B,MAAM,IAAI,CAAC,oBAAoB,CAC7B,MAAM,EACN,SAAS,EACT,CAAC,UAAU,CAAC,EACZ,UAAU,CAAC,UAAU,CACtB,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,oBAAoB,CAC7B,MAAM,EACN,SAAS,EACT,SAAS,EACT,SAAS,SAAS,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CACnD,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC;YAC9B,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACK,KAAK,CAAC,oBAAoB,CAChC,MAAyB,EACzB,SAAiB,EACjB,KAA+B,EAC/B,UAAkB;;QAElB,MAAM,IAAI,GAAsB,EAAE,CAAC;QACnC,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;YAC/B,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;gBAClC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QAED,MAAM,kBAAkB,GACtB,MAAA,KAAK,CAAC,CAAC,CAAC,0CAAE,kBAAkB,CAAC;QAC/B,MAAM,KAAK,GACT,MAAA,IAAI,CAAC,OAAO,CAAC,KAAK,mCAClB,CAAC,CAAC,EAAU,EAAiB,EAAE;YAC7B,OAAO,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEL,IAAI,SAAS,GAAY,IAAI,CAAC;QAC9B,IAAI,YAAY,GAAW,CAAC,CAAC;QAE7B,KACE,IAAI,OAAO,GAAW,CAAC,EACvB,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,EACxC,OAAO,EAAE,EACT,CAAC;YACD,YAAY,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACzB,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE;wBACvC,UAAU;wBACV,kBAAkB;qBACnB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE;wBAChC,UAAU;wBACV,kBAAkB;qBACnB,CAAC,CAAC;gBACL,CAAC;gBAED,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;oBAC/B,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1B,CAAC;gBACD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAChB,MAAM,CAAC,IAAI,CACT,qCAAqC,SAAS,KAAK,IAAI,CAAC,MAAM,+BAA+B,OAAO,GAAG,CACxG,CAAC;gBACJ,CAAC;gBACD,OAAO;YACT,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,CAAC;gBAEhB,IACE,CAAC,sBAAsB,CAAC,GAAG,CAAC;oBAC5B,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC,gBAAgB,EACzC,CAAC;oBACD,MAAM;gBACR,CAAC;gBAED,MAAM,QAAQ,GAAW,IAAI,CAAC,GAAG,CAC/B,IAAI,CAAC,OAAO,CAAC,eAAe,EAC5B,IAAI,CAAC,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CACzD,CAAC;gBACF,MAAM,aAAa,GAAW,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC;gBAClE,MAAM,CAAC,IAAI,CACT,qDAAqD,SAAS,KAAK,IAAI,CAAC,MAAM,kBAAkB,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,4CAA4C,aAAa,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC3P,CAAC;gBACF,MAAM,KAAK,CAAC,aAAa,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqB,IAAI,gBAAgB,CAAC;YACxD,SAAS;YACT,QAAQ,EAAE,YAAY;YACtB,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CACV,kDAAkD,SAAS,KAAK,IAAI,CAAC,MAAM,mBAAmB,KAAK,CAAC,MAAM,iBAAiB,CAC5H,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACzB,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;YAC/B,UAAU,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAiC,EACjC,OAAsB,EACtB,SAAgC;IAEhC,MAAM,GAAG,GAAkB,OAAO,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;QACxD,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE;QACb,6DAA6D;IAC/D,CAAC,CAAC,CAAC;IACH,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,MAAM,aAAa,GAAyB,IAAI,oBAAoB,CAClE,6BAA6B,EAAE,CAChC,CAAC;AAEF,eAAe,aAAa,CAAC"}

package/build/dist/Server/Utils/Telemetry/TelemetryWriterClient.js

@@ -0,0 +1,165 @@
+import ClusterKeyAuthorization from "../../Middleware/ClusterKeyAuthorization";
+import HTTPErrorResponse from "../../../Types/API/HTTPErrorResponse";
+import URL from "../../../Types/API/URL";
+import API from "../../../Utils/API";
+import { TransientInsertError, } from "./TelemetryFanInWriter";
+/*
+ * Worker-side client for the telemetry-writer tier.
+ *
+ * When TELEMETRY_WRITER_URL is set on a pod, its fan-in writer delivers
+ * every token group as one HTTP POST to the writer tier instead of
+ * inserting into ClickHouse itself. This decouples worker replica count
+ * from ClickHouse INSERT concurrency: telemetry insert load on ClickHouse
+ * is (writerReplicas × TELEMETRY_FANIN_MAX_CONCURRENT_INSERTS) regardless
+ * of how far the worker fleet scales out. (Reads and low-volume direct
+ * inserts that never go through the fan-in writer still use this pod's own
+ * ClickHouse pools.)
+ *
+ * Failure classification maps onto the fan-in writer's retry semantics:
+ * every payload carries a dedup token that is byte-identical across
+ * retries, so any AMBIGUOUS outcome (network error, timeout, writer pod
+ * dying mid-request) is safe to classify transient — a retry of a payload
+ * that actually landed is dropped server-side by ClickHouse. Only a
+ * definitive verdict from the writer (2xx success, 4xx/500 permanent
+ * failure) is treated as final.
+ */
+export const TELEMETRY_WRITER_INSERT_ROUTE = "/telemetry-writer/insert";
+export function getTelemetryWriterUrl() {
+    const raw = process.env["TELEMETRY_WRITER_URL"];
+    const trimmed = (raw || "").trim();
+    if (!trimmed) {
+        return null;
+    }
+    // Normalize away a trailing slash so route concatenation is predictable.
+    return trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed;
+}
+export function isTelemetryWriterForwardingEnabled() {
+    return getTelemetryWriterUrl() !== null;
+}
+export function getTelemetryWriterRequestTimeoutMs() {
+    const raw = process.env["TELEMETRY_WRITER_REQUEST_TIMEOUT_MS"];
+    const parsed = parseInt(raw || "", 10);
+    /*
+     * Generous (the writer holds the request until ClickHouse accepts the
+     * rows, which under load includes its batching wait and retry loop) but
+     * deliberately under the BullMQ job lock: the fan-in writer makes up to
+     * 6 attempts, and 6 x 90s + ~8s backoff ≈ 9.2 min must stay inside
+     * TELEMETRY_LOCK_DURATION_MS (10 min) — otherwise a saturated writer
+     * tier stalls jobs into redelivery churn instead of failing them
+     * cleanly for retry.
+     */
+    return !isNaN(parsed) && parsed > 0 ? parsed : 90000;
+}
+export function getTelemetryWriterMaxBodyBytes() {
+    const raw = process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"];
+    const parsed = parseInt(raw || "", 10);
+    /*
+     * Stay well under the writer route's 50 MB express.json limit (plus
+     * headroom for headers/encoding). Bodies above this are split before
+     * sending — a 413 would be classified permanent and drop rows.
+     */
+    return !isNaN(parsed) && parsed > 0 ? parsed : 30000000;
+}
+const defaultPost = async (args) => {
+    let response;
+    try {
+        response = await API.post({
+            url: URL.fromString(args.url),
+            data: args.body,
+            headers: Object.assign({}, ClusterKeyAuthorization.getClusterKeyHeaders()),
+            options: {
+                timeout: args.timeoutMs,
+                skipAuthRefresh: true,
+            },
+        });
+    }
+    catch (err) {
+        /*
+         * API.post throws (APIException) when no HTTP response exists at all —
+         * connection refused, reset, DNS, timeout. Outcome unknown; statusCode 0
+         * classifies it transient.
+         */
+        return {
+            statusCode: 0,
+            errorMessage: err instanceof Error ? err.message : String(err),
+        };
+    }
+    if (response instanceof HTTPErrorResponse) {
+        return {
+            statusCode: response.statusCode,
+            errorMessage: response.message,
+        };
+    }
+    return { statusCode: response.statusCode };
+};
+/*
+ * Writer-tier verdicts that mean "retry with the same token": overload
+ * shedding (429) and gateway/unavailability statuses where the request may
+ * or may not have been processed.
+ */
+const TRANSIENT_HTTP_STATUSES = new Set([429, 502, 503, 504]);
+function isDefinitiveStatus(statusCode) {
+    /*
+     * Anything outside real HTTP semantics (0, -1, undefined coerced) is a
+     * network-level failure with an unknown outcome — transient by the dedup
+     * token argument above.
+     */
+    return statusCode >= 100 && statusCode < 600;
+}
+export function createTelemetryWriterTransport(postFn) {
+    const post = postFn !== null && postFn !== void 0 ? postFn : defaultPost;
+    const postGroup = async (tableName, rows, dedupToken, clickhouseSettings, baseUrl, maxBodyBytes) => {
+        const body = Object.assign({ tableName: tableName, rows: rows, dedupToken: dedupToken }, (clickhouseSettings
+            ? { clickhouseSettings: clickhouseSettings }
+            : {}));
+        /*
+         * Byte-aware splitting: the writer route sits behind a 50 MB JSON
+         * parser, and a 413 is (correctly) classified permanent — so an
+         * oversized group would be dropped after retries, a failure mode the
+         * direct ClickHouse path never had. Halve deterministically instead:
+         * a BullMQ retry re-produces byte-identical rows, hence the identical
+         * split and identical derived tokens ("<token>#0"/"<token>#1"), so
+         * idempotence is preserved (async-insert dedup is a content hash per
+         * insert body, and the distinct suffixes keep tokens collision-free
+         * for any future sync-insert mode). Halves post sequentially — this
+         * all happens under one fan-in insert slot, and splitting must not
+         * multiply the pod's concurrent load.
+         */
+        if (rows.length > 1) {
+            const bodyBytes = Buffer.byteLength(JSON.stringify(body), "utf8");
+            if (bodyBytes > maxBodyBytes) {
+                const mid = Math.floor(rows.length / 2);
+                await postGroup(tableName, rows.slice(0, mid), `${dedupToken}#0`, clickhouseSettings, baseUrl, maxBodyBytes);
+                await postGroup(tableName, rows.slice(mid), `${dedupToken}#1`, clickhouseSettings, baseUrl, maxBodyBytes);
+                return;
+            }
+        }
+        const result = await post({
+            url: `${baseUrl}${TELEMETRY_WRITER_INSERT_ROUTE}`,
+            body: body,
+            timeoutMs: getTelemetryWriterRequestTimeoutMs(),
+        });
+        if (result.statusCode >= 200 && result.statusCode < 300) {
+            return;
+        }
+        const detail = `telemetry-writer insert into ${tableName} (${rows.length} rows) returned status ${result.statusCode}${result.errorMessage ? `: ${result.errorMessage}` : ""}`;
+        if (TRANSIENT_HTTP_STATUSES.has(result.statusCode) ||
+            !isDefinitiveStatus(result.statusCode)) {
+            throw new TransientInsertError(detail);
+        }
+        throw new Error(detail);
+    };
+    return async (target, rows, options) => {
+        const baseUrl = getTelemetryWriterUrl();
+        if (!baseUrl) {
+            /*
+             * The transport is only installed when the URL is configured; losing
+             * it mid-flight (env mutation in tests) is a permanent config error,
+             * not something a same-token retry can fix.
+             */
+            throw new Error("TelemetryWriterClient: TELEMETRY_WRITER_URL is not configured.");
+        }
+        await postGroup(target.model.tableName, rows, options.dedupToken, options.clickhouseSettings, baseUrl, getTelemetryWriterMaxBodyBytes());
+    };
+}
+//# sourceMappingURL=TelemetryWriterClient.js.map