@oneuptime/common 12.0.5 → 12.0.7

package/build/dist/Server/Utils/InstanceHealth/TelemetryIngestion.js

@@ -0,0 +1,406 @@
+import { ClickhouseDatabase } from "../../EnvironmentConfig";
+import { ClickhouseAppInstance, } from "../../Infrastructure/ClickhouseDatabase";
+import AnalyticsTableName from "../../../Types/AnalyticsDatabase/AnalyticsTableName";
+import logger from "../Logger";
+import { getClickhouseClusterName, getStorageTableName, } from "../AnalyticsDatabase/ClusterConfig";
+/*
+ * -----------------------------------------------------------------------------
+ * Telemetry ingestion probes for the master-admin OneUptime Health dashboard.
+ *
+ * Two views over the same three tables:
+ *
+ *   - by SIGNAL   — how many log / metric / trace rows landed in ClickHouse over
+ *                   the last minute, hour and day, plus each signal's actual
+ *                   (uncompressed) footprint. This answers "is the pipeline
+ *                   flowing, and how fast".
+ *   - by PROJECT  — the same three windows split per tenant, which answers
+ *                   "who is sending it" — the question behind a sudden ingest
+ *                   spike, a noisy-neighbour investigation or a capacity
+ *                   conversation with one customer.
+ *
+ * Only counts and size metadata are read; no telemetry row data ever leaves
+ * ClickHouse through here. Every query is capped with max_execution_time so a
+ * dashboard poll can never pin the cluster.
+ * -----------------------------------------------------------------------------
+ */
+export var TelemetryIngestionSignal;
+(function (TelemetryIngestionSignal) {
+    TelemetryIngestionSignal["Logs"] = "Logs";
+    TelemetryIngestionSignal["Metrics"] = "Metrics";
+    TelemetryIngestionSignal["Traces"] = "Traces";
+})(TelemetryIngestionSignal || (TelemetryIngestionSignal = {}));
+/*
+ * The three telemetry signals OneUptime ingests into ClickHouse and the
+ * event-time column each table is partitioned + primary-key ordered on. We count
+ * ingestion on THIS column (never `createdAt`) precisely because it is the
+ * partition key (toYYYYMMDD) and the leading primary-key column: ClickHouse can
+ * prune to the last day's partitions and use the primary index, so the count
+ * stays cheap even on multi-billion-row tables. `createdAt` — the true write
+ * time — is unindexed and unpartitioned here, so filtering on it would force a
+ * full-table scan; for a live pipeline event-time and write-time agree to within
+ * seconds, and the only divergence (historical backfill) is not what a
+ * "current ingestion rate" view is meant to show.
+ */
+export const TELEMETRY_INGESTION_SIGNALS = [
+    {
+        telemetryType: TelemetryIngestionSignal.Logs,
+        table: AnalyticsTableName.Log,
+        timeColumn: "time",
+    },
+    {
+        telemetryType: TelemetryIngestionSignal.Metrics,
+        table: AnalyticsTableName.Metric,
+        timeColumn: "time",
+    },
+    {
+        telemetryType: TelemetryIngestionSignal.Traces,
+        table: AnalyticsTableName.Span,
+        timeColumn: "startTime",
+    },
+];
+/*
+ * How many projects one signal's GROUP BY may return. Every OneUptime instance
+ * we have seen has far fewer active tenants than this, so the cap is a runaway
+ * guard rather than a real ceiling — but when it does bite, the caller is told
+ * (`truncated`) so the page can say the list is partial instead of quietly
+ * dropping tenants off the bottom.
+ */
+export const MAX_PROJECTS_PER_SIGNAL = 500;
+// Every diagnostic query here is read-only, and none of them may outlive this.
+const QUERY_TIMEOUT_IN_SECONDS = 10;
+function escapeStringLiteral(value) {
+    return value.replace(/'/g, "''");
+}
+function quoteIdentifier(value) {
+    return `\`${value.replace(/`/g, "``")}\``;
+}
+/*
+ * ClickHouse returns UInt64 aggregates as JSON strings. A value we could not
+ * read stays null and renders as "—" rather than as a confident zero.
+ */
+export function toCountOrNull(value) {
+    if (value === null || value === undefined || value === "") {
+        return null;
+    }
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : null;
+}
+/*
+ * Sum counts where null means "unknown". A total is only meaningful when at
+ * least one input reported a number, so an all-unknown sum stays unknown
+ * instead of collapsing to 0.
+ */
+export function sumCountsOrNull(values) {
+    const known = values.filter((value) => {
+        return value !== null;
+    });
+    if (known.length === 0) {
+        return null;
+    }
+    return known.reduce((sum, value) => {
+        return sum + value;
+    }, 0);
+}
+/*
+ * Counts for one table across the three windows. All three come from ONE scan:
+ * the WHERE clause bounds it to the last day (so only the last day's partitions
+ * are touched) and countIf picks the smaller windows out of that same pass. The
+ * upper `<= now()` bound stops a future-dated event from inflating the counts.
+ */
+export function buildSignalIngestionQuery(data) {
+    const timeColumn = quoteIdentifier(data.spec.timeColumn);
+    return ("SELECT " +
+        `countIf(${timeColumn} >= now() - INTERVAL 1 MINUTE) AS last_minute, ` +
+        `countIf(${timeColumn} >= now() - INTERVAL 1 HOUR) AS last_hour, ` +
+        "count() AS last_day " +
+        `FROM ${quoteIdentifier(data.databaseName)}.${quoteIdentifier(data.spec.table)} ` +
+        `WHERE ${timeColumn} >= now() - INTERVAL 1 DAY AND ${timeColumn} <= now()`);
+}
+/*
+ * The same three windows, split by tenant. projectId is the LEADING primary-key
+ * column of all three telemetry tables, so this is the one dimension the tables
+ * can be grouped on without paying for a full scan of the day's partitions.
+ *
+ * Ordered by the widest window descending so that, if the cap ever bites, it is
+ * the smallest tenants that fall off rather than an arbitrary slice; projectId
+ * breaks ties so the list is stable between refreshes.
+ */
+export function buildProjectIngestionQuery(data) {
+    const timeColumn = quoteIdentifier(data.spec.timeColumn);
+    return ("SELECT projectId AS project_id, " +
+        `countIf(${timeColumn} >= now() - INTERVAL 1 MINUTE) AS last_minute, ` +
+        `countIf(${timeColumn} >= now() - INTERVAL 1 HOUR) AS last_hour, ` +
+        "count() AS last_day " +
+        `FROM ${quoteIdentifier(data.databaseName)}.${quoteIdentifier(data.spec.table)} ` +
+        `WHERE ${timeColumn} >= now() - INTERVAL 1 DAY AND ${timeColumn} <= now() ` +
+        "GROUP BY projectId " +
+        "ORDER BY last_day DESC, project_id ASC " +
+        `LIMIT ${Math.max(1, Math.floor(data.limit))}`);
+}
+/*
+ * Total ACTUAL (uncompressed) data volume per telemetry table, read from
+ * system.parts metadata — the real data size, not the compressed bytes_on_disk.
+ * It is a metadata aggregate (no data scan), so it stays cheap regardless of
+ * table size.
+ *
+ * Two cluster subtleties are handled here:
+ *
+ *   1. The app-facing table names (LogItemV3, …) are Distributed wrappers, which
+ *      hold NO parts of their own — the rows live in the per-shard local storage
+ *      tables (`<tableName>Local`). So we filter system.parts on the *Local
+ *      names (and key the size map by them); the Distributed names would match
+ *      nothing and leave every "Actual size" cell blank.
+ *
+ *   2. system.parts is node-local, so a plain read would only size the connected
+ *      node's shard. We read it through `cluster(<name>, system.parts)` which —
+ *      like the Distributed read path the ingestion counts use — hits ONE replica
+ *      per shard, so sum(...) GROUP BY table totals every shard exactly once (no
+ *      replica double-counting; clusterAllReplicas would overcount by the
+ *      replication factor). On a single-node "cluster of one" this equals
+ *      reading system.parts directly.
+ */
+export function buildTelemetryStorageSizeQuery(data) {
+    const tableList = data.storageTableNames
+        .map((tableName) => {
+        return `'${escapeStringLiteral(tableName)}'`;
+    })
+        .join(", ");
+    return ("SELECT table, sum(data_uncompressed_bytes) AS uncompressed_bytes " +
+        `FROM cluster('${escapeStringLiteral(data.clusterName)}', system.parts) ` +
+        `WHERE active AND database = '${escapeStringLiteral(data.databaseName)}' AND table IN (${tableList}) ` +
+        "GROUP BY table");
+}
+/*
+ * Fold the three per-signal GROUP BY results into one row per project.
+ *
+ * A project that appears in one signal's rows but not another's really did
+ * ingest zero of that signal, so it gets 0 — but only when that signal's query
+ * SUCCEEDED. When the query failed the signal is unknown for every project and
+ * stays null, which is what keeps a missing table from reading as "this tenant
+ * stopped sending traces".
+ *
+ * Rows come back ordered by the day window descending, the order the page shows
+ * them in; projectId breaks ties so the order does not shuffle between polls.
+ */
+export function mergeProjectIngestionRows(signalResults) {
+    const countsByProject = new Map();
+    for (const signalResult of signalResults) {
+        if (!signalResult.available) {
+            continue;
+        }
+        for (const row of signalResult.rows) {
+            const byTelemetryType = countsByProject.get(row.projectId) ||
+                new Map();
+            byTelemetryType.set(signalResult.telemetryType, {
+                lastMinute: row.lastMinute,
+                lastHour: row.lastHour,
+                lastDay: row.lastDay,
+            });
+            countsByProject.set(row.projectId, byTelemetryType);
+        }
+    }
+    const projects = [];
+    for (const [projectId, byTelemetryType] of countsByProject) {
+        const signals = signalResults.map((signalResult) => {
+            const counts = signalResult.available
+                ? byTelemetryType.get(signalResult.telemetryType) || {
+                    lastMinute: 0,
+                    lastHour: 0,
+                    lastDay: 0,
+                }
+                : undefined;
+            return {
+                telemetryType: signalResult.telemetryType,
+                lastMinute: counts ? counts.lastMinute : null,
+                lastHour: counts ? counts.lastHour : null,
+                lastDay: counts ? counts.lastDay : null,
+            };
+        });
+        projects.push({
+            projectId,
+            projectName: null,
+            signals,
+            lastMinute: sumCountsOrNull(signals.map((signal) => {
+                return signal.lastMinute;
+            })),
+            lastHour: sumCountsOrNull(signals.map((signal) => {
+                return signal.lastHour;
+            })),
+            lastDay: sumCountsOrNull(signals.map((signal) => {
+                return signal.lastDay;
+            })),
+        });
+    }
+    return projects.sort((left, right) => {
+        var _a, _b;
+        return (((_a = right.lastDay) !== null && _a !== void 0 ? _a : 0) - ((_b = left.lastDay) !== null && _b !== void 0 ? _b : 0) ||
+            left.projectId.localeCompare(right.projectId));
+    });
+}
+/*
+ * ClickHouse only knows tenants by id. Names come from Postgres, and a project
+ * deleted while its telemetry is still inside the retention window has none —
+ * that project keeps a null name and the page labels it by id, rather than
+ * disappearing from a total it still contributes to.
+ */
+export function attachProjectNames(projects, namesByProjectId) {
+    return projects.map((project) => {
+        return Object.assign(Object.assign({}, project), { projectName: namesByProjectId.get(project.projectId) || null });
+    });
+}
+function getClient() {
+    return ClickhouseAppInstance.getDataSource();
+}
+async function queryJson(client, query) {
+    const result = (await (await client.query({
+        query,
+        format: "JSON",
+        clickhouse_settings: {
+            max_execution_time: QUERY_TIMEOUT_IN_SECONDS,
+        },
+    })).json());
+    return result.data || [];
+}
+/*
+ * Ingestion throughput per signal, with each signal's actual (uncompressed)
+ * footprint alongside it. Each table is probed independently so a missing table
+ * (e.g. an instance that only ingests logs) degrades gracefully rather than
+ * blanking the whole card.
+ */
+export async function getTelemetryIngestionBySignal() {
+    var _a;
+    const result = {
+        connected: false,
+        tables: [],
+    };
+    try {
+        const client = getClient();
+        if (!client) {
+            return result;
+        }
+        result["connected"] = true;
+        /*
+         * Guarded independently so a failure here never drops the ingestion counts
+         * below — a blank "Actual size" column is a much smaller loss than a blank
+         * ingestion rate.
+         */
+        const uncompressedBytesByStorageTable = new Map();
+        try {
+            const rows = await queryJson(client, buildTelemetryStorageSizeQuery({
+                clusterName: getClickhouseClusterName(),
+                databaseName: ClickhouseDatabase,
+                storageTableNames: TELEMETRY_INGESTION_SIGNALS.map((spec) => {
+                    return getStorageTableName(String(spec.table));
+                }),
+            }));
+            for (const row of rows) {
+                uncompressedBytesByStorageTable.set(String(row["table"]), toCountOrNull(row["uncompressed_bytes"]));
+            }
+        }
+        catch (err) {
+            logger.debug("AdminHealth: telemetry uncompressed-size query failed");
+            logger.debug(err);
+        }
+        for (const spec of TELEMETRY_INGESTION_SIGNALS) {
+            const entry = {
+                telemetryType: spec.telemetryType,
+                table: String(spec.table),
+                lastMinute: null,
+                lastHour: null,
+                lastDay: null,
+                uncompressedBytes: (_a = uncompressedBytesByStorageTable.get(getStorageTableName(String(spec.table)))) !== null && _a !== void 0 ? _a : null,
+                available: false,
+            };
+            try {
+                const rows = await queryJson(client, buildSignalIngestionQuery({
+                    spec,
+                    databaseName: ClickhouseDatabase,
+                }));
+                const row = rows[0] || {};
+                entry.lastMinute = toCountOrNull(row["last_minute"]);
+                entry.lastHour = toCountOrNull(row["last_hour"]);
+                entry.lastDay = toCountOrNull(row["last_day"]);
+                entry.available = true;
+            }
+            catch (err) {
+                logger.debug(`AdminHealth: telemetry ingestion query failed for ${spec.table}`);
+                logger.debug(err);
+            }
+            result.tables.push(entry);
+        }
+    }
+    catch (err) {
+        logger.error("AdminHealth: failed to read ClickHouse telemetry ingestion");
+        logger.error(err);
+    }
+    return result;
+}
+/*
+ * Ingestion split by tenant. Each signal is queried independently — one failing
+ * table must not blank the other two — and the three results are folded into one
+ * row per project. Names are NOT resolved here: they live in Postgres, and the
+ * caller attaches them with attachProjectNames.
+ */
+export async function getTelemetryIngestionByProject() {
+    const result = {
+        connected: false,
+        projects: [],
+        signals: [],
+        truncated: false,
+    };
+    try {
+        const client = getClient();
+        if (!client) {
+            return result;
+        }
+        result.connected = true;
+        const signalResults = [];
+        for (const spec of TELEMETRY_INGESTION_SIGNALS) {
+            const signalResult = {
+                telemetryType: spec.telemetryType,
+                available: false,
+                truncated: false,
+                rows: [],
+            };
+            try {
+                const rows = await queryJson(client, buildProjectIngestionQuery({
+                    spec,
+                    databaseName: ClickhouseDatabase,
+                    limit: MAX_PROJECTS_PER_SIGNAL,
+                }));
+                signalResult.rows = rows.map((row) => {
+                    return {
+                        projectId: String(row["project_id"] || ""),
+                        lastMinute: toCountOrNull(row["last_minute"]),
+                        lastHour: toCountOrNull(row["last_hour"]),
+                        lastDay: toCountOrNull(row["last_day"]),
+                    };
+                });
+                signalResult.available = true;
+                signalResult.truncated = rows.length >= MAX_PROJECTS_PER_SIGNAL;
+            }
+            catch (err) {
+                logger.debug(`AdminHealth: per-project telemetry ingestion query failed for ${spec.table}`);
+                logger.debug(err);
+            }
+            signalResults.push(signalResult);
+        }
+        result.projects = mergeProjectIngestionRows(signalResults);
+        result.signals = signalResults.map((signalResult) => {
+            return {
+                telemetryType: signalResult.telemetryType,
+                available: signalResult.available,
+                truncated: signalResult.truncated,
+            };
+        });
+        result.truncated = signalResults.some((signalResult) => {
+            return signalResult.truncated;
+        });
+    }
+    catch (err) {
+        logger.error("AdminHealth: failed to read per-project ClickHouse telemetry ingestion");
+        logger.error(err);
+    }
+    return result;
+}
+//# sourceMappingURL=TelemetryIngestion.js.map

package/build/dist/Server/Utils/InstanceHealth/TelemetryIngestion.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TelemetryIngestion.js","sourceRoot":"","sources":["../../../../../Server/Utils/InstanceHealth/TelemetryIngestion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EACL,qBAAqB,GAEtB,MAAM,yCAAyC,CAAC;AACjD,OAAO,kBAAkB,MAAM,qDAAqD,CAAC;AAErF,OAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,OAAO,EACL,wBAAwB,EACxB,mBAAmB,GACpB,MAAM,oCAAoC,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,CAAN,IAAY,wBAIX;AAJD,WAAY,wBAAwB;IAClC,yCAAa,CAAA;IACb,+CAAmB,CAAA;IACnB,6CAAiB,CAAA;AACnB,CAAC,EAJW,wBAAwB,KAAxB,wBAAwB,QAInC;AAqED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,2BAA2B,GACtC;IACE;QACE,aAAa,EAAE,wBAAwB,CAAC,IAAI;QAC5C,KAAK,EAAE,kBAAkB,CAAC,GAAG;QAC7B,UAAU,EAAE,MAAM;KACnB;IACD;QACE,aAAa,EAAE,wBAAwB,CAAC,OAAO;QAC/C,KAAK,EAAE,kBAAkB,CAAC,MAAM;QAChC,UAAU,EAAE,MAAM;KACnB;IACD;QACE,aAAa,EAAE,wBAAwB,CAAC,MAAM;QAC9C,KAAK,EAAE,kBAAkB,CAAC,IAAI;QAC9B,UAAU,EAAE,WAAW;KACxB;CACF,CAAC;AAEJ;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAW,GAAG,CAAC;AAEnD,+EAA+E;AAC/E,MAAM,wBAAwB,GAAW,EAAE,CAAC;AAI5C,SAAS,mBAAmB,CAAC,KAAa;IACxC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC;AAC5C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;QAC1D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAA4B;IAC1D,MAAM,KAAK,GAAkB,MAAM,CAAC,MAAM,CACxC,CAAC,KAAoB,EAAmB,EAAE;QACxC,OAAO,KAAK,KAAK,IAAI,CAAC;IACxB,CAAC,CACF,CAAC;IAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAW,EAAE,KAAa,EAAU,EAAE;QACzD,OAAO,GAAG,GAAG,KAAK,CAAC;IACrB,CAAC,EAAE,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,yBAAyB,CAAC,IAGzC;IACC,MAAM,UAAU,GAAW,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEjE,OAAO,CACL,SAAS;QACT,WAAW,UAAU,iDAAiD;QACtE,WAAW,UAAU,6CAA6C;QAClE,sBAAsB;QACtB,QAAQ,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,eAAe,CAC3D,IAAI,CAAC,IAAI,CAAC,KAAK,CAChB,GAAG;QACJ,SAAS,UAAU,kCAAkC,UAAU,WAAW,CAC3E,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,0BAA0B,CAAC,IAI1C;IACC,MAAM,UAAU,GAAW,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEjE,OAAO,CACL,kCAAkC;QAClC,WAAW,UAAU,iDAAiD;QACtE,WAAW,UAAU,6CAA6C;QAClE,sBAAsB;QACtB,QAAQ,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,eAAe,CAC3D,IAAI,CAAC,IAAI,CAAC,KAAK,CAChB,GAAG;QACJ,SAAS,UAAU,kCAAkC,UAAU,YAAY;QAC3E,qBAAqB;QACrB,yCAAyC;QACzC,SAAS,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAC/C,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,8BAA8B,CAAC,IAI9C;IACC,MAAM,SAAS,GAAW,IAAI,CAAC,iBAAiB;SAC7C,GAAG,CAAC,CAAC,SAAiB,EAAU,EAAE;QACjC,OAAO,IAAI,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC;IAC/C,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,CACL,mEAAmE;QACnE,iBAAiB,mBAAmB,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB;QACzE,gCAAgC,mBAAmB,CACjD,IAAI,CAAC,YAAY,CAClB,mBAAmB,SAAS,IAAI;QACjC,gBAAgB,CACjB,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,yBAAyB,CACvC,aAAgD;IAEhD,MAAM,eAAe,GAGjB,IAAI,GAAG,EAAE,CAAC;IAEd,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;QACzC,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,eAAe,GACnB,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC;gBAClC,IAAI,GAAG,EAAoC,CAAC;YAE9C,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,aAAa,EAAE;gBAC9C,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,OAAO,EAAE,GAAG,CAAC,OAAO;aACrB,CAAC,CAAC;YACH,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAqC,EAAE,CAAC;IAEtD,KAAK,MAAM,CAAC,SAAS,EAAE,eAAe,CAAC,IAAI,eAAe,EAAE,CAAC;QAC3D,MAAM,OAAO,GAA2C,aAAa,CAAC,GAAG,CACvE,CACE,YAAwC,EACP,EAAE;YACnC,MAAM,MAAM,GACV,YAAY,CAAC,SAAS;gBACpB,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI;oBACjD,UAAU,EAAE,CAAC;oBACb,QAAQ,EAAE,CAAC;oBACX,OAAO,EAAE,CAAC;iBACX;gBACH,CAAC,CAAC,SAAS,CAAC;YAEhB,OAAO;gBACL,aAAa,EAAE,YAAY,CAAC,aAAa;gBACzC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;gBAC7C,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;gBACzC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;aACxC,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC;YACZ,SAAS;YACT,WAAW,EAAE,IAAI;YACjB,OAAO;YACP,UAAU,EAAE,eAAe,CACzB,OAAO,CAAC,GAAG,CACT,CAAC,MAAuC,EAAiB,EAAE;gBACzD,OAAO,MAAM,CAAC,UAAU,CAAC;YAC3B,CAAC,CACF,CACF;YACD,QAAQ,EAAE,eAAe,CACvB,OAAO,CAAC,GAAG,CACT,CAAC,MAAuC,EAAiB,EAAE;gBACzD,OAAO,MAAM,CAAC,QAAQ,CAAC;YACzB,CAAC,CACF,CACF;YACD,OAAO,EAAE,eAAe,CACtB,OAAO,CAAC,GAAG,CACT,CAAC,MAAuC,EAAiB,EAAE;gBACzD,OAAO,MAAM,CAAC,OAAO,CAAC;YACxB,CAAC,CACF,CACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAClB,CACE,IAA+B,EAC/B,KAAgC,EACxB,EAAE;;QACV,OAAO,CACL,CAAC,MAAA,KAAK,CAAC,OAAO,mCAAI,CAAC,CAAC,GAAG,CAAC,MAAA,IAAI,CAAC,OAAO,mCAAI,CAAC,CAAC;YAC1C,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAC9C,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAA0C,EAC1C,gBAAqC;IAErC,OAAO,QAAQ,CAAC,GAAG,CACjB,CAAC,OAAkC,EAA6B,EAAE;QAChE,uCACK,OAAO,KACV,WAAW,EAAE,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,IAC5D;IACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,qBAAqB,CAAC,aAAa,EAAE,CAAC;AAC/C,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,MAAwB,EACxB,KAAa;IAEb,MAAM,MAAM,GAAyB,CAAC,MAAM,CAC1C,MAAM,MAAM,CAAC,KAAK,CAAC;QACjB,KAAK;QACL,MAAM,EAAE,MAAM;QACd,mBAAmB,EAAE;YACnB,kBAAkB,EAAE,wBAAwB;SAC7C;KACF,CAAC,CACH,CAAC,IAAI,EAAE,CAAyB,CAAC;IAElC,OAAO,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B;;IACjD,MAAM,MAAM,GAAmC;QAC7C,SAAS,EAAE,KAAK;QAChB,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,MAAM,GAA4B,SAAS,EAAE,CAAC;QAEpD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC;QAE3B;;;;WAIG;QACH,MAAM,+BAA+B,GACnC,IAAI,GAAG,EAAE,CAAC;QAEZ,IAAI,CAAC;YACH,MAAM,IAAI,GAAsB,MAAM,SAAS,CAC7C,MAAM,EACN,8BAA8B,CAAC;gBAC7B,WAAW,EAAE,wBAAwB,EAAE;gBACvC,YAAY,EAAE,kBAAkB;gBAChC,iBAAiB,EAAE,2BAA2B,CAAC,GAAG,CAChD,CAAC,IAAkC,EAAU,EAAE;oBAC7C,OAAO,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjD,CAAC,CACF;aACF,CAAC,CACH,CAAC;YAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,+BAA+B,CAAC,GAAG,CACjC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EACpB,aAAa,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACtE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,2BAA2B,EAAE,CAAC;YAC/C,MAAM,KAAK,GAA6B;gBACtC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;gBACzB,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,IAAI;gBACb,iBAAiB,EACf,MAAA,+BAA+B,CAAC,GAAG,CACjC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CACxC,mCAAI,IAAI;gBACX,SAAS,EAAE,KAAK;aACjB,CAAC;YAEF,IAAI,CAAC;gBACH,MAAM,IAAI,GAAsB,MAAM,SAAS,CAC7C,MAAM,EACN,yBAAyB,CAAC;oBACxB,IAAI;oBACJ,YAAY,EAAE,kBAAkB;iBACjC,CAAC,CACH,CAAC;gBACF,MAAM,GAAG,GAAe,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAEtC,KAAK,CAAC,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;gBACrD,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;gBACjD,KAAK,CAAC,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC/C,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;YACzB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CACV,qDAAqD,IAAI,CAAC,KAAK,EAAE,CAClE,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAC3E,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,8BAA8B;IAClD,MAAM,MAAM,GAAoC;QAC9C,SAAS,EAAE,KAAK;QAChB,QAAQ,EAAE,EAAE;QACZ,OAAO,EAAE,EAAE;QACX,SAAS,EAAE,KAAK;KACjB,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,MAAM,GAA4B,SAAS,EAAE,CAAC;QAEpD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;QAExB,MAAM,aAAa,GAAsC,EAAE,CAAC;QAE5D,KAAK,MAAM,IAAI,IAAI,2BAA2B,EAAE,CAAC;YAC/C,MAAM,YAAY,GAA+B;gBAC/C,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,SAAS,EAAE,KAAK;gBAChB,SAAS,EAAE,KAAK;gBAChB,IAAI,EAAE,EAAE;aACT,CAAC;YAEF,IAAI,CAAC;gBACH,MAAM,IAAI,GAAsB,MAAM,SAAS,CAC7C,MAAM,EACN,0BAA0B,CAAC;oBACzB,IAAI;oBACJ,YAAY,EAAE,kBAAkB;oBAChC,KAAK,EAAE,uBAAuB;iBAC/B,CAAC,CACH,CAAC;gBAEF,YAAY,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAC1B,CAAC,GAAe,EAAgC,EAAE;oBAChD,OAAO;wBACL,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;wBAC1C,UAAU,EAAE,aAAa,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;wBAC7C,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;wBACzC,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;qBACxC,CAAC;gBACJ,CAAC,CACF,CAAC;gBACF,YAAY,CAAC,SAAS,GAAG,IAAI,CAAC;gBAC9B,YAAY,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,IAAI,uBAAuB,CAAC;YAClE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CACV,iEAAiE,IAAI,CAAC,KAAK,EAAE,CAC9E,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,CAAC,QAAQ,GAAG,yBAAyB,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,CAAC,OAAO,GAAG,aAAa,CAAC,GAAG,CAChC,CACE,YAAwC,EAC2B,EAAE;YACrE,OAAO;gBACL,aAAa,EAAE,YAAY,CAAC,aAAa;gBACzC,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,SAAS,EAAE,YAAY,CAAC,SAAS;aAClC,CAAC;QACJ,CAAC,CACF,CAAC;QACF,MAAM,CAAC,SAAS,GAAG,aAAa,CAAC,IAAI,CACnC,CAAC,YAAwC,EAAW,EAAE;YACpD,OAAO,YAAY,CAAC,SAAS,CAAC;QAChC,CAAC,CACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CACV,wEAAwE,CACzE,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/build/dist/Server/Utils/LLM/LLMService.js

@@ -15,6 +15,7 @@ import LlmType from "../../../Types/LLM/LlmType";
 import BadDataException from "../../../Types/Exception/BadDataException";
 import logger from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
+import DataSourceEgressGuard from "../DataSource/EgressGuard";
 /**
  * Which spelling of the output-token cap an OpenAI-style endpoint accepts.
  *
@@ -365,7 +366,30 @@ class LLMService {
      * the offending parameter in the 400 — one parameter per response, hence the
      * loop.
      */
+    /*
+     * Request options for a call to a tenant-configured LLM endpoint.
+     *
+     * LlmProvider.baseUrl is writable by any project member while reading the
+     * apiKey is Owner/Admin-only, so an unguarded request lets a member repoint
+     * a provider at a host they control and collect the decrypted key out of the
+     * auth header — and, because provider error bodies are reflected back
+     * through the LLM provider API, use the same lever as a blind-free read
+     * primitive against the internal network.
+     *
+     * The address the guard checked is pinned into the socket, and redirects are
+     * refused: pinning covers only the validated host, so a 3xx would walk
+     * around it. Private ranges stay reachable on self-hosted installs, because
+     * a self-hosted Ollama on 10.x is the documented deployment.
+     */
+    static async buildGuardedRequestOptions(requestUrl, options) {
+        const { httpAgent, httpsAgent } = await DataSourceEgressGuard.assertUrlAllowedAndPin(requestUrl, {
+            targetLabel: "LLM provider",
+        });
+        return Object.assign(Object.assign({}, options), { doNotFollowRedirects: true, httpAgent,
+            httpsAgent });
+    }
     static async postOpenAIChatCompletion(data) {
+        var _a, _b;
         let adaptation = this.getInitialRequestAdaptation(data.config, data.modelName);
         /*
          * Which spellings have actually gone out on the wire. Read off the built
@@ -375,8 +399,17 @@ class LLMService {
          * byte-identical request.
          */
         const attemptedTokenLimitParams = new Set();
+        /*
+         * Validated once, before the first request: the adaptation loop below
+         * re-posts to the same URL, so re-resolving per attempt would only add
+         * DNS round trips (and a rebind window between them).
+         */
+        const requestOptions = await this.buildGuardedRequestOptions(data.requestUrl, {
+            retries: (_a = data.request.requestRetries) !== null && _a !== void 0 ? _a : 2,
+            exponentialBackoff: true,
+            timeout: (_b = data.request.requestTimeoutInMs) !== null && _b !== void 0 ? _b : 120000,
+        });
         const post = () => {
-            var _a, _b;
             const body = this.buildOpenAIRequestBody(data.modelName, data.request, adaptation);
             if (body["max_completion_tokens"] !== undefined) {
                 attemptedTokenLimitParams.add(TokenLimitParam.MaxCompletionTokens);
@@ -388,11 +421,7 @@ class LLMService {
                 url: URL.fromString(data.requestUrl),
                 data: body,
                 headers: data.headers,
-                options: {
-                    retries: (_a = data.request.requestRetries) !== null && _a !== void 0 ? _a : 2,
-                    exponentialBackoff: true,
-                    timeout: (_b = data.request.requestTimeoutInMs) !== null && _b !== void 0 ? _b : 120000,
-                },
+                options: requestOptions,
             });
         };
         let response = await post();
@@ -748,19 +777,20 @@ class LLMService {
             }
             requestData["tools"] = anthropicTools;
         }
+        const anthropicRequestUrl = `${baseUrl}/messages`;
         const response = await API.post({
-            url: URL.fromString(`${baseUrl}/messages`),
+            url: URL.fromString(anthropicRequestUrl),
             data: requestData,
             headers: {
                 "x-api-key": config.apiKey,
                 "anthropic-version": "2023-06-01",
                 "Content-Type": "application/json",
             },
-            options: {
+            options: await this.buildGuardedRequestOptions(anthropicRequestUrl, {
                 retries: (_b = request.requestRetries) !== null && _b !== void 0 ? _b : 2,
                 exponentialBackoff: true,
                 timeout: (_c = request.requestTimeoutInMs) !== null && _c !== void 0 ? _c : 120000,
-            },
+            }),
         });
         const anthropicLogAttributes = {
             llmType: config.llmType,
@@ -864,17 +894,18 @@ class LLMService {
         if (request.tools && request.tools.length > 0) {
             requestData["tools"] = this.toOpenAITools(request.tools);
         }
+        const ollamaRequestUrl = `${config.baseUrl}/api/chat`;
         const response = await API.post({
-            url: URL.fromString(`${config.baseUrl}/api/chat`),
+            url: URL.fromString(ollamaRequestUrl),
             data: requestData,
             headers: {
                 "Content-Type": "application/json",
             },
-            options: {
+            options: await this.buildGuardedRequestOptions(ollamaRequestUrl, {
                 retries: (_b = request.requestRetries) !== null && _b !== void 0 ? _b : 2,
                 exponentialBackoff: true,
                 timeout: (_c = request.requestTimeoutInMs) !== null && _c !== void 0 ? _c : 300000, // Ollama may be slower
-            },
+            }),
         });
         const ollamaLogAttributes = {
             llmType: config.llmType,