@archimonde12/llm-proxy 1.0.0

package/dist/http.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.joinUrl = joinUrl;
+exports.postJson = postJson;
+exports.postJsonStream = postJsonStream;
+/**
+ * Join `base` (origin and optional path prefix) with `path`.
+ * `new URL("/v1/...", base)` drops any path on `base` because a leading `/` is
+ * origin-absolute; this helper preserves prefixes like `/api`.
+ */
+function joinUrl(base, path) {
+    const b = base.replace(/\/+$/, "");
+    const p = path.replace(/^\/+/, "");
+    return `${b}/${p}`;
+}
+async function postJson(url, body, opts) {
+    const controller = new AbortController();
+    const timeoutMs = opts?.timeoutMs ?? 1800_000;
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const res = await fetch(url, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                ...(opts?.headers ?? {}),
+            },
+            body: JSON.stringify(body),
+            signal: controller.signal,
+        });
+        const json = (await res.json());
+        return { status: res.status, headers: res.headers, json };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+// Like postJson, but returns the raw streaming body. Timeout applies only to the
+// initial request (until headers are received), not the entire stream duration.
+async function postJsonStream(url, body, opts) {
+    const controller = new AbortController();
+    const timeoutMs = opts?.timeoutMs ?? 1800_000;
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const res = await fetch(url, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                ...(opts?.headers ?? {}),
+            },
+            body: JSON.stringify(body),
+            signal: controller.signal,
+        });
+        if (res.status !== 200) {
+            const { messages, tools, ...rest } = body;
+            const safeHeaders = opts?.headers && typeof opts.headers === "object"
+                ? Object.fromEntries(Object.entries(opts.headers).map(([k, v]) => [
+                    k,
+                    /^authorization$/i.test(k) ? "<redacted>" : v,
+                ]))
+                : undefined;
+            console.log("postJsonStream", JSON.stringify(rest), JSON.stringify(safeHeaders));
+            console.log("postJsonStream res", res);
+        }
+        return { status: res.status, headers: res.headers, body: res.body };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}

package/dist/index.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+require("dotenv/config");
+const server_1 = require("./server");
+const load_1 = require("./config/load");
+const startupLog_1 = require("./startupLog");
+async function main() {
+    const port = Number(process.env.PORT ?? 8787);
+    const host = process.env.HOST ?? "0.0.0.0";
+    const loaded = await (0, load_1.loadModelsFile)({
+        envPath: process.env.MODELS_PATH,
+    });
+    (0, startupLog_1.logStartupPreamble)({
+        nodeVersion: process.version,
+        nodeEnv: process.env.NODE_ENV ?? "development",
+        host,
+        port,
+        modelsPath: loaded.source.path,
+        modelsSourceKind: loaded.source.kind,
+        createdDefaultModelsFile: loaded.createdDefaultFile,
+    });
+    const app = await (0, server_1.buildServer)({ bindHost: host, initial: loaded });
+    const address = await app.listen({ port, host });
+    (0, startupLog_1.logListenBanner)({ address: String(address), port });
+}
+main().catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error(err);
+    process.exit(1);
+});

package/dist/observability/metrics.js

@@ -0,0 +1,102 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMetrics = createMetrics;
+exports.registerMetrics = registerMetrics;
+const prom_client_1 = __importDefault(require("prom-client"));
+function routeLabel(req) {
+    const anyReq = req;
+    const route = anyReq.routeOptions?.url;
+    if (typeof route === "string" && route)
+        return route;
+    return req.url.split("?")[0] ?? "<unknown>";
+}
+function createMetrics() {
+    const registry = new prom_client_1.default.Registry();
+    prom_client_1.default.collectDefaultMetrics({ register: registry });
+    const httpRequestsTotal = new prom_client_1.default.Counter({
+        name: "llm_proxy_http_requests_total",
+        help: "HTTP requests completed",
+        registers: [registry],
+        labelNames: ["method", "route", "status_code"],
+    });
+    const httpRequestDurationSeconds = new prom_client_1.default.Histogram({
+        name: "llm_proxy_http_request_duration_seconds",
+        help: "HTTP request duration in seconds",
+        registers: [registry],
+        labelNames: ["method", "route", "status_code"],
+        buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+    });
+    const upstreamErrorsTotal = new prom_client_1.default.Counter({
+        name: "llm_proxy_upstream_errors_total",
+        help: "Upstream errors by adapter/model",
+        registers: [registry],
+        labelNames: ["model_id", "adapter"],
+    });
+    const tokensInTotal = new prom_client_1.default.Counter({
+        name: "llm_proxy_tokens_in_total",
+        help: "Total input/prompt tokens (best-effort)",
+        registers: [registry],
+        labelNames: ["model_id", "adapter"],
+    });
+    const tokensOutTotal = new prom_client_1.default.Counter({
+        name: "llm_proxy_tokens_out_total",
+        help: "Total output/completion tokens (best-effort)",
+        registers: [registry],
+        labelNames: ["model_id", "adapter"],
+    });
+    const tokensTotal = new prom_client_1.default.Counter({
+        name: "llm_proxy_tokens_total",
+        help: "Total tokens (best-effort)",
+        registers: [registry],
+        labelNames: ["model_id", "adapter"],
+    });
+    const observeTokens = (args) => {
+        const labels = { model_id: args.modelId, adapter: args.adapter };
+        if (typeof args.tokensIn === "number" && Number.isFinite(args.tokensIn)) {
+            tokensInTotal.inc(labels, args.tokensIn);
+        }
+        if (typeof args.tokensOut === "number" && Number.isFinite(args.tokensOut)) {
+            tokensOutTotal.inc(labels, args.tokensOut);
+        }
+        if (typeof args.tokensTotal === "number" && Number.isFinite(args.tokensTotal)) {
+            tokensTotal.inc(labels, args.tokensTotal);
+        }
+    };
+    return {
+        registry,
+        httpRequestsTotal,
+        httpRequestDurationSeconds,
+        upstreamErrorsTotal,
+        tokensInTotal,
+        tokensOutTotal,
+        tokensTotal,
+        observeTokens,
+    };
+}
+async function registerMetrics(app, metrics) {
+    app.addHook("onRequest", async (req) => {
+        req.__metricsStartAt = process.hrtime.bigint();
+    });
+    app.addHook("onResponse", async (req, reply) => {
+        const startAt = req.__metricsStartAt;
+        const durationSeconds = startAt
+            ? Number(process.hrtime.bigint() - startAt) / 1e9
+            : undefined;
+        const labels = {
+            method: req.method,
+            route: routeLabel(req),
+            status_code: String(reply.statusCode),
+        };
+        metrics.httpRequestsTotal.inc(labels, 1);
+        if (typeof durationSeconds === "number") {
+            metrics.httpRequestDurationSeconds.observe(labels, durationSeconds);
+        }
+    });
+    app.get("/metrics", async (_req, reply) => {
+        reply.header("content-type", metrics.registry.contentType);
+        return await metrics.registry.metrics();
+    });
+}

package/dist/observability/modelMessageDebugStore.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ModelMessageDebugStore = void 0;
+class Ring {
+    capacity;
+    buf;
+    count = 0;
+    constructor(capacity) {
+        this.capacity = capacity;
+        if (capacity < 1)
+            throw new Error("capacity must be >= 1");
+        this.buf = new Array(capacity);
+    }
+    push(item) {
+        const i = this.count % this.capacity;
+        this.buf[i] = item;
+        this.count++;
+    }
+    /** Newest first */
+    list(limit) {
+        const cap = this.capacity;
+        const n = Math.min(limit, Math.min(this.count, cap));
+        const out = [];
+        for (let k = 0; k < n; k++) {
+            const idx = (this.count - 1 - k + cap) % cap;
+            const e = this.buf[idx];
+            if (e)
+                out.push(e);
+        }
+        return out;
+    }
+}
+class ModelMessageDebugStore {
+    perModelCapacity;
+    byModel = new Map();
+    constructor(perModelCapacity) {
+        this.perModelCapacity = perModelCapacity;
+        if (perModelCapacity < 1)
+            throw new Error("perModelCapacity must be >= 1");
+    }
+    record(ev) {
+        let ring = this.byModel.get(ev.modelId);
+        if (!ring) {
+            ring = new Ring(this.perModelCapacity);
+            this.byModel.set(ev.modelId, ring);
+        }
+        ring.push(ev);
+    }
+    /** Newest first */
+    getRecent(modelId, limit, roles) {
+        const ring = this.byModel.get(modelId);
+        if (!ring)
+            return [];
+        const items = ring.list(limit * 3);
+        if (!roles || roles.length === 0)
+            return items.slice(0, limit);
+        const allowed = new Set(roles);
+        const out = [];
+        for (const it of items) {
+            if (!allowed.has(it.role))
+                continue;
+            out.push(it);
+            if (out.length >= limit)
+                break;
+        }
+        return out;
+    }
+}
+exports.ModelMessageDebugStore = ModelMessageDebugStore;

package/dist/observability/modelRequestStore.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ModelRequestStore = void 0;
+exports.modelRequestHistoryCapacityFromEnv = modelRequestHistoryCapacityFromEnv;
+class ModelRequestStore {
+    capacity;
+    buf;
+    count = 0;
+    byId = new Map();
+    constructor(capacity) {
+        this.capacity = capacity;
+        if (capacity < 1)
+            throw new Error("capacity must be >= 1");
+        this.buf = new Array(capacity);
+    }
+    record(entry) {
+        const i = this.count % this.capacity;
+        const prev = this.buf[i];
+        if (prev)
+            this.byId.delete(prev.requestId);
+        this.buf[i] = entry;
+        this.byId.set(entry.requestId, entry);
+        this.count++;
+    }
+    /** Newest first */
+    getRecent(limit) {
+        const cap = this.capacity;
+        const n = Math.min(limit, Math.min(this.count, cap));
+        const out = [];
+        for (let k = 0; k < n; k++) {
+            const idx = (this.count - 1 - k + cap) % cap;
+            const e = this.buf[idx];
+            if (e)
+                out.push(e);
+        }
+        return out;
+    }
+    getById(requestId) {
+        return this.byId.get(requestId) ?? null;
+    }
+}
+exports.ModelRequestStore = ModelRequestStore;
+;
+function modelRequestHistoryCapacityFromEnv() {
+    const raw = process.env.MODEL_REQUEST_HISTORY_MAX?.trim();
+    if (!raw)
+        return 5000;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || n < 1)
+        return 5000;
+    return Math.min(Math.floor(n), 50_000);
+}

package/dist/observability/requestId.js

@@ -0,0 +1,21 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.genRequestId = genRequestId;
+exports.registerRequestId = registerRequestId;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+function genRequestId(req) {
+    const hdr = req.headers["x-request-id"] ??
+        req.headers["X-Request-Id"] ??
+        req.headers["x-request-id".toLowerCase()];
+    if (typeof hdr === "string" && hdr.trim())
+        return hdr.trim();
+    return node_crypto_1.default.randomUUID();
+}
+async function registerRequestId(app) {
+    app.addHook("onRequest", async (req, reply) => {
+        reply.header("x-request-id", req.id);
+    });
+}

package/dist/observability/requestRecorder.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestRecorder = void 0;
+exports.requestHistoryCapacityFromEnv = requestHistoryCapacityFromEnv;
+class RequestRecorder {
+    capacity;
+    buf;
+    head = 0;
+    count = 0;
+    constructor(capacity) {
+        this.capacity = capacity;
+        if (capacity < 1)
+            throw new Error("capacity must be >= 1");
+        this.buf = new Array(capacity);
+    }
+    record(entry) {
+        const i = this.count % this.capacity;
+        this.buf[i] = entry;
+        this.count++;
+        if (this.count <= this.capacity)
+            this.head = 0;
+        else
+            this.head = this.count % this.capacity;
+    }
+    /** Newest first */
+    getRecent(limit) {
+        const cap = this.capacity;
+        const n = Math.min(limit, Math.min(this.count, cap));
+        const out = [];
+        for (let k = 0; k < n; k++) {
+            const idx = (this.count - 1 - k + cap) % cap;
+            const e = this.buf[idx];
+            if (e)
+                out.push(e);
+        }
+        return out;
+    }
+}
+exports.RequestRecorder = RequestRecorder;
+function requestHistoryCapacityFromEnv() {
+    const raw = process.env.REQUEST_HISTORY_MAX?.trim();
+    if (!raw)
+        return 200;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || n < 1)
+        return 200;
+    return Math.min(Math.floor(n), 10_000);
+}

package/dist/observability/summary.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildMetricsSummary = buildMetricsSummary;
+/**
+ * Small JSON snapshot from existing prom-client metrics (no extra counters).
+ */
+async function buildMetricsSummary(metrics) {
+    const out = {
+        uptimeSeconds: process.uptime(),
+    };
+    const http = await metrics.httpRequestsTotal.get();
+    let total = 0;
+    let e4 = 0;
+    let e5 = 0;
+    let e1 = 0;
+    for (const v of http.values) {
+        total += v.value;
+        const sc = v.labels?.status_code;
+        if (typeof sc !== "string")
+            continue;
+        const code = Number(sc);
+        if (!Number.isFinite(code))
+            continue;
+        if (code >= 500)
+            e5 += v.value;
+        else if (code >= 400)
+            e4 += v.value;
+        else if (code < 200)
+            e1 += v.value;
+    }
+    out.httpRequestsTotal = total;
+    out.httpErrors4xx = e4;
+    out.httpErrors5xx = e5;
+    out.httpErrors1xx = e1;
+    const up = await metrics.upstreamErrorsTotal.get();
+    let upSum = 0;
+    for (const v of up.values)
+        upSum += v.value;
+    out.upstreamErrorsTotal = upSum;
+    const hist = await metrics.httpRequestDurationSeconds.get();
+    let count = 0;
+    let sum = 0;
+    for (const v of hist.values) {
+        const mn = v.metricName;
+        if (mn?.endsWith("_count"))
+            count += v.value;
+        if (mn === "llm_proxy_http_request_duration_seconds_sum")
+            sum += v.value;
+    }
+    out.requestDurationSeconds = {
+        count,
+        sum,
+        meanSeconds: count > 0 ? sum / count : undefined,
+    };
+    return out;
+}

package/dist/observability/tokenUsage.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractUsageFromChatCompletionResponse = extractUsageFromChatCompletionResponse;
+exports.extractUsageFromSseChunk = extractUsageFromSseChunk;
+function num(v) {
+    return typeof v === "number" && Number.isFinite(v) ? v : undefined;
+}
+function extractUsageFromChatCompletionResponse(adapter, body) {
+    if (!body || typeof body !== "object")
+        return null;
+    if (adapter === "openai_compatible" || adapter === "deepseek") {
+        const usage = body.usage;
+        if (!usage || typeof usage !== "object")
+            return null;
+        const tokensIn = num(usage.prompt_tokens);
+        const tokensOut = num(usage.completion_tokens);
+        const tokensTotal = num(usage.total_tokens);
+        if (typeof tokensIn !== "number" &&
+            typeof tokensOut !== "number" &&
+            typeof tokensTotal !== "number") {
+            return null;
+        }
+        return { tokensIn, tokensOut, tokensTotal };
+    }
+    // Best-effort only; adapters may not expose usage consistently.
+    return null;
+}
+function extractUsageFromSseChunk(adapter, chunk) {
+    if (!chunk || typeof chunk !== "object")
+        return null;
+    if (adapter === "openai_compatible" || adapter === "deepseek") {
+        const usage = chunk.usage;
+        if (!usage || typeof usage !== "object")
+            return null;
+        const tokensIn = num(usage.prompt_tokens);
+        const tokensOut = num(usage.completion_tokens);
+        const tokensTotal = num(usage.total_tokens);
+        if (typeof tokensIn !== "number" &&
+            typeof tokensOut !== "number" &&
+            typeof tokensTotal !== "number") {
+            return null;
+        }
+        return { tokensIn, tokensOut, tokensTotal };
+    }
+    return null;
+}