@archimonde12/llm-proxy 1.0.0

package/dist/server.js

@@ -0,0 +1,442 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildServer = buildServer;
+const fastify_1 = __importDefault(require("fastify"));
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const static_1 = __importDefault(require("@fastify/static"));
+const load_1 = require("./config/load");
+const config_1 = require("./config");
+const adapters_1 = require("./adapters");
+const metrics_1 = require("./observability/metrics");
+const requestId_1 = require("./observability/requestId");
+const tokenUsage_1 = require("./observability/tokenUsage");
+const auth_1 = require("./admin/auth");
+const routes_1 = require("./admin/routes");
+const upstreamProbe_1 = require("./upstreamProbe");
+const requestRecorder_1 = require("./observability/requestRecorder");
+const modelRequestStore_1 = require("./observability/modelRequestStore");
+const modelMessageDebugStore_1 = require("./observability/modelMessageDebugStore");
+const version_1 = require("./version");
+function shouldLog(level) {
+    const configured = (process.env.LOG_LEVEL ?? "").toLowerCase();
+    if (!configured)
+        return true;
+    const order = {
+        debug: 10,
+        info: 20,
+        warn: 30,
+        error: 40,
+    };
+    const cfg = ["debug", "info", "warn", "error"].includes(configured)
+        ? configured
+        : "info";
+    return order[level] >= order[cfg];
+}
+function logStreamIsTTY(level) {
+    return level === "error" || level === "warn"
+        ? process.stderr.isTTY
+        : process.stdout.isTTY;
+}
+function formatLogLine(level, message, meta) {
+    if (!logStreamIsTTY(level)) {
+        return meta ? `${message} ${JSON.stringify(meta)}` : message;
+    }
+    const badge = {
+        debug: "\x1b[94m\x1b[2m[debug]\x1b[0m",
+        info: "\x1b[96m[info]\x1b[0m",
+        warn: "\x1b[33m\x1b[1m[warn]\x1b[0m",
+        error: "\x1b[31m\x1b[1m[error]\x1b[0m",
+    };
+    if (!meta) {
+        return `${badge[level]} ${message}`;
+    }
+    // meta: dim but readable (avoid dark gray)
+    return `${badge[level]} ${message} \x1b[2m\x1b[37m${JSON.stringify(meta)}\x1b[0m`;
+}
+function log(level, message, meta) {
+    if (!shouldLog(level))
+        return;
+    const line = formatLogLine(level, message, meta);
+    // eslint-disable-next-line no-console
+    (level === "error"
+        ? console.error
+        : level === "warn"
+            ? console.warn
+            : console.log)(line);
+}
+async function pipeWebStreamToNode(stream, res) {
+    const reader = stream.getReader();
+    try {
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            if (value)
+                res.write(Buffer.from(value));
+        }
+    }
+    finally {
+        try {
+            reader.releaseLock();
+        }
+        catch {
+            // ignore
+        }
+    }
+}
+async function pipeSseWebStreamToNodeWithUsage(adapter, stream, res, onUsage) {
+    const reader = stream.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    const maybeHandleEvent = (eventBlock) => {
+        const lines = eventBlock.split("\n");
+        for (const line of lines) {
+            const trimmed = line.trimEnd();
+            if (!trimmed.startsWith("data:"))
+                continue;
+            const data = trimmed.slice("data:".length).trimStart();
+            if (!data || data === "[DONE]")
+                continue;
+            try {
+                const obj = JSON.parse(data);
+                const usage = (0, tokenUsage_1.extractUsageFromSseChunk)(adapter, obj);
+                if (usage)
+                    onUsage(usage);
+            }
+            catch {
+                // ignore non-JSON chunks
+            }
+        }
+    };
+    try {
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            if (!value)
+                continue;
+            res.write(Buffer.from(value));
+            buffer += decoder.decode(value, { stream: true });
+            if (buffer.length > 1024 * 1024)
+                buffer = buffer.slice(-256 * 1024);
+            let idx;
+            while ((idx = buffer.indexOf("\n\n")) >= 0) {
+                const block = buffer.slice(0, idx);
+                buffer = buffer.slice(idx + 2);
+                if (block)
+                    maybeHandleEvent(block);
+            }
+        }
+    }
+    finally {
+        try {
+            reader.releaseLock();
+        }
+        catch {
+            // ignore
+        }
+    }
+}
+async function buildServer(opts) {
+    const app = (0, fastify_1.default)({
+        logger: process.env.LOG_LEVEL ? true : false,
+        genReqId: requestId_1.genRequestId,
+    });
+    let state;
+    if (opts.initial) {
+        state = {
+            modelsFile: opts.initial.modelsFile,
+            activeConfigPath: node_path_1.default.resolve(opts.initial.source.path),
+            bootstrapSource: opts.initial.source,
+            configGeneration: 1,
+        };
+    }
+    else if (opts.modelsPath) {
+        const abs = node_path_1.default.resolve(opts.modelsPath);
+        const mf = await (0, load_1.loadModelsFileFromPath)(abs);
+        state = {
+            modelsFile: mf,
+            activeConfigPath: abs,
+            bootstrapSource: { kind: "cli_flag", path: abs },
+            configGeneration: 1,
+        };
+    }
+    else {
+        throw new Error("buildServer: provide initial or modelsPath");
+    }
+    const metrics = (0, metrics_1.createMetrics)();
+    const recorder = new requestRecorder_1.RequestRecorder((0, requestRecorder_1.requestHistoryCapacityFromEnv)());
+    const modelRequests = new modelRequestStore_1.ModelRequestStore((0, modelRequestStore_1.modelRequestHistoryCapacityFromEnv)());
+    const modelMessages = new modelMessageDebugStore_1.ModelMessageDebugStore(10);
+    const enforceLocalhost = (0, auth_1.shouldEnforceLocalhostGuard)(opts.bindHost);
+    const pkgV = (0, version_1.packageVersion)();
+    if (enforceLocalhost) {
+        app.addHook("onRequest", async (req, reply) => {
+            const p = (req.url ?? "").split("?")[0] ?? "";
+            if (p === "/" || p.startsWith("/ui")) {
+                if (!(0, auth_1.isLocalhostRequest)(req))
+                    return (0, auth_1.sendForbiddenNonLocal)(reply);
+            }
+        });
+    }
+    await (0, requestId_1.registerRequestId)(app);
+    await (0, metrics_1.registerMetrics)(app, metrics);
+    app.addHook("onResponse", async (req, reply) => {
+        const startAt = req.__startAt;
+        const durationMs = startAt
+            ? Number(process.hrtime.bigint() - startAt) / 1e6
+            : 0;
+        const url = req.url.split("?")[0] ?? req.url;
+        const modelId = req.__modelId;
+        const modelAdapter = req.__modelAdapter;
+        const tokens = req.__tokenUsage;
+        let error;
+        if (reply.statusCode >= 400) {
+            error = `HTTP ${reply.statusCode}`;
+        }
+        recorder.record({
+            requestId: String(req.id),
+            ts: Date.now(),
+            method: req.method,
+            path: url,
+            status: reply.statusCode,
+            durationMs: Math.round(durationMs * 1000) / 1000,
+            modelId,
+            adapter: modelAdapter,
+            error,
+        });
+        if (modelId && url === "/v1/chat/completions") {
+            modelRequests.record({
+                ts: Date.now(),
+                requestId: String(req.id),
+                modelId,
+                endpoint: url,
+                status: reply.statusCode,
+                latencyMs: Math.round(durationMs * 1000) / 1000,
+                usage: tokens &&
+                    (typeof tokens.tokensIn === "number" ||
+                        typeof tokens.tokensOut === "number" ||
+                        typeof tokens.tokensTotal === "number")
+                    ? {
+                        prompt_tokens: tokens.tokensIn,
+                        completion_tokens: tokens.tokensOut,
+                        total_tokens: tokens.tokensTotal,
+                    }
+                    : undefined,
+                error,
+            });
+        }
+    });
+    app.addHook("onRequest", async (req) => {
+        req.__startAt = process.hrtime.bigint();
+        const path = (req.url ?? "").split("?")[0] ?? "";
+        if (!path.startsWith("/admin")) {
+            log("info", "incoming_request", {
+                id: req.id,
+                method: req.method,
+                url: req.url,
+                ip: req.ip,
+            });
+        }
+    });
+    app.addHook("onResponse", async (req, reply) => {
+        const startAt = req.__startAt;
+        const durationMs = startAt
+            ? Number(process.hrtime.bigint() - startAt) / 1e6
+            : undefined;
+        const modelId = req.__modelId;
+        const modelAdapter = req.__modelAdapter;
+        const tokens = req.__tokenUsage;
+        const path = (req.url ?? "").split("?")[0] ?? "";
+        if (!path.startsWith("/admin")) {
+            log("info", "request_complete", {
+                id: req.id,
+                method: req.method,
+                url: req.url,
+                statusCode: reply.statusCode,
+                durationMs: durationMs ? Math.round(durationMs * 1000) / 1000 : undefined,
+                modelId,
+                adapter: modelAdapter,
+                tokensIn: tokens?.tokensIn,
+                tokensOut: tokens?.tokensOut,
+                tokensTotal: tokens?.tokensTotal,
+            });
+        }
+    });
+    app.setErrorHandler(async (err, req, reply) => {
+        const statusCode = typeof err?.statusCode === "number"
+            ? err.statusCode
+            : 500;
+        log("error", "request_error", {
+            id: req.id,
+            method: req.method,
+            url: req.url,
+            statusCode,
+            message: err?.message,
+        });
+        reply.code(statusCode);
+        return { error: { message: err?.message ?? "Internal error" } };
+    });
+    app.get("/healthz", async () => ({ ok: true }));
+    app.get("/readyz", async (req, reply) => {
+        const deep = req.query?.deep === "1" || req.query?.deep === 1;
+        if (!deep)
+            return { ok: true };
+        const checks = await Promise.all(state.modelsFile.models.map(async (m) => {
+            const base = m.baseUrl.replace(/\/+$/, "");
+            const r = await (0, upstreamProbe_1.probeModelUpstream)({
+                ...m,
+                timeoutMs: Math.min(m.timeoutMs ?? 1500, 5000),
+            });
+            return {
+                id: m.id,
+                ok: r.ok,
+                status: r.status,
+                baseUrl: base,
+                ...(!r.ok ? { error: r.message } : {}),
+            };
+        }));
+        const allOk = checks.every((c) => c.ok);
+        if (!allOk)
+            reply.code(503);
+        return { ok: allOk, upstreams: checks };
+    });
+    app.get("/v1/models", async () => {
+        return {
+            object: "list",
+            data: state.modelsFile.models.map((m) => ({
+                id: m.id,
+                object: "model",
+                owned_by: "local",
+            })),
+        };
+    });
+    app.post("/v1/chat/completions", async (req, reply) => {
+        if (!req.body || typeof req.body !== "object") {
+            reply.code(400);
+            return { error: { message: "Missing JSON body" } };
+        }
+        try {
+            const modelCfg = (0, config_1.resolveModelConfig)(state.modelsFile, req.body.model);
+            const adapter = (0, adapters_1.createAdapter)(modelCfg);
+            req.__modelId = modelCfg.id;
+            req.__modelAdapter = modelCfg.adapter;
+            log("info", "chat_completions", {
+                id: req.id,
+                model: modelCfg.id,
+                stream: Boolean(req.body.stream),
+            });
+            // Capture input messages for deep-debug (per-model ring buffer, roles system/user only).
+            const endpoint = "/v1/chat/completions";
+            const msgs = req.body?.messages;
+            if (Array.isArray(msgs)) {
+                for (let i = 0; i < msgs.length; i++) {
+                    const m = msgs[i];
+                    const role = m?.role;
+                    if (role !== "system" && role !== "user")
+                        continue;
+                    modelMessages.record({
+                        id: `${String(req.id)}:${i}:${role}`,
+                        ts: Date.now(),
+                        modelId: modelCfg.id,
+                        requestId: String(req.id),
+                        endpoint,
+                        role,
+                        rawMessageJson: m,
+                    });
+                }
+            }
+            if (req.body.stream) {
+                if (!adapter.chatCompletionsStream) {
+                    reply.code(400);
+                    return {
+                        error: {
+                            message: `Model '${modelCfg.id}' does not support streaming`,
+                        },
+                    };
+                }
+                const streamResult = await adapter.chatCompletionsStream(req.body);
+                const contentType = streamResult.headers["content-type"] ??
+                    "text/event-stream; charset=utf-8";
+                reply.raw.statusCode = streamResult.status;
+                reply.raw.setHeader("content-type", contentType);
+                reply.raw.setHeader("cache-control", streamResult.headers["cache-control"] ?? "no-cache");
+                reply.raw.setHeader("connection", streamResult.headers["connection"] ?? "keep-alive");
+                if (!streamResult.body) {
+                    reply.raw.end();
+                    return reply;
+                }
+                const streamUsage = {
+                    current: null,
+                };
+                const recordUsage = (u) => {
+                    streamUsage.current = u;
+                };
+                if (String(contentType).toLowerCase().includes("text/event-stream")) {
+                    await pipeSseWebStreamToNodeWithUsage(modelCfg.adapter, streamResult.body, reply.raw, recordUsage);
+                }
+                else {
+                    await pipeWebStreamToNode(streamResult.body, reply.raw);
+                }
+                const lastUsage = streamUsage.current;
+                if (lastUsage && streamResult.status >= 200 && streamResult.status < 300) {
+                    metrics.observeTokens({
+                        modelId: modelCfg.id,
+                        adapter: modelCfg.adapter,
+                        ...lastUsage,
+                    });
+                    req.__tokenUsage = lastUsage;
+                }
+                reply.raw.end();
+                return reply;
+            }
+            else {
+                const result = await adapter.chatCompletions(req.body);
+                reply.code(result.status);
+                reply.header("content-type", "application/json");
+                const usage = (0, tokenUsage_1.extractUsageFromChatCompletionResponse)(modelCfg.adapter, result.body);
+                if (usage && result.status >= 200 && result.status < 300) {
+                    metrics.observeTokens({
+                        modelId: modelCfg.id,
+                        adapter: modelCfg.adapter,
+                        ...usage,
+                    });
+                    req.__tokenUsage = usage;
+                }
+                return result.body;
+            }
+        }
+        catch (err) {
+            const statusCode = typeof err?.statusCode === "number" ? err.statusCode : 502;
+            reply.code(statusCode);
+            const modelId = req.__modelId;
+            const modelAdapter = req.__modelAdapter;
+            if (modelId && modelAdapter && statusCode >= 500) {
+                metrics.upstreamErrorsTotal.inc({ model_id: String(modelId), adapter: String(modelAdapter) }, 1);
+            }
+            return { error: { message: err?.message ?? "Upstream error" } };
+        }
+    });
+    await (0, routes_1.registerAdminRoutes)(app, {
+        state,
+        metrics,
+        recorder,
+        modelRequests,
+        modelMessages,
+        packageVersion: pkgV,
+        enforceLocalhost,
+    });
+    const uiDist = node_path_1.default.join(__dirname, "..", "ui", "dist");
+    if (node_fs_1.default.existsSync(uiDist)) {
+        await app.register(static_1.default, {
+            root: uiDist,
+            prefix: "/ui/",
+        });
+        app.get("/", async (_req, reply) => reply.redirect("/ui/", 302));
+    }
+    return app;
+}

package/dist/startupLog.js

@@ -0,0 +1,114 @@
+"use strict";
+/** Colored startup / listen logs (ANSI when stdout is a TTY). */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logStartupPreamble = logStartupPreamble;
+exports.logListenBanner = logListenBanner;
+const tty = process.stdout.isTTY;
+function ansi(open, text) {
+    return tty ? `${open}${text}\x1b[0m` : text;
+}
+const color = {
+    dim: (s) => ansi("\x1b[2m", s),
+    bold: (s) => ansi("\x1b[1m", s),
+    cyan: (s) => ansi("\x1b[36m", s),
+    brightCyan: (s) => ansi("\x1b[96m", s),
+    green: (s) => ansi("\x1b[32m", s),
+    brightGreen: (s) => ansi("\x1b[92m", s),
+    yellow: (s) => ansi("\x1b[33m", s),
+    magenta: (s) => ansi("\x1b[35m", s),
+    gray: (s) => ansi("\x1b[90m", s),
+    /** label + value combined */
+    labelValue: (label, value) => `${color.gray(label)}  ${color.brightCyan(value)}`,
+};
+function logStartupPreamble(opts) {
+    const w = 56;
+    const inner = w - 4;
+    const border = (s) => color.cyan(s);
+    const top = border(`╔${"═".repeat(w - 2)}╗`);
+    const mid = border(`║${" ".repeat(w - 2)}║`);
+    const bot = border(`╚${"═".repeat(w - 2)}╝`);
+    const name = "llm-proxy";
+    const titleLine = border("║") +
+        "  " +
+        color.bold(color.brightCyan(name)) +
+        " ".repeat(Math.max(0, inner - 2 - name.length)) +
+        "  " +
+        border("║");
+    // eslint-disable-next-line no-console
+    console.log("");
+    // eslint-disable-next-line no-console
+    console.log(top);
+    // eslint-disable-next-line no-console
+    console.log(mid);
+    // eslint-disable-next-line no-console
+    console.log(titleLine);
+    // eslint-disable-next-line no-console
+    console.log(mid);
+    // eslint-disable-next-line no-console
+    console.log(bot);
+    // eslint-disable-next-line no-console
+    console.log("");
+    const rows = [
+        ["Node.js", opts.nodeVersion],
+        ["Environment", opts.nodeEnv],
+        ["Bind", `${opts.host}:${opts.port}`],
+        ["models.json", `${opts.modelsPath} (${opts.modelsSourceKind})`],
+    ];
+    const labelW = Math.max(...rows.map(([k]) => k.length), 11);
+    for (const [k, v] of rows) {
+        // eslint-disable-next-line no-console
+        console.log(`  ${color.labelValue(k.padEnd(labelW), v)}`);
+    }
+    if (opts.createdDefaultModelsFile) {
+        // eslint-disable-next-line no-console
+        console.log("");
+        // eslint-disable-next-line no-console
+        console.log(`  ${color.yellow("Note:")} ${color.dim("No models.json was found — wrote a starter file with an example Ollama model.")}`);
+        // eslint-disable-next-line no-console
+        console.log(`  ${color.dim(`Edit: ${opts.modelsPath}`)}`);
+    }
+    // eslint-disable-next-line no-console
+    console.log("");
+    // eslint-disable-next-line no-console
+    console.log(color.dim("  Endpoints: /healthz  /readyz  /metrics  /v1/models  /v1/chat/completions  /admin/*  /ui/"));
+    // eslint-disable-next-line no-console
+    console.log("");
+}
+function logListenBanner(opts) {
+    const localUrl = `http://127.0.0.1:${opts.port}`;
+    const sep = color.dim("─".repeat(58));
+    // eslint-disable-next-line no-console
+    console.log(sep);
+    // eslint-disable-next-line no-console
+    console.log(`  ${color.bold(color.brightGreen("Listening"))}  ${color.brightCyan(opts.address)}`);
+    // eslint-disable-next-line no-console
+    console.log(sep);
+    // eslint-disable-next-line no-console
+    console.log("");
+    // eslint-disable-next-line no-console
+    console.log(`  ${color.bold(color.yellow("Local"))}`);
+    // eslint-disable-next-line no-console
+    console.log(`    ${color.brightCyan(localUrl)}`);
+    // eslint-disable-next-line no-console
+    console.log("");
+    // eslint-disable-next-line no-console
+    console.log(`  ${color.bold(color.magenta("Public HTTPS tunnel"))}`);
+    // eslint-disable-next-line no-console
+    console.log(color.dim("  Run one of these in another terminal to expose this server:"));
+    // eslint-disable-next-line no-console
+    console.log("");
+    // eslint-disable-next-line no-console
+    console.log(`  ${color.gray("ngrok")}`);
+    // eslint-disable-next-line no-console
+    console.log(`    ${color.green(`ngrok http ${opts.port}`)}`);
+    // eslint-disable-next-line no-console
+    console.log("");
+    // eslint-disable-next-line no-console
+    console.log(`  ${color.gray("Cloudflare Tunnel (cloudflared)")}`);
+    // eslint-disable-next-line no-console
+    console.log(`    ${color.green(`cloudflared tunnel --url ${localUrl}`)}`);
+    // eslint-disable-next-line no-console
+    console.log(color.dim("    Install: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation/"));
+    // eslint-disable-next-line no-console
+    console.log("");
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/upstreamProbe.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.listModelsProbePath = listModelsProbePath;
+exports.probeModelUpstream = probeModelUpstream;
+const mergeHeaders_1 = require("./config/mergeHeaders");
+const http_1 = require("./http");
+/** Path for a lightweight GET that lists models (same auth as chat POST). */
+function listModelsProbePath(adapter) {
+    switch (adapter) {
+        case "ollama":
+            return "api/tags";
+        case "openai_compatible":
+        case "deepseek":
+            return "v1/models";
+    }
+}
+/**
+ * GET list-models endpoint with the same outbound headers as chat completions
+ * (Authorization / custom api key header / config.headers).
+ */
+async function probeModelUpstream(mc) {
+    const path = listModelsProbePath(mc.adapter);
+    const url = (0, http_1.joinUrl)(mc.baseUrl, path);
+    const timeoutMs = Math.min(mc.timeoutMs ?? 5000, 10_000);
+    const merged = (0, mergeHeaders_1.mergeModelOutboundHeaders)(mc);
+    const controller = new AbortController();
+    const t = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const res = await fetch(url, {
+            method: "GET",
+            headers: {
+                accept: "application/json",
+                ...(merged ?? {}),
+            },
+            signal: controller.signal,
+        });
+        return {
+            ok: res.ok,
+            status: res.status,
+            message: res.ok ? "Reachable" : `HTTP ${res.status}`,
+        };
+    }
+    catch (err) {
+        return {
+            ok: false,
+            status: 0,
+            message: err?.name === "AbortError" ? "Timeout" : (err?.message ?? String(err)),
+        };
+    }
+    finally {
+        clearTimeout(t);
+    }
+}

package/dist/version.js

@@ -0,0 +1,19 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.packageVersion = packageVersion;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+function packageVersion() {
+    try {
+        const pkgPath = node_path_1.default.join(__dirname, "..", "package.json");
+        const raw = node_fs_1.default.readFileSync(pkgPath, "utf8");
+        const j = JSON.parse(raw);
+        return j.version ?? "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}