@acarmisc/backstage-plugin-litellm-backend 0.23.0 → 0.24.0

package/config.d.ts

@@ -234,6 +234,45 @@ export interface Config {
       metadata?: { [key: string]: string };
     };
 
+    /**
+     * Langfuse answer-quality scores for the Insights tab ("Quality scores"
+     * card, GET /insights/quality). Generic Metrics API v2 integration: any
+     * Langfuse host works, and a disabled, unconfigured, or v3 server
+     * degrades to `{configured:false}` instead of erroring.
+     */
+    langfuse?: {
+      /**
+       * When true the /insights/quality route queries Langfuse.
+       * @default false
+       */
+      enabled?: boolean;
+
+      /**
+       * Langfuse host, e.g. https://langfuse.example.com.
+       * @visibility backend
+       */
+      baseUrl?: string;
+
+      /**
+       * Langfuse public key (Metrics API Basic-auth username).
+       * @visibility backend
+       */
+      publicKey?: string;
+
+      /**
+       * Langfuse secret key (Metrics API Basic-auth password).
+       * @visibility secret
+       */
+      secretKey?: string;
+
+      /**
+       * The Langfuse observation field holding the LiteLLM user id.
+       * Deployments whose agents tag traces differently change this one key.
+       * @default "userId"
+       */
+      userIdField?: string;
+    };
+
     /**
      * Caching configuration for backend data fetches.
      */

package/dist/client.js

@@ -724,22 +724,23 @@ class LiteLLMClient {
         return this.normalizeSpendLogs(raw, page, pageSize);
     }
     normalizeSpendLogs(raw, page, pageSize) {
-        const rows = Array.isArray(raw)
-            ? raw
-            : Array.isArray(raw?.data)
-                ? raw.data
-                : Array.isArray(raw?.results)
-                    ? raw.results
-                    : Array.isArray(raw?.logs)
-                        ? raw.logs
-                        : [];
+        const rows = [];
+        if (Array.isArray(raw))
+            rows.push(...raw);
+        else if (Array.isArray(raw?.data))
+            rows.push(...raw.data);
+        else if (Array.isArray(raw?.results))
+            rows.push(...raw.results);
+        else if (Array.isArray(raw?.logs))
+            rows.push(...raw.logs);
         const toNum = (v, fallback = 0) => {
-            const n = typeof v === 'number'
-                ? v
-                : typeof v === 'string' && v.trim() !== ''
-                    ? Number(v)
-                    : NaN;
-            return Number.isFinite(n) ? n : fallback;
+            if (typeof v === 'number')
+                return Number.isFinite(v) ? v : fallback;
+            if (typeof v === 'string' && v.trim() !== '') {
+                const n = Number(v);
+                return Number.isFinite(n) ? n : fallback;
+            }
+            return fallback;
         };
         const data = rows.map((r) => {
             const promptTokens = toNum(r?.prompt_tokens ?? r?.promptTokens ?? r?.input_tokens);

package/dist/index.cjs.js

@@ -8691,25 +8691,33 @@ var LiteLLMClient = class {
     return this.normalizeSpendLogs(raw, page, pageSize);
   }
   normalizeSpendLogs(raw, page, pageSize) {
-    const rows = Array.isArray(raw) ? raw : Array.isArray(raw?.data) ? raw.data : Array.isArray(raw?.results) ? raw.results : Array.isArray(raw?.logs) ? raw.logs : [];
-    const toNum = (v, fallback = 0) => {
-      const n = typeof v === "number" ? v : typeof v === "string" && v.trim() !== "" ? Number(v) : NaN;
-      return Number.isFinite(n) ? n : fallback;
+    const rows = [];
+    if (Array.isArray(raw)) rows.push(...raw);
+    else if (Array.isArray(raw?.data)) rows.push(...raw.data);
+    else if (Array.isArray(raw?.results)) rows.push(...raw.results);
+    else if (Array.isArray(raw?.logs)) rows.push(...raw.logs);
+    const toNum2 = (v, fallback = 0) => {
+      if (typeof v === "number") return Number.isFinite(v) ? v : fallback;
+      if (typeof v === "string" && v.trim() !== "") {
+        const n = Number(v);
+        return Number.isFinite(n) ? n : fallback;
+      }
+      return fallback;
     };
     const data = rows.map((r) => {
-      const promptTokens = toNum(
+      const promptTokens = toNum2(
         r?.prompt_tokens ?? r?.promptTokens ?? r?.input_tokens
       );
-      const completionTokens = toNum(
+      const completionTokens = toNum2(
         r?.completion_tokens ?? r?.completionTokens ?? r?.output_tokens
       );
       return {
         requestId: String(r?.request_id ?? r?.requestId ?? r?.id ?? ""),
         model: String(r?.model ?? r?.model_name ?? ""),
-        spend: toNum(r?.spend ?? r?.cost),
+        spend: toNum2(r?.spend ?? r?.cost),
         promptTokens,
         completionTokens,
-        totalTokens: toNum(
+        totalTokens: toNum2(
           r?.total_tokens ?? r?.totalTokens,
           promptTokens + completionTokens
         ),
@@ -8725,7 +8733,7 @@ var LiteLLMClient = class {
     });
     return {
       data,
-      total: toNum(raw?.total ?? raw?.total_count, data.length),
+      total: toNum2(raw?.total ?? raw?.total_count, data.length),
       page,
       pageSize,
       truncated: data.length >= pageSize
@@ -9522,6 +9530,21 @@ function redactTeamMemberUsage(usage) {
   };
 }
 
+// src/langfuse.ts
+function readLangfuseConfig(config) {
+  const enabled = config.getOptionalBoolean("litellm.langfuse.enabled") ?? false;
+  return {
+    enabled,
+    baseUrl: config.getOptionalString("litellm.langfuse.baseUrl") ?? null,
+    publicKey: config.getOptionalString("litellm.langfuse.publicKey") ?? null,
+    secretKey: config.getOptionalString("litellm.langfuse.secretKey") ?? null,
+    userIdField: config.getOptionalString("litellm.langfuse.userIdField") ?? "userId"
+  };
+}
+function isLangfuseConfigured(cfg) {
+  return cfg.enabled && !!cfg.baseUrl && !!cfg.publicKey && !!cfg.secretKey;
+}
+
 // src/routes/middleware/withUser.ts
 function createRequireUser(ctx) {
   return async (req, res, next) => {
@@ -11558,6 +11581,143 @@ function registerTeamUsageRoutes(router, ctx) {
   }
 }
 
+// src/langfuseClient.ts
+var DEFAULT_TIMEOUT2 = 15e3;
+var LangfuseUpstreamError = class extends Error {
+  constructor(status, statusText) {
+    super(`Langfuse API error: ${status} ${statusText}`);
+    this.status = status;
+  }
+};
+async function queryMetrics(baseUrl, publicKey, secretKey, queryObj, timeout = DEFAULT_TIMEOUT2) {
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const url = `${baseUrl.replace(/\/$/, "")}/api/public/v2/metrics?query=${encodeURIComponent(
+      JSON.stringify(queryObj)
+    )}`;
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: {
+        Authorization: `Basic ${Buffer.from(`${publicKey}:${secretKey}`).toString("base64")}`
+      }
+    });
+    if (!response.ok) {
+      throw new LangfuseUpstreamError(response.status, response.statusText);
+    }
+    return response.json();
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+function toNum(v) {
+  if (typeof v === "number") return Number.isFinite(v) ? v : null;
+  if (typeof v === "string" && v.trim() !== "") {
+    const n = Number(v);
+    return Number.isFinite(n) ? n : null;
+  }
+  return null;
+}
+function pick(row, names) {
+  const scopes = [row, row?.metrics, row?.dimensions];
+  for (const scope of scopes) {
+    if (!scope || typeof scope !== "object") continue;
+    for (const name of names) {
+      const n = toNum(scope[name]);
+      if (n !== null) return n;
+    }
+  }
+  return null;
+}
+function pickText(row, names) {
+  const scopes = [row, row?.metrics, row?.dimensions];
+  for (const scope of scopes) {
+    if (!scope || typeof scope !== "object") continue;
+    for (const name of names) {
+      const v = scope[name];
+      if (typeof v === "string" && v) return v;
+    }
+  }
+  return null;
+}
+function metricRows(body) {
+  if (Array.isArray(body)) return body;
+  if (Array.isArray(body?.data)) return body.data;
+  if (Array.isArray(body?.results)) return body.results;
+  if (Array.isArray(body?.rows)) return body.rows;
+  return [];
+}
+function parseObservations(body) {
+  const byModel = metricRows(body).map((row) => ({
+    model: pickText(row, ["providedModelName", "model", "modelName"]) ?? "",
+    cost: pick(row, ["sum_totalCost", "totalCost", "sum_total_cost", "cost", "sum_cost"]) ?? 0,
+    tokens: pick(row, ["sum_totalTokens", "totalTokens", "sum_total_tokens", "tokens"]) ?? 0,
+    avgLatencyMs: pick(row, ["avg_latency", "latency", "avg_latency_ms", "latencyMs"]),
+    requests: pick(row, ["count_id", "count", "id_count", "requests", "observations"]) ?? 0
+  })).filter((r) => r.model !== "");
+  byModel.sort((a, b) => b.cost - a.cost);
+  return byModel;
+}
+function parseScores(body) {
+  const scores = metricRows(body).map((row) => ({
+    name: pickText(row, ["name", "scoreName"]) ?? "",
+    avg: pick(row, ["avg_value", "value", "avg"]) ?? NaN,
+    count: pick(row, ["count_id", "count", "id_count", "scores"]) ?? 0
+  })).filter((s) => s.name !== "" && Number.isFinite(s.avg));
+  scores.sort((a, b) => a.name.localeCompare(b.name));
+  return scores;
+}
+function toTimestamp(day2, endOfDay) {
+  if (/^\d{4}-\d{2}-\d{2}$/.test(day2)) {
+    return endOfDay ? `${day2}T23:59:59.999Z` : `${day2}T00:00:00.000Z`;
+  }
+  return day2;
+}
+async function getQualitySummary(cfg, opts) {
+  if (!isLangfuseConfigured(cfg)) return null;
+  const { baseUrl, publicKey, secretKey, userIdField } = cfg;
+  const fromTimestamp = toTimestamp(opts.from, false);
+  const toTimestampValue = toTimestamp(opts.to, true);
+  try {
+    const observations = await queryMetrics(baseUrl, publicKey, secretKey, {
+      view: "observations",
+      metrics: [
+        { measure: "totalCost", aggregation: "SUM" },
+        { measure: "totalTokens", aggregation: "SUM" },
+        { measure: "latency", aggregation: "AVG" },
+        { measure: "id", aggregation: "COUNT" }
+      ],
+      dimensions: [{ field: "providedModelName" }],
+      filters: [
+        { column: userIdField, operator: "=", value: opts.userId, type: "string" }
+      ],
+      fromTimestamp,
+      toTimestamp: toTimestampValue,
+      config: { row_limit: 50 }
+    });
+    let scores = [];
+    try {
+      const scoresBody = await queryMetrics(baseUrl, publicKey, secretKey, {
+        view: "scores-numeric",
+        metrics: [
+          { measure: "value", aggregation: "AVG" },
+          { measure: "id", aggregation: "COUNT" }
+        ],
+        dimensions: [{ field: "name" }],
+        fromTimestamp,
+        toTimestamp: toTimestampValue,
+        config: { row_limit: 50 }
+      });
+      scores = parseScores(scoresBody);
+    } catch {
+      scores = [];
+    }
+    return { configured: true, byModel: parseObservations(observations), scores };
+  } catch {
+    return null;
+  }
+}
+
 // src/routes/insights.ts
 function registerInsightsRoutes(router, ctx) {
   const { client, catalogClient, auth, logger, auditGroup } = ctx;
@@ -11596,6 +11756,41 @@ function registerInsightsRoutes(router, ctx) {
       sendError(res, error, logger, "fetch spend logs");
     }
   });
+  router.get("/insights/quality", requireUser, async (req, res) => {
+    try {
+      const { start_date, end_date, userId } = req.query;
+      if (!start_date || !end_date) {
+        res.status(400).json({ error: "start_date and end_date are required" });
+        return;
+      }
+      const tokenEntityRef = res.locals.tokenEntityRef;
+      await getProvisionedUser(ctx, res);
+      const callerId = res.locals.userId;
+      const isAudit = auditGroup ? await isUserMemberOfGroup(
+        tokenEntityRef,
+        auditGroup,
+        catalogClient,
+        auth,
+        logger
+      ) : false;
+      if (!isLangfuseConfigured(ctx.langfuseCfg)) {
+        res.json({ configured: false });
+        return;
+      }
+      const summary = await getQualitySummary(ctx.langfuseCfg, {
+        from: start_date,
+        to: end_date,
+        userId: isAudit ? userId || callerId : callerId
+      });
+      res.json(summary ?? { configured: false });
+    } catch (error) {
+      if (error instanceof ProvisioningError) {
+        res.status(error.status).json(error.body);
+        return;
+      }
+      sendError(res, error, logger, "fetch quality summary");
+    }
+  });
 }
 
 // node_modules/jose/dist/node/esm/runtime/base64url.js
@@ -13336,6 +13531,7 @@ async function createRouter(options) {
   const teamBudgetVisibility = readTeamBudgetVisibility(config);
   const opencodeCfg = readOpencodeConfig(config);
   const memberBreakdownCfg = readMemberBreakdownConfig(config);
+  const langfuseCfg = readLangfuseConfig(config);
   const catalogClient = options.catalogClient ?? new import_catalog_client.CatalogClient({ discoveryApi: discovery });
   if (provisioningEnabled) {
     logger.info(
@@ -13382,6 +13578,7 @@ async function createRouter(options) {
     teamBudgetVisibility,
     opencodeCfg,
     memberBreakdownCfg,
+    langfuseCfg,
     // Helper functions - defined inline below this object
     authorizeKeyAction: async (req, keyId) => {
       if (!req.res || !req.res.locals.tokenEntityRef || !req.res.locals.userId) {