@acarmisc/backstage-plugin-litellm-backend 0.11.0 → 0.12.0

package/dist/router.js

@@ -46,6 +46,44 @@ const bridge_1 = require("./bridge");
 const permissions_1 = require("./permissions");
 const teamAdmin_1 = require("./teamAdmin");
 const teamCreateInFlight = new Map();
+/** Backoff schedule for `withTeamFetchRetry` — 3 retries over ~1.3s total. */
+const TEAM_FETCH_RETRY_DELAYS_MS = [100, 300, 900];
+/**
+ * Only 5xx (and network/timeout) failures are treated as transient — a 4xx
+ * like "team not found" or "forbidden" is a deterministic answer from the
+ * upstream and retrying it would just add latency for the same result.
+ */
+function isRetryableTeamFetchError(err) {
+    if (err instanceof client_1.LiteLLMUpstreamError) {
+        return err.status >= 500;
+    }
+    return true;
+}
+/**
+ * Retries a LiteLLM team-info fetch with the given backoff schedule.
+ *
+ * LiteLLM proxies are commonly run behind multiple replicas; a request can
+ * land on a replica that hasn't yet caught up with a very recent write
+ * (team create/update, membership change), or on one that is transiently
+ * unhealthy. Both surface as a failed `getTeamInfo` call that would
+ * otherwise be swallowed (e.g. GET /teams silently drops the team from the
+ * response) or bubble up as a spurious 500/404 right after a write this
+ * same request just made. A short retry smooths over that window without
+ * masking a genuine, persistent failure.
+ */
+async function withTeamFetchRetry(fn, delaysMs = TEAM_FETCH_RETRY_DELAYS_MS) {
+    for (let attempt = 0;; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            if (attempt >= delaysMs.length || !isRetryableTeamFetchError(err)) {
+                throw err;
+            }
+            await new Promise(resolve => setTimeout(resolve, delaysMs[attempt]));
+        }
+    }
+}
 async function createRouter(options) {
     const { config, logger, auth, discovery, permissions } = options;
     const baseUrl = config.getString('litellm.baseUrl');
@@ -510,8 +548,8 @@ async function createRouter(options) {
                 res.json([]);
                 return;
             }
-            const teams = await Promise.all(userInfo.teams.map(teamId => client.getTeamInfo(teamId).catch(err => {
-                logger.warn(`Failed to fetch team ${teamId}: ${err.message}`);
+            const teams = await Promise.all(userInfo.teams.map(teamId => withTeamFetchRetry(() => client.getTeamInfo(teamId)).catch(err => {
+                logger.warn(`Failed to fetch team ${teamId} after retries: ${err.message}`);
                 return null;
             })));
             res.json(teams.filter(Boolean));
@@ -773,7 +811,7 @@ async function createRouter(options) {
         }
         let existing;
         try {
-            existing = await client.getTeamInfo(teamId);
+            existing = await withTeamFetchRetry(() => client.getTeamInfo(teamId));
         }
         catch (err) {
             if (err instanceof client_1.LiteLLMUpstreamError && err.status === 404) {
@@ -885,7 +923,7 @@ async function createRouter(options) {
                 member: litellmUserId,
                 owningGroup,
             });
-            const updated = await client.getTeamInfo(teamId);
+            const updated = await withTeamFetchRetry(() => client.getTeamInfo(teamId));
             res.json(updated);
         }
         catch (err) {
@@ -920,7 +958,7 @@ async function createRouter(options) {
                 member: litellmUserId,
                 owningGroup,
             });
-            const updated = await client.getTeamInfo(teamId);
+            const updated = await withTeamFetchRetry(() => client.getTeamInfo(teamId));
             res.json(updated);
         }
         catch (err) {

package/dist/types.cjs.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/types.ts"],
-  "sourcesContent": ["export interface UserInfo {\n  user_id: string;\n  user_email?: string;\n  email?: string;\n  teams?: string[];\n  models?: string[];\n  max_budget?: number;\n  spend?: number;\n  current_spend?: number;\n  soft_limit?: number;\n  hard_limit?: number;\n  /** Backstage-computed: true when the user is a member of litellm.audit.group */\n  can_view_audit?: boolean;\n}\n\nexport interface TeamMember {\n  user_id: string;\n  user_email?: string;\n  role: 'admin' | 'user';\n}\n\nexport interface TeamObjectPermission {\n  /** Vector store ids/names attached to the team. */\n  vector_stores?: string[];\n  /** MCP server ids/names attached to the team. */\n  mcp_servers?: string[];\n  /** MCP access group names attached to the team. */\n  mcp_access_groups?: string[];\n}\n\nexport interface TeamInfo {\n  team_id: string;\n  team_alias?: string;\n  max_budget?: number;\n  /** Spend-reset period for max_budget, e.g. \"30d\". */\n  budget_duration?: string;\n  spend: number;\n  members_with_roles?: TeamMember[];\n  models?: string[];\n  tpm_limit?: number;\n  rpm_limit?: number;\n  /** Arbitrary metadata stored on the team record. */\n  metadata?: Record<string, unknown>;\n  /** Knowledge bases (vector stores) and MCP servers attached to the team. */\n  object_permission?: TeamObjectPermission;\n  /** Whether the team is blocked/deactivated. */\n  blocked?: boolean;\n  /** Max budget available to individual team members (if set per-member). */\n  team_member_budget?: number;\n}\n\nexport interface CreateTeamRequest {\n  team_alias: string;\n  models?: string[];\n  max_budget?: number;\n  budget_duration?: string;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  metadata?: Record<string, unknown>;\n  object_permission?: TeamObjectPermission;\n}\n\nexport interface UpdateTeamRequest extends Partial<CreateTeamRequest> {\n  team_id: string;\n  blocked?: boolean;\n}\n\nexport interface CreateTeamResponse {\n  team_id: string;\n  team_alias?: string;\n}\n\nexport interface VirtualKey {\n  key: string;\n  token: string;\n  key_alias?: string;\n  created_at: string;\n  expires_at?: string;\n  spend: number;\n  max_budget?: number;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  models?: string[];\n  user_id?: string;\n  blocked?: boolean;\n}\n\n/**\n * Shape of a single entry inside LiteLLM's `/user/info` `keys` array.\n * Differs from VirtualKey: uses `expires` (not `expires_at`), exposes\n * both a hashed `token` and a masked `key_name`, and fields are nullable\n * rather than optional.\n */\nexport interface LiteLLMUserKey {\n  token: string;\n  key_name?: string;\n  key_alias?: string | null;\n  spend?: number;\n  expires?: string | null;\n  models?: string[];\n  tpm_limit?: number | null;\n  rpm_limit?: number | null;\n  max_budget?: number | null;\n  user_id?: string | null;\n  team_id?: string | null;\n  created_at: string;\n  blocked?: boolean | null;\n}\n\nexport interface ModelInfo {\n  model_name: string;\n  mode: string;\n  supports_function_calling?: boolean;\n  supports_vision?: boolean;\n  input_cost_per_token?: number;\n  output_cost_per_token?: number;\n  max_input_tokens?: number;\n  max_output_tokens?: number;\n  /** Access group names this model belongs to (litellm.model_info.access_groups). A team's `models` list can reference a group name instead of a literal model_name. */\n  access_groups?: string[];\n}\n\nexport interface UsageModelBreakdown {\n  total_spend: number;\n  total_tokens: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n}\n\nexport interface UsageKeyBreakdown {\n  key_alias?: string;\n  team_id?: string | null;\n  models: string[];\n  total_spend: number;\n  total_tokens: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n}\n\nexport interface UsageDailyPoint {\n  date: string;\n  spend: number;\n  total_tokens: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n}\n\nexport interface UsageDailyModelPoint {\n  date: string;\n  model: string;\n  spend: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  total_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n}\n\nexport interface UsageMetrics {\n  total_spend: number;\n  total_tokens: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n  usage_by_model: Record<string, UsageModelBreakdown>;\n  usage_by_key: Record<string, UsageKeyBreakdown>;\n  daily_usage: UsageDailyPoint[];\n  daily_by_model: UsageDailyModelPoint[];\n}\n\nexport interface GenerateKeyRequest {\n  alias?: string;\n  models?: string[];\n  duration?: string;\n  max_budget?: number;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  user_id?: string;\n  team_id?: string;\n  key_type?: string;\n  metadata?: Record<string, string>;\n}\n\nexport interface UpdateKeyRequest {\n  key: string;\n  key_alias?: string;\n  models?: string[];\n  max_budget?: number;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  team_id?: string;\n  duration?: string;\n}\n\nexport interface GenerateKeyResponse {\n  key: string;\n  key_alias?: string;\n  expires_at?: string;\n  max_budget?: number;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  models?: string[];\n}\n\nexport interface DeleteKeyRequest {\n  keys: string[];\n}\n\nexport interface LiteLLMConfig {\n  baseUrl: string;\n  masterKey: string;\n}\n\nexport interface ProvisioningDefaults {\n  maxBudget: number;\n  budgetDuration: string;\n  models: string[];\n  teams: string[];\n  tpmLimit?: number;\n  rpmLimit?: number;\n  /**\n   * LiteLLM user role applied on /user/new. Defaults to \"internal_user\"\n   * which grants self-service Create/Delete/View on the user's own keys.\n   * Valid values: proxy_admin, proxy_admin_viewer, internal_user,\n   * internal_user_viewer, team.\n   */\n  userRole?: string;\n  metadata: Record<string, string>;\n}\n\nexport interface RoleConfig {\n  group: string;\n  maxBudget?: number;\n  budgetDuration?: string;\n  models?: string[];\n  teams?: string[];\n  tpmLimit?: number;\n  rpmLimit?: number;\n  userRole?: string;\n  metadata?: Record<string, string>;\n}\n\nexport interface CreateUserRequest {\n  user_id: string;\n  user_email?: string;\n  user_alias?: string;\n  user_role?: string;\n  max_budget?: number;\n  budget_duration?: string;\n  models?: string[];\n  teams?: string[];\n  tpm_limit?: number;\n  rpm_limit?: number;\n  metadata?: Record<string, string>;\n  auto_create_key?: boolean;\n}\n\nexport interface CreateUserResponse {\n  user_id: string;\n  user_email?: string;\n  max_budget?: number;\n  models?: string[];\n  teams?: string[];\n}\n\nexport interface AuditLogEntry {\n  id: string;\n  updated_at: string;\n  changed_by?: string;\n  changed_by_api_key?: string;\n  action?: string;\n  table_name?: string;\n  object_id?: string;\n  before_value?: Record<string, unknown> | null;\n  updated_values?: Record<string, unknown> | null;\n}\n\nexport interface PaginatedAuditLogs {\n  audit_logs: AuditLogEntry[];\n  total: number;\n  page: number;\n  page_size: number;\n  total_pages: number;\n}\n\nexport interface AuditLogsParams {\n  page?: number;\n  page_size?: number;\n  start_date?: string;\n  end_date?: string;\n  action?: string;\n  table_name?: string;\n  changed_by?: string;\n  sort_by?: string;\n  sort_order?: 'asc' | 'desc';\n}\n"],
+  "sourcesContent": ["export interface UserInfo {\n  user_id: string;\n  user_email?: string;\n  email?: string;\n  teams?: string[];\n  models?: string[];\n  max_budget?: number;\n  spend?: number;\n  current_spend?: number;\n  soft_limit?: number;\n  hard_limit?: number;\n  /** Spend-reset period for max_budget, e.g. \"30d\". */\n  budget_duration?: string;\n  /** Backstage-computed: true when the user is a member of litellm.audit.group */\n  can_view_audit?: boolean;\n}\n\nexport interface TeamMember {\n  user_id: string;\n  user_email?: string;\n  role: 'admin' | 'user';\n}\n\nexport interface TeamObjectPermission {\n  /** Vector store ids/names attached to the team. */\n  vector_stores?: string[];\n  /** MCP server ids/names attached to the team. */\n  mcp_servers?: string[];\n  /** MCP access group names attached to the team. */\n  mcp_access_groups?: string[];\n}\n\nexport interface TeamInfo {\n  team_id: string;\n  team_alias?: string;\n  max_budget?: number;\n  /** Spend-reset period for max_budget, e.g. \"30d\". */\n  budget_duration?: string;\n  spend: number;\n  members_with_roles?: TeamMember[];\n  models?: string[];\n  tpm_limit?: number;\n  rpm_limit?: number;\n  /** Arbitrary metadata stored on the team record. */\n  metadata?: Record<string, unknown>;\n  /** Knowledge bases (vector stores) and MCP servers attached to the team. */\n  object_permission?: TeamObjectPermission;\n  /** Whether the team is blocked/deactivated. */\n  blocked?: boolean;\n  /** Max budget available to individual team members (if set per-member). */\n  team_member_budget?: number;\n}\n\nexport interface CreateTeamRequest {\n  team_alias: string;\n  models?: string[];\n  max_budget?: number;\n  budget_duration?: string;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  metadata?: Record<string, unknown>;\n  object_permission?: TeamObjectPermission;\n}\n\nexport interface UpdateTeamRequest extends Partial<CreateTeamRequest> {\n  team_id: string;\n  blocked?: boolean;\n}\n\nexport interface CreateTeamResponse {\n  team_id: string;\n  team_alias?: string;\n}\n\nexport interface VirtualKey {\n  key: string;\n  token: string;\n  key_alias?: string;\n  created_at: string;\n  expires_at?: string;\n  spend: number;\n  max_budget?: number;\n  budget_duration?: string;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  models?: string[];\n  user_id?: string;\n  blocked?: boolean;\n}\n\n/**\n * Shape of a single entry inside LiteLLM's `/user/info` `keys` array.\n * Differs from VirtualKey: uses `expires` (not `expires_at`), exposes\n * both a hashed `token` and a masked `key_name`, and fields are nullable\n * rather than optional.\n */\nexport interface LiteLLMUserKey {\n  token: string;\n  key_name?: string;\n  key_alias?: string | null;\n  spend?: number;\n  expires?: string | null;\n  models?: string[];\n  tpm_limit?: number | null;\n  rpm_limit?: number | null;\n  max_budget?: number | null;\n  budget_duration?: string | null;\n  user_id?: string | null;\n  team_id?: string | null;\n  created_at: string;\n  blocked?: boolean | null;\n}\n\nexport interface ModelInfo {\n  model_name: string;\n  mode: string;\n  supports_function_calling?: boolean;\n  supports_vision?: boolean;\n  input_cost_per_token?: number;\n  output_cost_per_token?: number;\n  max_input_tokens?: number;\n  max_output_tokens?: number;\n  /** Access group names this model belongs to (litellm.model_info.access_groups). A team's `models` list can reference a group name instead of a literal model_name. */\n  access_groups?: string[];\n}\n\nexport interface UsageModelBreakdown {\n  total_spend: number;\n  total_tokens: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n}\n\nexport interface UsageKeyBreakdown {\n  key_alias?: string;\n  team_id?: string | null;\n  models: string[];\n  total_spend: number;\n  total_tokens: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n}\n\nexport interface UsageDailyPoint {\n  date: string;\n  spend: number;\n  total_tokens: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n}\n\nexport interface UsageDailyModelPoint {\n  date: string;\n  model: string;\n  spend: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  total_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n}\n\nexport interface UsageMetrics {\n  total_spend: number;\n  total_tokens: number;\n  prompt_tokens: number;\n  completion_tokens: number;\n  api_requests: number;\n  successful_requests: number;\n  failed_requests: number;\n  usage_by_model: Record<string, UsageModelBreakdown>;\n  usage_by_key: Record<string, UsageKeyBreakdown>;\n  daily_usage: UsageDailyPoint[];\n  daily_by_model: UsageDailyModelPoint[];\n}\n\nexport interface GenerateKeyRequest {\n  alias?: string;\n  models?: string[];\n  duration?: string;\n  max_budget?: number;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  user_id?: string;\n  team_id?: string;\n  key_type?: string;\n  metadata?: Record<string, string>;\n}\n\nexport interface UpdateKeyRequest {\n  key: string;\n  key_alias?: string;\n  models?: string[];\n  max_budget?: number;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  team_id?: string;\n  duration?: string;\n}\n\nexport interface GenerateKeyResponse {\n  key: string;\n  key_alias?: string;\n  expires_at?: string;\n  max_budget?: number;\n  tpm_limit?: number;\n  rpm_limit?: number;\n  models?: string[];\n}\n\nexport interface DeleteKeyRequest {\n  keys: string[];\n}\n\nexport interface LiteLLMConfig {\n  baseUrl: string;\n  masterKey: string;\n}\n\nexport interface ProvisioningDefaults {\n  maxBudget: number;\n  budgetDuration: string;\n  models: string[];\n  teams: string[];\n  tpmLimit?: number;\n  rpmLimit?: number;\n  /**\n   * LiteLLM user role applied on /user/new. Defaults to \"internal_user\"\n   * which grants self-service Create/Delete/View on the user's own keys.\n   * Valid values: proxy_admin, proxy_admin_viewer, internal_user,\n   * internal_user_viewer, team.\n   */\n  userRole?: string;\n  metadata: Record<string, string>;\n}\n\nexport interface RoleConfig {\n  group: string;\n  maxBudget?: number;\n  budgetDuration?: string;\n  models?: string[];\n  teams?: string[];\n  tpmLimit?: number;\n  rpmLimit?: number;\n  userRole?: string;\n  metadata?: Record<string, string>;\n}\n\nexport interface CreateUserRequest {\n  user_id: string;\n  user_email?: string;\n  user_alias?: string;\n  user_role?: string;\n  max_budget?: number;\n  budget_duration?: string;\n  models?: string[];\n  teams?: string[];\n  tpm_limit?: number;\n  rpm_limit?: number;\n  metadata?: Record<string, string>;\n  auto_create_key?: boolean;\n}\n\nexport interface CreateUserResponse {\n  user_id: string;\n  user_email?: string;\n  max_budget?: number;\n  models?: string[];\n  teams?: string[];\n}\n\nexport interface AuditLogEntry {\n  id: string;\n  updated_at: string;\n  changed_by?: string;\n  changed_by_api_key?: string;\n  action?: string;\n  table_name?: string;\n  object_id?: string;\n  before_value?: Record<string, unknown> | null;\n  updated_values?: Record<string, unknown> | null;\n}\n\nexport interface PaginatedAuditLogs {\n  audit_logs: AuditLogEntry[];\n  total: number;\n  page: number;\n  page_size: number;\n  total_pages: number;\n}\n\nexport interface AuditLogsParams {\n  page?: number;\n  page_size?: number;\n  start_date?: string;\n  end_date?: string;\n  action?: string;\n  table_name?: string;\n  changed_by?: string;\n  sort_by?: string;\n  sort_order?: 'asc' | 'desc';\n}\n"],
   "mappings": ";;;;;;;;;;;;;;;;AAAA;AAAA;",
   "names": []
 }

package/dist/types.d.ts

@@ -9,6 +9,8 @@ export interface UserInfo {
     current_spend?: number;
     soft_limit?: number;
     hard_limit?: number;
+    /** Spend-reset period for max_budget, e.g. "30d". */
+    budget_duration?: string;
     /** Backstage-computed: true when the user is a member of litellm.audit.group */
     can_view_audit?: boolean;
 }
@@ -71,6 +73,7 @@ export interface VirtualKey {
     expires_at?: string;
     spend: number;
     max_budget?: number;
+    budget_duration?: string;
     tpm_limit?: number;
     rpm_limit?: number;
     models?: string[];
@@ -93,6 +96,7 @@ export interface LiteLLMUserKey {
     tpm_limit?: number | null;
     rpm_limit?: number | null;
     max_budget?: number | null;
+    budget_duration?: string | null;
     user_id?: string | null;
     team_id?: string | null;
     created_at: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@acarmisc/backstage-plugin-litellm-backend",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "The Backstage backend plugin for LiteLLM governance",
   "backstage": {
     "role": "backend-plugin",