@homeflare/distilled-litellm 0.2.0

package/package.json

@@ -0,0 +1,75 @@
+{
+  "name": "@homeflare/distilled-litellm",
+  "version": "0.2.0",
+  "description": "Interim npm home for @distilled.cloud/litellm — see docs/distilled-interim.md. Not published upstream yet; the kit aliases @distilled.cloud/litellm onto this package until it is.",
+  "homepage": "https://github.com/taslabs-net/homeflare-kit/tree/main/packages/distilled-litellm#readme",
+  "bugs": "https://github.com/taslabs-net/homeflare-kit/issues",
+  "license": "Apache-2.0",
+  "author": "Timothy Schneider",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/taslabs-net/homeflare-kit.git",
+    "directory": "packages/distilled-litellm"
+  },
+  "files": [
+    "dist",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "bun": "./src/index.ts",
+      "worker": "./src/index.ts",
+      "default": "./dist/index.js"
+    },
+    "./Credentials": {
+      "types": "./dist/credentials.d.ts",
+      "bun": "./src/credentials.ts",
+      "worker": "./src/credentials.ts",
+      "default": "./dist/credentials.js"
+    },
+    "./Errors": {
+      "types": "./dist/errors.d.ts",
+      "bun": "./src/errors.ts",
+      "worker": "./src/errors.ts",
+      "default": "./dist/errors.js"
+    },
+    "./Traits": {
+      "types": "./dist/traits.d.ts",
+      "bun": "./src/traits.ts",
+      "worker": "./src/traits.ts",
+      "default": "./dist/traits.js"
+    },
+    "./Protocol": {
+      "types": "./dist/protocol.d.ts",
+      "bun": "./src/protocol.ts",
+      "worker": "./src/protocol.ts",
+      "default": "./dist/protocol.js"
+    },
+    "./Retry": {
+      "types": "./dist/retry.d.ts",
+      "bun": "./src/retry.ts",
+      "worker": "./src/retry.ts",
+      "default": "./dist/retry.js"
+    },
+    "./*": {
+      "types": "./dist/services/*.d.ts",
+      "bun": "./src/services/*.ts",
+      "worker": "./src/services/*.ts",
+      "default": "./dist/services/*.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@distilled.cloud/core": "1.0.0-rc.12"
+  },
+  "peerDependencies": {
+    "effect": "4.0.0-rc.115"
+  }
+}

package/src/credentials.ts

@@ -0,0 +1,97 @@
+/**
+ * LiteLLM credentials — hand-written.
+ *
+ * The `Credentials` service resolves `{ apiKey, apiBaseUrl }` per request;
+ * the protocol layer formats the `Authorization: Bearer <apiKey>` header
+ * from it.
+ *
+ * LiteLLM Proxy is self-hosted, so — like Forgejo, and unlike a
+ * single-tenant SaaS — there is no default API root: the instance URL is
+ * part of the credential. `apiKey` is either the proxy's own master key
+ * (`general_settings.master_key` / `LITELLM_MASTER_KEY`) or a virtual key
+ * with the `proxy_admin` role — every `/key`, `/team`, `/user`, `/budget`,
+ * `/model` and `/config` mutation this package's services reach requires
+ * `user_api_key_dict.user_role == PROXY_ADMIN`
+ * (`litellm/proxy/auth/user_api_key_auth.py`, `proxy_server.py`'s
+ * `update_config_general_settings` at tag `v1.100.0` — see `protocol.ts`).
+ * A read-scoped virtual key can call the list/info GETs but every mutation
+ * answers `401` (LiteLLM does not distinguish "wrong key" from
+ * "right key, wrong role" — see `errors.ts`).
+ */
+import * as EffectConfig from "effect/Config";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import { ConfigError } from "@distilled.cloud/core/errors";
+
+export interface Config {
+  readonly apiKey: Redacted.Redacted<string>;
+  /** The proxy's origin, trailing slash stripped. No default — see the header. */
+  readonly apiBaseUrl: string;
+}
+
+export class Credentials extends Context.Service<
+  Credentials,
+  Effect.Effect<Config>
+>()("LitellmCredentials") {}
+
+const normalizeBaseUrl = (baseUrl: string): string => {
+  // Linear on purpose: a `/\/+$/` regex backtracks polynomially on a long run of
+  // "/" that is not at the end (CodeQL js/polynomial-redos, the same defect
+  // NetBox's and Paperless-ngx's sibling SDKs already fixed for this identical
+  // helper), and this is caller input.
+  let end = baseUrl.length;
+  while (end > 0 && baseUrl.charCodeAt(end - 1) === 47) end--;
+  return baseUrl.slice(0, end);
+};
+
+const envConfig = EffectConfig.all({
+  // `LITELLM_PROXY_URL` / `LITELLM_PROXY_API_KEY` are the names LiteLLM's
+  // own `litellm` CLI client reads (`litellm/proxy/client/cli/main.py` at
+  // the tag) — using the vendor's own names means a consumer who already
+  // has a `.env` for the `litellm` CLI needs nothing new for this package.
+  apiKey: EffectConfig.String("LITELLM_PROXY_API_KEY"),
+  baseUrl: EffectConfig.String("LITELLM_PROXY_URL"),
+});
+
+export const CredentialsFromEnv = Layer.succeed(
+  Credentials,
+  envConfig.pipe(
+    Effect.mapError(
+      () =>
+        new ConfigError({
+          message:
+            "LITELLM_PROXY_URL and LITELLM_PROXY_API_KEY environment variables are required",
+        }),
+    ),
+    Effect.map(({ apiKey, baseUrl }) => ({
+      apiKey: Redacted.make(apiKey),
+      apiBaseUrl: normalizeBaseUrl(baseUrl),
+    })),
+    Effect.orDie,
+  ),
+);
+
+/**
+ * Convenience layer from a plain key and the proxy's origin.
+ *
+ * ⛔ THE KEY IS NEVER A PROP (house rule, S25 — Alchemy persists attributes
+ * unencrypted). A caller building this layer from a stack should read the
+ * key fresh from the environment/a secret store, never from committed
+ * state.
+ */
+export const credentials = (config: {
+  readonly apiKey: string | Redacted.Redacted<string>;
+  readonly baseUrl: string;
+}): Layer.Layer<Credentials> =>
+  Layer.succeed(
+    Credentials,
+    Effect.succeed({
+      apiKey:
+        typeof config.apiKey === "string"
+          ? Redacted.make(config.apiKey)
+          : config.apiKey,
+      apiBaseUrl: normalizeBaseUrl(config.baseUrl),
+    }),
+  );

package/src/errors.ts

@@ -0,0 +1,108 @@
+/**
+ * LiteLLM-specific error types.
+ *
+ * Re-exports the common HTTP errors from core and adds the LiteLLM-specific
+ * unknown-error and parse-error wrappers.
+ *
+ * LiteLLM Proxy's failure envelope is genuinely heterogeneous — measured
+ * against the pinned `v1.100.0` source, NOT assumed from the spec (whose own
+ * `ErrorResponse` schema names only one of the real shapes). A `ProxyException`
+ * (`proxy_server.py`'s `openai_exception_handler` at the tag, the path a
+ * `router.py` "no healthy deployment" style failure funnels through) answers
+ * a BARE `{"error": {"message", "type", "param", "code"}}` — no `detail`
+ * wrapper at all. Plain FastAPI `HTTPException(status_code, detail=...)` —
+ * the dominant pattern across every `/key`, `/team`, `/user`, `/budget`,
+ * `/model` and `/config` route in `management_endpoints/*.py` — answers
+ * `{"detail": "<string>"}` OR `{"detail": {"error": ..., "message"?: ...}}`,
+ * NOT the `{"detail": [...]}` array `RequestValidationError` uses for 422
+ * (see `HTTPValidationError`/`ValidationError` in the vendor spec, decoded
+ * by core's shared `UnprocessableEntity`). See `protocol.ts`'s
+ * `errorEnvelope`/`messageFrom` for the exact extraction order — a sibling
+ * `message` field always wins over a short `error` code, verified against
+ * `litellm_pre_call_utils.py`'s `reject_url_valued_destination`. There is no
+ * machine-readable code on most routes (LiteLLM's own `code` field is
+ * usually the stringified status, e.g. `"400"`), so the SDK dispatches on
+ * the HTTP status: the per-operation classes generated from each
+ * operation's declared responses first, then core's shared status map, then
+ * {@link UnknownLitellmError}.
+ *
+ * ⛔ The vendor's own OpenAPI document (`litellm-openapi` @ 1.100.0, dumped
+ * by the vendor's own `gen-api-types.mjs` — see `scripts/convert.ts`)
+ * declares almost NO error responses beyond the FastAPI-automatic `422` —
+ * measured across all 696 paths, the only per-operation `400`/`401`/`403`/
+ * `404`/`408`/`429`/`500`/`503` declarations anywhere outside the
+ * `/openai/deployments/{model}/chat/completions` LLM-passthrough route are
+ * `GET /audit/{id}`'s `404`/`500` (both already in the spec — no patch
+ * needed). None of the admin surface this package's tag split covers
+ * (`model_management`, `key_management`, `team_management`,
+ * `internal_user_management`, `budget_management`, the untagged
+ * `/config/pass_through_endpoint*` family) declares anything beyond `422`.
+ * `401`/`429`/`500`/`502`/`503`/`504` still typecheck on every operation —
+ * they ride the shared `LitellmOpError` union via core's
+ * `DEFAULT_ERROR_STATUSES` (see `protocol.ts`) — but a status OUTSIDE that
+ * default set (`400`, `403`, `404`, `409`, `423`) that the spec doesn't
+ * declare for a given operation would decode correctly at runtime (core's
+ * shared `HTTP_STATUS_MAP` fallback in `protocol-rest.ts` still maps it) but
+ * NOT appear in that operation's TypeScript error union — an untyped escape
+ * the Typed Error Doctrine exists to catch. `patches/misc/…json` fixes the
+ * one such case this package has vendor-source evidence for (see its own
+ * comment); the rest of the admin surface is unverified beyond the spec —
+ * see this package's README.
+ */
+export {
+  BadGateway,
+  BadRequest,
+  Conflict,
+  ConfigError,
+  Forbidden,
+  GatewayTimeout,
+  InternalServerError,
+  Locked,
+  NotFound,
+  ServiceUnavailable,
+  TooManyRequests,
+  Unauthorized,
+  UnprocessableEntity,
+  HTTP_STATUS_MAP,
+  DEFAULT_ERRORS,
+  API_ERRORS,
+} from "@distilled.cloud/core/errors";
+import type { DefaultErrors as CoreDefaultErrors } from "@distilled.cloud/core/errors";
+
+import * as Schema from "effect/Schema";
+import * as Category from "@distilled.cloud/core/category";
+
+/**
+ * Unknown LiteLLM error — returned when a failed response's HTTP status has
+ * no mapped error class, or the body doesn't match the `ProxyException`
+ * envelope. Carries the raw body for later cataloging.
+ */
+export class UnknownLitellmError extends Schema.TaggedError<UnknownLitellmError>()(
+  "UnknownLitellmError",
+  {
+    code: Schema.optional(Schema.String),
+    message: Schema.optional(Schema.String),
+    body: Schema.Unknown,
+  },
+).pipe(Category.withServerError) {}
+
+/** Schema parse error wrapper. */
+export class LitellmParseError extends Schema.TaggedError<LitellmParseError>()(
+  "LitellmParseError",
+  {
+    body: Schema.Unknown,
+    cause: Schema.Unknown,
+  },
+).pipe(Category.withParseError) {}
+
+/**
+ * Errors any LiteLLM operation may surface in addition to the per-operation
+ * typed status errors.
+ */
+export type ClientErrors = UnknownLitellmError | LitellmParseError;
+
+/**
+ * Default LiteLLM operation errors: the shared HTTP status errors from core
+ * plus the client-level fallback/decode errors.
+ */
+export type DefaultErrors = CoreDefaultErrors | ClientErrors;

package/src/index.ts

@@ -0,0 +1,33 @@
+/**
+ * @distilled.cloud/litellm — LiteLLM Proxy admin API SDK for Effect.
+ *
+ * `./services` is generated by `scripts/generate.ts` from the Smithy models in
+ * `.generated-specs` (written by `scripts/convert.ts` from LiteLLM's pinned
+ * `app.openapi()` document, tag v1.100.0 — see README.md). Everything else
+ * in this folder is hand-written.
+ *
+ * One module per LiteLLM OpenAPI tag — `model_management`, `key_management`,
+ * `team_management`, `internal_user_management`, `budget_management`,
+ * `misc` (the untagged `/config/pass_through_endpoint*` family and other
+ * UI/debug/login/MCP routes), … — the same split LiteLLM's own API
+ * reference groups by. See README.md for the full tag → admin-surface map.
+ *
+ * @example
+ * ```ts
+ * import * as Litellm from "@distilled.cloud/litellm";
+ *
+ * const key = yield* Litellm.Services.keyManagement.generateKeyFn({
+ *   key_alias: "my-service",
+ * });
+ * ```
+ */
+export * from "./credentials.ts";
+export * from "./errors.ts";
+export * as T from "./traits.ts";
+export {
+  LitellmProtocol,
+  type LitellmOpError,
+  type LitellmOpContext,
+} from "./protocol.ts";
+export * as Retry from "./retry.ts";
+export * as Services from "./services/index.ts";

package/src/protocol.ts

@@ -0,0 +1,160 @@
+/**
+ * LitellmProtocol — the shared bearer-REST protocol instantiated for
+ * LiteLLM Proxy.
+ *
+ * LiteLLM speaks plain JSON with no consistent success envelope — some
+ * responses are the resource itself, others wrap a list in a named field
+ * (`{"keys": [...]}`, `{"endpoints": [...]}`) per-resource, decoded by each
+ * operation's own generated schema, not by this protocol.
+ *
+ * Authentication: `Authorization: Bearer <key>` — LiteLLM's own `litellm`
+ * CLI client uses it (`litellm/proxy/client/cli/main.py` at tag v1.100.0),
+ * and every `/key`, `/team`, `/user`, `/budget`, `/model` and `/config`
+ * route's `Depends(user_api_key_auth)` accepts it. ⚠️ VERIFIED AGAINST THE
+ * SPEC, AND FOUND WRONG: the vendor's own OpenAPI document (`litellm-openapi`
+ * @ 1.100.0) declares exactly one `securitySchemes` entry —
+ * `APIKeyHeader` (`type: apiKey`, `in: header`, `name: x-litellm-api-key`,
+ * `description: "Bearer token"`) — not a `Bearer`/`http`-scheme security
+ * requirement, and no operation references it (`security` is `null` at the
+ * document root and on every operation). FastAPI's OpenAPI generator only
+ * introspects a route's declared `Security()`/`Depends()` dependency for
+ * the scheme it documents; `user_api_key_auth` accepts an `Authorization:
+ * Bearer <key>` header by reading it directly out of the request in Python
+ * (not through a `Security(APIKeyHeader(...))` FastAPI can introspect), so
+ * the spec's own securityScheme names a header LiteLLM's docs, CLI and this
+ * package do not actually send. `Bearer` is used here because it is what
+ * the vendor's own client sends and what homeflare's existing
+ * `packages/alchemy/src/litellm/client.ts` already verified works — the
+ * spec is necessary input, not sufficient, exactly the case the Typed Error
+ * Doctrine exists for.
+ *
+ * Failure envelope: heterogeneous, measured against the pinned v1.100.0
+ * source (`litellm/proxy/proxy_server.py`'s `ProxyException` handler at the
+ * tag, and every `management_endpoints/*.py` file) — see `errorEnvelope`
+ * below and `errors.ts`'s header for the two shapes and why.
+ *
+ * Pagination: ad hoc per resource (`page`/`size` on some list routes,
+ * `offset`/`limit` on others), no shared cursor/token — see
+ * `scripts/generate.ts`.
+ */
+import * as Effect from "effect/Effect";
+import type * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import type * as HttpClient from "effect/unstable/http/HttpClient";
+import type * as HttpClientError from "effect/unstable/http/HttpClientError";
+import type * as API from "@distilled.cloud/core/api";
+import type { ConfigError } from "@distilled.cloud/core/errors";
+import {
+  makeRestProtocol,
+  type RestErrorEnvelope,
+} from "@distilled.cloud/core/protocol-rest";
+import { Credentials, type Config } from "./credentials.ts";
+import { UnknownLitellmError, type DefaultErrors } from "./errors.ts";
+
+/**
+ * Error channel shared by every generated LiteLLM operation. Generated
+ * service files annotate operations with `API.OperationMethod<I, O,
+ * LitellmOpError, LitellmOpContext>` explicitly so the compiler never
+ * infers these back out of the schema generics.
+ */
+export type LitellmOpError =
+  | DefaultErrors
+  | ConfigError
+  | HttpClientError.HttpClientError;
+
+/** Context (requirements) shared by every generated LiteLLM operation. */
+export type LitellmOpContext = Credentials | HttpClient.HttpClient;
+
+/**
+ * Extract `{ message, code }` from a failed response body. Two real shapes,
+ * both observed in the pinned source (see this file's header):
+ *
+ * 1. A genuine `ProxyException` (`proxy_server.py`'s
+ *    `openai_exception_handler`): `{"error": {"message", "type", "param",
+ *    "code"}}` — no `detail` wrapper at all.
+ * 2. Plain FastAPI `HTTPException(status_code, detail=...)` — the vast
+ *    majority of raises across `key_management_endpoints.py`,
+ *    `team_endpoints.py`, `internal_user_endpoints.py`,
+ *    `budget_management_endpoints.py`, `model_management_endpoints.py`,
+ *    `pass_through_endpoints.py` and `litellm_pre_call_utils.py` — where
+ *    `detail` is EITHER a plain string (`detail=f"..."`) →
+ *    `{"detail": "<string>"}`, OR a dict shaped one of two ways:
+ *    `{"error": "<full sentence>"}` (the common `budget_management_endpoints.py`
+ *    shape — the whole message IS the `error` field), or an OpenAI-triple
+ *    `{"error": "<short code>", "message": "<full sentence>", "param": ...}`
+ *    (`litellm_pre_call_utils.py`'s `reject_url_valued_destination`, hit on
+ *    EVERY proxied LLM call via `add_litellm_data_to_request` — not a corner
+ *    case). ⚠️ MEASURED: when both `message` and `error` are present on the
+ *    SAME object, `error` is a short code (`"invalid_request"`) and
+ *    `message` is the human-readable text — `message` must win, or a real,
+ *    actionable error collapses to a code word. There is no consistent
+ *    machine-readable `code` on this path — `code` is only ever a
+ *    `ProxyException`'s stringified HTTP status, so it is not threaded
+ *    through here.
+ */
+const messageFrom = (v: unknown): string | undefined => {
+  if (v === null || typeof v !== "object" || Array.isArray(v)) {
+    return undefined;
+  }
+  const o = v as Record<string, unknown>;
+  // A sibling `message` always wins over a short `error` code — see this
+  // function's header (the `reject_url_valued_destination` shape).
+  if (typeof o.message === "string") return o.message;
+  if (typeof o.error === "string") return o.error;
+  if (
+    o.error !== null &&
+    typeof o.error === "object" &&
+    !Array.isArray(o.error)
+  ) {
+    const inner = o.error as Record<string, unknown>;
+    if (typeof inner.message === "string") return inner.message;
+  }
+  return undefined;
+};
+
+const errorEnvelope = (body: unknown): RestErrorEnvelope | undefined => {
+  if (body === null || typeof body !== "object") return undefined;
+  const b = body as Record<string, unknown>;
+
+  // Shape 1: a bare ProxyException envelope, or a top-level string `error`
+  // (defensive — not observed, but cheap to accept).
+  if (typeof b.error === "string") return { message: b.error };
+  const proxyExceptionMessage = messageFrom(b);
+  if (proxyExceptionMessage !== undefined)
+    return { message: proxyExceptionMessage };
+
+  // Shape 2: FastAPI's `{"detail": ...}`.
+  if (typeof b.detail === "string") return { message: b.detail };
+  const detailMessage = messageFrom(b.detail);
+  if (detailMessage !== undefined) return { message: detailMessage };
+
+  return undefined;
+};
+
+export const LitellmProtocol: Layer.Layer<API.Protocol> =
+  makeRestProtocol<Config>({
+    // Resolved on the CALLING fiber per request (the layer is memoized per
+    // process); the Credentials service holds an effect, so a key rotated
+    // between calls is picked up without rebuilding the layer.
+    credentials: Effect.gen(function* () {
+      const resolve = yield* Credentials;
+      return yield* resolve;
+    }),
+    baseUrl: (creds) => creds.apiBaseUrl,
+    headers: (creds) => ({
+      Authorization: `Bearer ${Redacted.value(creds.apiKey)}`,
+      Accept: "application/json",
+    }),
+    errorEnvelope,
+    unknownError: ({ code, message, body }) =>
+      new UnknownLitellmError({
+        code:
+          typeof code === "string"
+            ? code
+            : code !== undefined
+              ? String(code)
+              : undefined,
+        message,
+        body,
+      }),
+  });

package/src/retry.ts

@@ -0,0 +1,74 @@
+/**
+ * LiteLLM retry surface — a veneer over `@distilled.cloud/core/retry`.
+ *
+ * The `Retry` service tag is threaded into every generated operation via
+ * `API.make({ retry: Retry })`, so a caller-installed policy applies to all
+ * LiteLLM calls below it and core's `makeDefault` is the fallback when none
+ * is provided.
+ *
+ * @example
+ * ```ts
+ * import * as Litellm from "@distilled.cloud/litellm";
+ *
+ * myEffect.pipe(Litellm.Retry.transient);
+ * ```
+ */
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Retries from "@distilled.cloud/core/retry";
+
+export type Options = Retries.Options;
+export type Factory = Retries.Factory;
+export type Policy = Retries.Policy;
+
+/** Context tag for configuring retry behavior of LiteLLM API calls. */
+export class Retry extends Context.Service<Retry, Policy>()("LitellmRetry") {}
+
+/** Provides a custom retry policy to every LiteLLM API call below it. */
+export const policy: {
+  (
+    options: Options,
+  ): <A, E, R>(
+    effect: Effect.Effect<A, E, R>,
+  ) => Effect.Effect<A, E, Exclude<R, Retry>>;
+  (
+    factory: Factory,
+  ): <A, E, R>(
+    effect: Effect.Effect<A, E, R>,
+  ) => Effect.Effect<A, E, Exclude<R, Retry>>;
+} = (optionsOrFactory: Options | Factory) =>
+  Effect.provide(Layer.succeed(Retry, optionsOrFactory));
+
+/** Disables all automatic retries. */
+export const none: <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+) => Effect.Effect<A, E, Exclude<R, Retry>> = Effect.provide(
+  Layer.succeed(Retry, { while: () => false }),
+);
+
+/**
+ * The default retry policy (core's): transient/throttling/retryable errors,
+ * capped exponential backoff with jitter, server `retryAfter` hints honored
+ * with precedence.
+ */
+export const makeDefault: Factory = Retries.makeDefault;
+
+export const jittered = Retries.jittered;
+export const capped = Retries.capped;
+
+/** Retry options that retry all throttling errors indefinitely. */
+export const throttlingOptions: Options = Retries.throttlingOptions;
+
+/** Retries all throttling errors indefinitely (honoring server hints). */
+export const throttling: <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+) => Effect.Effect<A, E, Exclude<R, Retry>> = policy(Retries.throttlingFactory);
+
+/** Retry options that retry all transient errors indefinitely. */
+export const transientOptions: Options = Retries.transientOptions;
+
+/** Retries all transient errors indefinitely (honoring server hints). */
+export const transient: <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+) => Effect.Effect<A, E, Exclude<R, Retry>> = policy(Retries.transientFactory);