@careob/llm-gateway 1.0.0 → 1.1.0

package/README.md

@@ -0,0 +1,202 @@
+# @careob/llm-gateway
+
+Lightweight LLM gateway for Node.js with multi-key rotation, rate-limit handling, provider failover, and spend tracking.
+
+## Features
+
+- **12+ providers** — OpenAI, Anthropic, Gemini, OpenRouter, DeepSeek, Groq, Mistral, Together, Qwen, Zhipu, Moonshot, Yi
+- **Key rotation** — round-robin across multiple API keys per provider
+- **Rate-limit handling** — automatic cooldown, exponential backoff with jitter on 429s
+- **Retry classification** — 400s fail fast, 401/403 quarantine keys, 429/5xx retry with backoff
+- **Provider failover** — capability-aware fallback to alternate providers
+- **Safe errors** — API keys and raw provider bodies are never exposed in error messages
+- **AbortSignal** — cancel requests with a caller-provided `AbortSignal`
+- **Spend tracking** — per-request cost estimation and usage logging
+- **Streaming** — SSE streaming with stream-idle timeout detection
+- **Express middleware** — drop-in proxy for Express apps
+- **Auto-discovery** — detects API keys from environment variables
+- **Zero dependencies** — only uses `express` as an optional peer dependency
+
+## Install
+
+```bash
+npm install @careob/llm-gateway
+```
+
+## Quick Start
+
+```typescript
+import { LLMGateway } from '@careob/llm-gateway';
+
+const gateway = new LLMGateway({
+  keys: [
+    'sk-proj-your-openai-key',
+    { key: 'sk-ant-your-anthropic-key', provider: 'anthropic' },
+  ],
+  defaultModel: 'gpt-4o',
+});
+
+const response = await gateway.chat({
+  messages: [{ role: 'user', content: 'Hello!' }],
+});
+
+console.log(response.choices[0].message.content);
+```
+
+### Auto-discover keys from environment
+
+```typescript
+// Reads OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY, etc.
+const gateway = LLMGateway.fromEnv({ defaultModel: 'gpt-4o' });
+```
+
+## Multi-Key Rotation
+
+Pass multiple keys for the same provider to distribute load:
+
+```typescript
+const gateway = new LLMGateway({
+  keys: [
+    { key: 'sk-proj-key-1', provider: 'openai', label: 'prod-1' },
+    { key: 'sk-proj-key-2', provider: 'openai', label: 'prod-2' },
+    { key: 'sk-proj-key-3', provider: 'openai', label: 'prod-3' },
+  ],
+  defaultModel: 'gpt-4o',
+});
+```
+
+Keys are rotated round-robin. When a key hits a 429, it's automatically cooled down and the next key is used.
+
+## Streaming
+
+```typescript
+for await (const chunk of gateway.chatStream({
+  model: 'gpt-4o',
+  messages: [{ role: 'user', content: 'Write a poem' }],
+})) {
+  process.stdout.write(chunk.choices[0]?.delta?.content ?? '');
+}
+```
+
+## OpenRouter
+
+Access 200+ models through a single API key via [OpenRouter](https://openrouter.ai):
+
+```typescript
+const gateway = new LLMGateway({
+  keys: ['sk-or-v1-your-openrouter-key'],
+  defaultModel: 'openai/gpt-4o',
+  openRouterReferer: 'https://your-app.com',
+  openRouterTitle: 'Your App Name',
+});
+
+// Use namespaced model identifiers
+const response = await gateway.chat({
+  model: 'anthropic/claude-sonnet-4',
+  messages: [{ role: 'user', content: 'Hello!' }],
+});
+```
+
+OpenRouter keys (`sk-or-*`) are auto-detected. Models use namespaced identifiers like `openai/gpt-4o`, `anthropic/claude-sonnet-4`, `google/gemini-2.5-flash`, etc.
+
+## Express Proxy
+
+Mount the gateway as an API endpoint in your Express app:
+
+```typescript
+import express from 'express';
+import { LLMGateway, createExpressProxy } from '@careob/llm-gateway';
+
+const app = express();
+app.use(express.json());
+
+const gateway = LLMGateway.fromEnv({ defaultModel: 'gpt-4o' });
+const proxy = createExpressProxy(gateway, {
+  authorize: (req) => req.headers['x-api-key'] === 'your-secret',
+  extractMetadata: (req) => ({ userId: req.headers['x-user-id'] as string }),
+});
+
+app.post('/v1/chat/completions', proxy.chatCompletions);
+app.get('/v1/health', proxy.health);
+
+app.listen(3000);
+```
+
+## Spend Tracking
+
+```typescript
+const gateway = new LLMGateway({
+  keys: ['sk-proj-...'],
+  defaultModel: 'gpt-4o',
+  onUsage: (log) => {
+    console.log(`${log.model} | ${log.totalTokens} tokens | $${log.estimatedCostUsd.toFixed(6)}`);
+    // Save to your database, send to analytics, etc.
+  },
+  onAllKeysExhausted: (provider) => {
+    console.warn(`All ${provider} keys exhausted!`);
+  },
+});
+```
+
+## Configuration
+
+| Option | Default | Description |
+|---|---|---|
+| `keys` | *required* | Array of API key strings or `KeyInput` objects |
+| `defaultModel` | — | Model to use when not specified in request |
+| `maxRetries` | `3` | Retries across keys before failing |
+| `cooldownMs` | `60000` | Cooldown duration for rate-limited keys (ms) |
+| `timeoutMs` | `30000` | Connection timeout (ms) |
+| `streamTimeoutMs` | `15000` | Stream idle timeout — no data received (ms) |
+| `onUsage` | — | Callback for usage/spend logging |
+| `onAllKeysExhausted` | — | Callback when all keys for a provider are exhausted |
+| `openRouterReferer` | — | Your site URL for OpenRouter rankings (sent as `HTTP-Referer`) |
+| `openRouterTitle` | — | Your app name for OpenRouter rankings (sent as `X-Title`) |
+
+### KeyInput
+
+```typescript
+{
+  key: string;
+  provider?: string;   // Auto-detected from key prefix if omitted
+  rpm?: number;        // Requests per minute limit
+  tpm?: number;        // Tokens per minute limit
+  label?: string;      // Human-readable label (e.g. "prod-key-1")
+}
+```
+
+## Supported Providers
+
+| Provider | Format | Models |
+|---|---|---|
+| OpenAI | openai | gpt-4o, gpt-4o-mini, o1, o3-mini, o4-mini, ... |
+| Anthropic | anthropic | claude-opus-4, claude-sonnet-4, claude-haiku-3.5 |
+| Google Gemini | gemini | gemini-2.5-pro, gemini-2.5-flash, ... |
+| OpenRouter | openai | openai/gpt-4o, anthropic/claude-sonnet-4, google/gemini-2.5-flash, ... |
+| DeepSeek | openai | deepseek-chat, deepseek-coder, deepseek-reasoner |
+| Groq | openai | llama-3.3-70b, mixtral-8x7b, ... |
+| Mistral | openai | mistral-large, mistral-small, codestral |
+| Together AI | openai | Llama-3.3-70B, DeepSeek-R1, ... |
+| Qwen | openai | qwen-turbo, qwen-plus, qwen-max |
+| Zhipu | openai | glm-4, glm-4-flash, glm-4-plus |
+| Moonshot | openai | moonshot-v1-8k/32k/128k |
+| Yi | openai | yi-large, yi-medium, yi-spark |
+
+## Health Check
+
+```typescript
+const stats = gateway.getStats();
+// [{ provider: 'openai', label: 'prod-1', healthy: true, requestsThisMinute: 12, ... }]
+```
+
+## Development
+
+```bash
+npm test              # run tests
+npm run test:coverage # run tests with coverage report
+npm run ci            # typecheck + build + test
+```
+
+## License
+
+MIT

package/dist/gateway.d.ts

@@ -3,15 +3,11 @@ export declare class LLMGateway {
     private pool;
     private config;
     private providersByKey;
+    private openRouterDef;
     constructor(config: GatewayConfig);
-    /**
-     * Auto-discover keys from environment variables.
-     * Scans for OPENAI_API_KEY, ANTHROPIC_KEY, GEMINI_API_KEY_1, DEEPSEEK_KEY, etc.
-     */
     static fromEnv(overrides?: Omit<GatewayConfig, 'keys'>, env?: Record<string, string | undefined>): LLMGateway;
     chat(request: ChatRequest): Promise<ChatResponse>;
     chatStream(request: ChatRequest): AsyncGenerator<StreamChunk, void, unknown>;
-    /** Get health/stats for all registered keys */
     getStats(): {
         provider: string;
         label: string;
@@ -21,10 +17,10 @@ export declare class LLMGateway {
         totalErrors: number;
         cooldownUntil: number;
     }[];
-    /** List all registered providers */
     getProviders(): string[];
     private registerKeys;
-    private resolveProviders;
+    private getProviderDef;
+    private resolveTargets;
     private logUsage;
 }
 export type GatewayErrorCode = 'CONFIG_ERROR' | 'PROVIDER_ERROR' | 'NETWORK_ERROR' | 'TIMEOUT' | 'ALL_EXHAUSTED' | 'UNSUPPORTED';

package/dist/gateway.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"gateway.d.ts","sourceRoot":"","sources":["../src/gateway.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EACV,aAAa,EAEb,WAAW,EACX,YAAY,EACZ,WAAW,EAIZ,MAAM,YAAY,CAAC;AAEpB,qBAAa,UAAU;IACrB,OAAO,CAAC,IAAI,CAAU;IACtB,OAAO,CAAC,MAAM,CAAgF;IAC9F,OAAO,CAAC,cAAc,CAAkC;gBAE5C,MAAM,EAAE,aAAa;IAWjC;;;OAGG;IACH,MAAM,CAAC,OAAO,CACZ,SAAS,GAAE,IAAI,CAAC,aAAa,EAAE,MAAM,CAAM,EAC3C,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GACvC,UAAU;IAWP,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAwEhD,UAAU,CAAC,OAAO,EAAE,WAAW,GAAG,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,OAAO,CAAC;IAwFnF,+CAA+C;IAC/C,QAAQ;;;;;;;;;IAIR,oCAAoC;IACpC,YAAY;IAMZ,OAAO,CAAC,YAAY;IA8BpB,OAAO,CAAC,gBAAgB;IAqBxB,OAAO,CAAC,QAAQ;CAoCjB;AAED,MAAM,MAAM,gBAAgB,GACxB,cAAc,GACd,gBAAgB,GAChB,eAAe,GACf,SAAS,GACT,eAAe,GACf,aAAa,CAAC;AAElB,qBAAa,YAAa,SAAQ,KAAK;IAG5B,IAAI,EAAE,gBAAgB;IACtB,UAAU,CAAC,EAAE,MAAM;gBAF1B,OAAO,EAAE,MAAM,EACR,IAAI,EAAE,gBAAgB,EACtB,UAAU,CAAC,EAAE,MAAM,YAAA;CAK7B"}
+{"version":3,"file":"gateway.d.ts","sourceRoot":"","sources":["../src/gateway.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EACV,aAAa,EAEb,WAAW,EACX,YAAY,EACZ,WAAW,EAIZ,MAAM,YAAY,CAAC;AAEpB,qBAAa,UAAU;IACrB,OAAO,CAAC,IAAI,CAAU;IACtB,OAAO,CAAC,MAAM,CAAyG;IACvH,OAAO,CAAC,cAAc,CAAkC;IACxD,OAAO,CAAC,aAAa,CAA4B;gBAErC,MAAM,EAAE,aAAa;IAYjC,MAAM,CAAC,OAAO,CACZ,SAAS,GAAE,IAAI,CAAC,aAAa,EAAE,MAAM,CAAM,EAC3C,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GACvC,UAAU;IAWP,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAkGhD,UAAU,CAAC,OAAO,EAAE,WAAW,GAAG,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,OAAO,CAAC;IAuInF,QAAQ;;;;;;;;;IAIR,YAAY;IAMZ,OAAO,CAAC,YAAY;IAqCpB,OAAO,CAAC,cAAc;IAKtB,OAAO,CAAC,cAAc;IA4CtB,OAAO,CAAC,QAAQ;CAoCjB;AA6FD,MAAM,MAAM,gBAAgB,GACxB,cAAc,GACd,gBAAgB,GAChB,eAAe,GACf,SAAS,GACT,eAAe,GACf,aAAa,CAAC;AAElB,qBAAa,YAAa,SAAQ,KAAK;IAG5B,IAAI,EAAE,gBAAgB;IACtB,UAAU,CAAC,EAAE,MAAM;gBAF1B,OAAO,EAAE,MAAM,EACR,IAAI,EAAE,gBAAgB,EACtB,UAAU,CAAC,EAAE,MAAM,YAAA;CAK7B"}