@splashcodex/api-key-manager 4.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -186
- package/dist/index.d.ts +22 -2
- package/dist/index.js +144 -5
- package/dist/index.js.map +1 -1
- package/dist/persistence/file.d.ts +43 -0
- package/dist/persistence/file.js +82 -0
- package/dist/persistence/file.js.map +1 -0
- package/dist/persistence/index.d.ts +7 -0
- package/dist/persistence/index.js +12 -0
- package/dist/persistence/index.js.map +1 -0
- package/dist/persistence/memory.d.ts +30 -0
- package/dist/persistence/memory.js +43 -0
- package/dist/persistence/memory.js.map +1 -0
- package/dist/presets/base.d.ts +138 -0
- package/dist/presets/base.js +227 -0
- package/dist/presets/base.js.map +1 -0
- package/dist/presets/gemini.d.ts +60 -0
- package/dist/presets/gemini.js +77 -0
- package/dist/presets/gemini.js.map +1 -0
- package/dist/presets/index.d.ts +10 -0
- package/dist/presets/index.js +16 -0
- package/dist/presets/index.js.map +1 -0
- package/dist/presets/multi.d.ts +120 -0
- package/dist/presets/multi.js +210 -0
- package/dist/presets/multi.js.map +1 -0
- package/dist/presets/openai.d.ts +45 -0
- package/dist/presets/openai.js +62 -0
- package/dist/presets/openai.js.map +1 -0
- package/package.json +75 -3
- package/src/index.ts +163 -4
- package/src/persistence/file.ts +89 -0
- package/src/persistence/index.ts +7 -0
- package/src/persistence/memory.ts +43 -0
- package/src/presets/base.ts +323 -0
- package/src/presets/gemini.ts +84 -0
- package/src/presets/index.ts +10 -0
- package/src/presets/multi.ts +280 -0
- package/src/presets/openai.ts +69 -0
package/README.md
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
|
-
# @splashcodex/ApiKeyManager
|
|
1
|
+
# @splashcodex/ApiKeyManager v5.0 — Ecosystem Edition
|
|
2
2
|
|
|
3
|
-
> Universal API Key
|
|
3
|
+
> Universal API Key Management Gateway with Provider Presets, Built-in Persistence, and Multi-Provider Vault.
|
|
4
4
|
|
|
5
5
|
[](https://www.npmjs.com/package/@splashcodex/ApiKeyManager)
|
|
6
6
|
|
|
7
|
+
## New in v5.0 (Ecosystem Edition)
|
|
8
|
+
|
|
9
|
+
- **Provider Presets** — One-line setup for `GeminiManager`, `OpenAIManager`, and `MultiManager`.
|
|
10
|
+
- **Automatic Env Parsing** — Reads `GOOGLE_GEMINI_API_KEY`, `OPENAI_API_KEY`, etc. (supports JSON arrays and comma-separated strings).
|
|
11
|
+
- **Built-in Persistence** — `FileStorage` (survives restarts) and `MemoryStorage` included.
|
|
12
|
+
- **Singleton Pattern** — Thread-safe singletons with `getInstance()` and `Result<T>` pattern.
|
|
13
|
+
- **Multi-Provider Vault** — Manage multiple providers (`gemini`, `openai`, `anthropic`) from a single entry point.
|
|
14
|
+
|
|
7
15
|
## Features
|
|
8
16
|
|
|
9
17
|
- **Circuit Breaker** — Keys transition through `CLOSED → OPEN → HALF_OPEN → DEAD`
|
|
@@ -12,15 +20,9 @@
|
|
|
12
20
|
- **`execute()` Wrapper** — Single method: get key → call → latency → retry → fallback
|
|
13
21
|
- **Event Emitter** — Typed lifecycle hooks for monitoring & alerting
|
|
14
22
|
- **Auto-Retry with Backoff** — Built-in retry loop with exponential backoff + jitter
|
|
15
|
-
- **
|
|
16
|
-
- **
|
|
17
|
-
- **
|
|
18
|
-
- **Health Checks** — Periodic key validation and auto-recovery
|
|
19
|
-
- **Bulkhead / Concurrency** — Limits concurrent `execute()` calls
|
|
20
|
-
- **Semantic Cache** *(v4 NEW)* — Cosine-similarity cache with pluggable embeddings
|
|
21
|
-
- **Recursion Guard** *(v4 NEW)* — Prevents infinite loops when `getEmbedding` calls `execute()`
|
|
22
|
-
- **State Persistence** — Survives restarts via pluggable storage
|
|
23
|
-
- **100% Backward Compatible** — v1.x, v2.x, and v3.x code works without changes
|
|
23
|
+
- **Semantic Cache** — Cosine-similarity cache with pluggable embeddings
|
|
24
|
+
- **Streaming Support** — `executeStream()` with initial retry + cache replay
|
|
25
|
+
- **100% Backward Compatible** — v1.x through v4.x code works without changes
|
|
24
26
|
|
|
25
27
|
## Installation
|
|
26
28
|
|
|
@@ -28,230 +30,142 @@
|
|
|
28
30
|
npm install @splashcodex/api-key-manager
|
|
29
31
|
```
|
|
30
32
|
|
|
31
|
-
## Quick Start
|
|
33
|
+
## Quick Start (v5 Presets)
|
|
34
|
+
|
|
35
|
+
The fastest way to get started in any CodeDex repository.
|
|
36
|
+
|
|
37
|
+
### Gemini Preset
|
|
38
|
+
Reads `GOOGLE_GEMINI_API_KEY` or `GEMINI_API_KEY` from environment.
|
|
32
39
|
|
|
33
40
|
```typescript
|
|
34
|
-
import {
|
|
41
|
+
import { GeminiManager } from '@splashcodex/api-key-manager/presets/gemini';
|
|
35
42
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const key = manager.getKey();
|
|
39
|
-
manager.markSuccess(key!);
|
|
43
|
+
const result = GeminiManager.getInstance();
|
|
44
|
+
if (!result.success) throw result.error;
|
|
40
45
|
|
|
41
|
-
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
);
|
|
46
|
+
const gemini = result.data;
|
|
47
|
+
const response = await gemini.execute(async (key) => {
|
|
48
|
+
// result.data is the underlying ApiKeyManager
|
|
49
|
+
return await callGemini(key, "Hello!");
|
|
50
|
+
});
|
|
46
51
|
```
|
|
47
52
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
Automatically cache API responses by semantic similarity. Identical or near-identical prompts return cached results without consuming API quota.
|
|
53
|
+
### Multi-Provider Vault
|
|
54
|
+
Perfect for gateways or complex bots handling multiple models.
|
|
51
55
|
|
|
52
56
|
```typescript
|
|
53
|
-
import {
|
|
57
|
+
import { MultiManager } from '@splashcodex/api-key-manager/presets/multi';
|
|
54
58
|
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
getEmbedding: async (text) => {
|
|
60
|
-
// Your embedding function (e.g. OpenAI, Gemini, local model)
|
|
61
|
-
return await myEmbeddingModel.embed(text);
|
|
59
|
+
const vault = MultiManager.getInstance({
|
|
60
|
+
providers: {
|
|
61
|
+
gemini: { envKeys: ['GOOGLE_GEMINI_API_KEY'] },
|
|
62
|
+
openai: { envKeys: ['OPENAI_API_KEY'] }
|
|
62
63
|
}
|
|
63
|
-
|
|
64
|
-
});
|
|
64
|
+
}).data!;
|
|
65
65
|
|
|
66
|
-
//
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
// Second call → Semantic Cache HIT (no API call)
|
|
70
|
-
const r2 = await manager.execute(apiFn, { prompt: 'How is the weather today?' });
|
|
66
|
+
// Route by provider
|
|
67
|
+
const res = await vault.execute(fn, { provider: 'gemini' });
|
|
71
68
|
```
|
|
72
69
|
|
|
73
|
-
|
|
74
|
-
> the cache automatically skips on nested calls to prevent infinite recursion.
|
|
70
|
+
---
|
|
75
71
|
|
|
76
|
-
##
|
|
72
|
+
## v5.0 — Architecture & Persistence
|
|
77
73
|
|
|
78
|
-
|
|
74
|
+
### Built-in Persistence
|
|
75
|
+
State (cooling down keys, dead keys) now survives application restarts by default.
|
|
79
76
|
|
|
80
77
|
```typescript
|
|
78
|
+
import { FileStorage } from '@splashcodex/api-key-manager/persistence/file';
|
|
79
|
+
|
|
81
80
|
const manager = new ApiKeyManager(keys, {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
concurrency: 10
|
|
81
|
+
storage: new FileStorage({
|
|
82
|
+
filePath: './api_state.json'
|
|
83
|
+
})
|
|
86
84
|
});
|
|
87
|
-
|
|
88
|
-
const result = await manager.execute(
|
|
89
|
-
async (key, signal) => {
|
|
90
|
-
const res = await fetch(url, { headers: { 'x-api-key': key }, signal });
|
|
91
|
-
return res.json();
|
|
92
|
-
},
|
|
93
|
-
{ maxRetries: 3, timeoutMs: 10000 }
|
|
94
|
-
);
|
|
95
|
-
// Handles: key selection → cache → timeout → retry → fallback → latency tracking
|
|
96
85
|
```
|
|
97
86
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
Monitor every state change:
|
|
87
|
+
### Subpath Imports
|
|
88
|
+
To keep your bundles small, you can import only what you need:
|
|
101
89
|
|
|
102
90
|
```typescript
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
manager.on('keyRecovered', (key) => log(`Key ${key} recovered`));
|
|
106
|
-
manager.on('retry', (key, attempt, delay) => log(`Retry #${attempt} in ${delay}ms`));
|
|
107
|
-
manager.on('fallback', (reason) => log(`Fallback triggered: ${reason}`));
|
|
108
|
-
manager.on('allKeysExhausted', () => alert('No healthy keys!'));
|
|
109
|
-
manager.on('bulkheadRejected', () => metrics.increment('rejected'));
|
|
110
|
-
manager.on('healthCheckPassed', (key) => log(`${key} healthy`));
|
|
111
|
-
manager.on('healthCheckFailed', (key, err) => log(`${key} unhealthy`));
|
|
91
|
+
import { GeminiManager } from '@splashcodex/api-key-manager/presets/gemini';
|
|
92
|
+
import { FileStorage } from '@splashcodex/api-key-manager/persistence/file';
|
|
112
93
|
```
|
|
113
94
|
|
|
114
|
-
|
|
95
|
+
---
|
|
115
96
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
```typescript
|
|
119
|
-
import { ApiKeyManager, WeightedStrategy } from '@splashcodex/api-key-manager';
|
|
120
|
-
|
|
121
|
-
const manager = new ApiKeyManager(
|
|
122
|
-
[
|
|
123
|
-
{ key: 'free-key-1', weight: 1.0 },
|
|
124
|
-
{ key: 'free-key-2', weight: 1.0 },
|
|
125
|
-
{ key: 'paid-backup', weight: 0.1 },
|
|
126
|
-
],
|
|
127
|
-
{ strategy: new WeightedStrategy() }
|
|
128
|
-
);
|
|
129
|
-
```
|
|
97
|
+
## v4.0 — Semantic Cache
|
|
130
98
|
|
|
131
|
-
|
|
99
|
+
Automatically cache API responses by semantic similarity.
|
|
132
100
|
|
|
133
101
|
```typescript
|
|
134
|
-
|
|
102
|
+
const manager = new ApiKeyManager(['key1', 'key2'], {
|
|
103
|
+
semanticCache: {
|
|
104
|
+
threshold: 0.92,
|
|
105
|
+
getEmbedding: async (text) => await myModel.embed(text)
|
|
106
|
+
}
|
|
107
|
+
});
|
|
135
108
|
|
|
136
|
-
|
|
137
|
-
|
|
109
|
+
// Cache HIT if prompt is semantically similar
|
|
110
|
+
const r1 = await manager.execute(apiFn, { prompt: 'What is the weather?' });
|
|
111
|
+
const r2 = await manager.execute(apiFn, { prompt: 'How is the weather today?' });
|
|
138
112
|
```
|
|
139
113
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
Route requests to specific providers:
|
|
114
|
+
### v4.1 — Streaming Support
|
|
115
|
+
Real-time response handling with the same resilience as `execute()`.
|
|
143
116
|
|
|
144
117
|
```typescript
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
{ key: 'AIza-gemini', weight: 0.5, provider: 'gemini' },
|
|
149
|
-
]);
|
|
150
|
-
|
|
151
|
-
const openaiKey = manager.getKeyByProvider('openai');
|
|
152
|
-
const geminiKey = manager.getKeyByProvider('gemini');
|
|
153
|
-
```
|
|
118
|
+
const stream = await manager.executeStream(async (key) => {
|
|
119
|
+
return await gemini.generateContentStream({ prompt: "..." });
|
|
120
|
+
}, { prompt: "..." });
|
|
154
121
|
|
|
155
|
-
|
|
122
|
+
for await (const chunk of stream) {
|
|
123
|
+
console.log(chunk.text());
|
|
124
|
+
}
|
|
125
|
+
```
|
|
156
126
|
|
|
157
|
-
|
|
127
|
+
---
|
|
158
128
|
|
|
159
|
-
|
|
160
|
-
manager.setHealthCheck(async (key) => {
|
|
161
|
-
const res = await fetch(`https://api.openai.com/v1/models`, {
|
|
162
|
-
headers: { Authorization: `Bearer ${key}` }
|
|
163
|
-
});
|
|
164
|
-
return res.ok;
|
|
165
|
-
});
|
|
166
|
-
|
|
167
|
-
manager.startHealthChecks(60_000); // Check every 60 seconds
|
|
168
|
-
// manager.stopHealthChecks(); // Stop when done
|
|
169
|
-
```
|
|
129
|
+
## execute() Wrapper
|
|
170
130
|
|
|
171
|
-
|
|
131
|
+
Wraps the entire lifecycle into one method:
|
|
172
132
|
|
|
173
133
|
```typescript
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
if (classification.retryable) {
|
|
182
|
-
const delay = manager.calculateBackoff(attempt);
|
|
183
|
-
await sleep(delay);
|
|
184
|
-
}
|
|
185
|
-
}
|
|
134
|
+
const result = await manager.execute(
|
|
135
|
+
async (key, signal) => {
|
|
136
|
+
const res = await fetch(url, { headers: { 'x-api-key': key }, signal });
|
|
137
|
+
return res.json();
|
|
138
|
+
},
|
|
139
|
+
{ maxRetries: 3, timeoutMs: 10000 }
|
|
140
|
+
);
|
|
186
141
|
```
|
|
187
142
|
|
|
188
143
|
## API Reference
|
|
189
144
|
|
|
190
|
-
###
|
|
145
|
+
### Presets
|
|
191
146
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
```
|
|
147
|
+
| Class | Env Vars | Description |
|
|
148
|
+
|-------|----------|-------------|
|
|
149
|
+
| `GeminiManager` | `GOOGLE_GEMINI_API_KEY`, `GEMINI_API_KEY` | Gemini-optimized singleton |
|
|
150
|
+
| `OpenAIManager` | `OPENAI_API_KEY` | OpenAI-optimized singleton |
|
|
151
|
+
| `MultiManager` | Custom | Vault for multiple provider pools |
|
|
152
|
+
|
|
153
|
+
### Persistence
|
|
154
|
+
|
|
155
|
+
| Class | Description |
|
|
156
|
+
|-------|-------------|
|
|
157
|
+
| `FileStorage` | Persists to a JSON file (recommended for servers) |
|
|
158
|
+
| `MemoryStorage` | In-memory only (best for serverless/short-lived) |
|
|
205
159
|
|
|
206
|
-
### Methods
|
|
160
|
+
### Core Methods
|
|
207
161
|
|
|
208
162
|
| Method | Description |
|
|
209
163
|
|--------|-------------|
|
|
210
|
-
| `
|
|
211
|
-
| `
|
|
212
|
-
| `
|
|
213
|
-
| `
|
|
214
|
-
| `
|
|
215
|
-
| `execute(fn, options?)` | Full lifecycle wrapper with retry/timeout |
|
|
216
|
-
| `calculateBackoff(attempt)` | Get backoff delay with jitter |
|
|
217
|
-
| `getStats()` | Get pool health statistics |
|
|
218
|
-
| `getKeyCount()` | Count of non-DEAD keys |
|
|
219
|
-
| `setHealthCheck(fn)` | Set health check function |
|
|
220
|
-
| `startHealthChecks(ms)` | Start periodic health checks |
|
|
221
|
-
| `stopHealthChecks()` | Stop health checks |
|
|
222
|
-
|
|
223
|
-
### Events
|
|
224
|
-
|
|
225
|
-
| Event | Payload | Trigger |
|
|
226
|
-
|-------|---------|---------|
|
|
227
|
-
| `keyDead` | `key: string` | Key marked as permanently dead |
|
|
228
|
-
| `circuitOpen` | `key: string` | Key circuit opened (cooldown) |
|
|
229
|
-
| `circuitHalfOpen` | `key: string` | Key entering test phase |
|
|
230
|
-
| `keyRecovered` | `key: string` | Key recovered from failure |
|
|
231
|
-
| `fallback` | `reason: string` | Fallback function invoked |
|
|
232
|
-
| `allKeysExhausted` | — | All keys dead, no fallback |
|
|
233
|
-
| `retry` | `key, attempt, delayMs` | Retry attempt starting |
|
|
234
|
-
| `executeSuccess` | `key, durationMs` | execute() completed successfully |
|
|
235
|
-
| `executeFailed` | `key, error` | execute() attempt failed |
|
|
236
|
-
| `bulkheadRejected` | — | Concurrency limit reached |
|
|
237
|
-
| `healthCheckPassed` | `key: string` | Health check succeeded |
|
|
238
|
-
| `healthCheckFailed` | `key, error` | Health check failed |
|
|
239
|
-
|
|
240
|
-
### Custom Errors
|
|
241
|
-
|
|
242
|
-
| Error | When |
|
|
243
|
-
|-------|------|
|
|
244
|
-
| `TimeoutError` | Request exceeded `timeoutMs` |
|
|
245
|
-
| `BulkheadRejectionError` | Concurrency limit exceeded |
|
|
246
|
-
| `AllKeysExhaustedError` | All keys dead, no fallback |
|
|
247
|
-
|
|
248
|
-
### Strategies
|
|
249
|
-
|
|
250
|
-
| Strategy | Algorithm | Best For |
|
|
251
|
-
|----------|-----------|----------|
|
|
252
|
-
| `StandardStrategy` | Least Failures → LRU | General use |
|
|
253
|
-
| `WeightedStrategy` | Probabilistic by weight | Cost optimization |
|
|
254
|
-
| `LatencyStrategy` | Lowest avg latency | Performance |
|
|
164
|
+
| `execute(fn, opts)` | Standard wrapper |
|
|
165
|
+
| `executeStream(fn, opts)` | Streaming wrapper |
|
|
166
|
+
| `getStats()` | Get pool health |
|
|
167
|
+
| `getKey()` | Manual key rotation |
|
|
168
|
+
| `markFailed(key, err)` | Manual failure reporting |
|
|
255
169
|
|
|
256
170
|
## License
|
|
257
171
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Universal ApiKeyManager
|
|
2
|
+
* Universal ApiKeyManager v5.0 — Ecosystem Edition
|
|
3
3
|
* Implements: Rotation, Circuit Breaker, Persistence, Exponential Backoff, Strategies,
|
|
4
4
|
* Event Emitter, Fallback, execute(), Timeout, Auto-Retry, Provider Tags,
|
|
5
5
|
* Health Checks, Bulkhead/Concurrency
|
|
6
|
+
* NEW in v5.0: Provider Presets (GeminiManager, OpenAIManager, MultiManager),
|
|
7
|
+
* Built-in Persistence (FileStorage, MemoryStorage)
|
|
6
8
|
* Gemini-Specific: finishReason handling, Safety blocks, RECITATION detection
|
|
7
9
|
*/
|
|
8
10
|
import { EventEmitter } from 'events';
|
|
11
|
+
export { FileStorage } from './persistence/file';
|
|
12
|
+
export type { FileStorageOptions } from './persistence/file';
|
|
13
|
+
export { MemoryStorage } from './persistence/memory';
|
|
9
14
|
export interface KeyState {
|
|
10
15
|
key: string;
|
|
11
16
|
failCount: number;
|
|
@@ -103,7 +108,7 @@ export declare class WeightedStrategy implements LoadBalancingStrategy {
|
|
|
103
108
|
next(candidates: KeyState[]): KeyState | null;
|
|
104
109
|
}
|
|
105
110
|
/**
|
|
106
|
-
* Latency Strategy: Pick lowest average latency
|
|
111
|
+
* Latency Strategy: Pick lowest average latency with LRU tie-break
|
|
107
112
|
*/
|
|
108
113
|
export declare class LatencyStrategy implements LoadBalancingStrategy {
|
|
109
114
|
next(candidates: KeyState[]): KeyState | null;
|
|
@@ -185,6 +190,21 @@ export declare class ApiKeyManager extends EventEmitter {
|
|
|
185
190
|
execute<T>(fn: (key: string, signal?: AbortSignal) => Promise<T>, options?: ExecuteOptions & {
|
|
186
191
|
prompt?: string;
|
|
187
192
|
}): Promise<T>;
|
|
193
|
+
/**
|
|
194
|
+
* Executes a streaming function (AsyncGenerator) with retry logic and semantic caching.
|
|
195
|
+
*
|
|
196
|
+
* @example
|
|
197
|
+
* const stream = await manager.executeStream(async (key) => {
|
|
198
|
+
* return await gemini.generateContentStream({ prompt: "..." });
|
|
199
|
+
* }, { prompt: "..." });
|
|
200
|
+
*
|
|
201
|
+
* for await (const chunk of stream) {
|
|
202
|
+
* console.log(chunk.text());
|
|
203
|
+
* }
|
|
204
|
+
*/
|
|
205
|
+
executeStream<T>(fn: (key: string, signal?: AbortSignal) => AsyncGenerator<T, any, unknown>, options?: ExecuteOptions & {
|
|
206
|
+
prompt?: string;
|
|
207
|
+
}): AsyncGenerator<T, any, unknown>;
|
|
188
208
|
private _executeWithRetry;
|
|
189
209
|
private _executeWithTimeout;
|
|
190
210
|
private _sleep;
|
package/dist/index.js
CHANGED
|
@@ -1,14 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* Universal ApiKeyManager
|
|
3
|
+
* Universal ApiKeyManager v5.0 — Ecosystem Edition
|
|
4
4
|
* Implements: Rotation, Circuit Breaker, Persistence, Exponential Backoff, Strategies,
|
|
5
5
|
* Event Emitter, Fallback, execute(), Timeout, Auto-Retry, Provider Tags,
|
|
6
6
|
* Health Checks, Bulkhead/Concurrency
|
|
7
|
+
* NEW in v5.0: Provider Presets (GeminiManager, OpenAIManager, MultiManager),
|
|
8
|
+
* Built-in Persistence (FileStorage, MemoryStorage)
|
|
7
9
|
* Gemini-Specific: finishReason handling, Safety blocks, RECITATION detection
|
|
8
10
|
*/
|
|
9
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.ApiKeyManager = exports.SemanticCache = exports.LatencyStrategy = exports.WeightedStrategy = exports.StandardStrategy = exports.AllKeysExhaustedError = exports.BulkheadRejectionError = exports.TimeoutError = void 0;
|
|
12
|
+
exports.ApiKeyManager = exports.SemanticCache = exports.LatencyStrategy = exports.WeightedStrategy = exports.StandardStrategy = exports.AllKeysExhaustedError = exports.BulkheadRejectionError = exports.TimeoutError = exports.MemoryStorage = exports.FileStorage = void 0;
|
|
11
13
|
const events_1 = require("events");
|
|
14
|
+
// ─── Re-exports: Persistence ─────────────────────────────────────────────────
|
|
15
|
+
// Persistence adapters can be imported from root or via subpath
|
|
16
|
+
var file_1 = require("./persistence/file");
|
|
17
|
+
Object.defineProperty(exports, "FileStorage", { enumerable: true, get: function () { return file_1.FileStorage; } });
|
|
18
|
+
var memory_1 = require("./persistence/memory");
|
|
19
|
+
Object.defineProperty(exports, "MemoryStorage", { enumerable: true, get: function () { return memory_1.MemoryStorage; } });
|
|
12
20
|
// ─── Config ──────────────────────────────────────────────────────────────────
|
|
13
21
|
const CONFIG = {
|
|
14
22
|
MAX_CONSECUTIVE_FAILURES: 5,
|
|
@@ -83,13 +91,17 @@ class WeightedStrategy {
|
|
|
83
91
|
}
|
|
84
92
|
exports.WeightedStrategy = WeightedStrategy;
|
|
85
93
|
/**
|
|
86
|
-
* Latency Strategy: Pick lowest average latency
|
|
94
|
+
* Latency Strategy: Pick lowest average latency with LRU tie-break
|
|
87
95
|
*/
|
|
88
96
|
class LatencyStrategy {
|
|
89
97
|
next(candidates) {
|
|
90
98
|
if (candidates.length === 0)
|
|
91
99
|
return null;
|
|
92
|
-
candidates.sort((a, b) =>
|
|
100
|
+
candidates.sort((a, b) => {
|
|
101
|
+
if (a.averageLatency !== b.averageLatency)
|
|
102
|
+
return a.averageLatency - b.averageLatency;
|
|
103
|
+
return a.lastUsed - b.lastUsed; // LRU tie-break
|
|
104
|
+
});
|
|
93
105
|
return candidates[0];
|
|
94
106
|
}
|
|
95
107
|
}
|
|
@@ -501,6 +513,127 @@ class ApiKeyManager extends events_1.EventEmitter {
|
|
|
501
513
|
this.activeCalls--;
|
|
502
514
|
}
|
|
503
515
|
}
|
|
516
|
+
/**
|
|
517
|
+
* Executes a streaming function (AsyncGenerator) with retry logic and semantic caching.
|
|
518
|
+
*
|
|
519
|
+
* @example
|
|
520
|
+
* const stream = await manager.executeStream(async (key) => {
|
|
521
|
+
* return await gemini.generateContentStream({ prompt: "..." });
|
|
522
|
+
* }, { prompt: "..." });
|
|
523
|
+
*
|
|
524
|
+
* for await (const chunk of stream) {
|
|
525
|
+
* console.log(chunk.text());
|
|
526
|
+
* }
|
|
527
|
+
*/
|
|
528
|
+
async *executeStream(fn, options) {
|
|
529
|
+
const maxRetries = options?.maxRetries ?? 0;
|
|
530
|
+
const timeoutMs = options?.timeoutMs;
|
|
531
|
+
const finishReason = options?.finishReason;
|
|
532
|
+
const prompt = options?.prompt;
|
|
533
|
+
const provider = options?.provider;
|
|
534
|
+
// 1. Semantic Cache Check
|
|
535
|
+
let currentPromptVector = null;
|
|
536
|
+
if (this.semanticCache && this.getEmbeddingFn && prompt && !this._isResolvingEmbedding) {
|
|
537
|
+
try {
|
|
538
|
+
this._isResolvingEmbedding = true;
|
|
539
|
+
currentPromptVector = await this.getEmbeddingFn(prompt);
|
|
540
|
+
const cachedResponse = this.semanticCache.get(currentPromptVector);
|
|
541
|
+
if (cachedResponse !== null) {
|
|
542
|
+
console.log(`[Semantic Cache HIT] Streaming cached response for prompt: "${prompt.slice(0, 30)}..."`);
|
|
543
|
+
this.emit('executeSuccess', 'CACHE_HIT_STREAM', 0);
|
|
544
|
+
// Replay full response as a single chunk (or iterate if it's an array)
|
|
545
|
+
if (Array.isArray(cachedResponse)) {
|
|
546
|
+
for (const chunk of cachedResponse)
|
|
547
|
+
yield chunk;
|
|
548
|
+
}
|
|
549
|
+
else {
|
|
550
|
+
yield cachedResponse;
|
|
551
|
+
}
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
catch (e) {
|
|
556
|
+
console.warn('[Semantic Cache Check Failed] Proceeding to live stream', e);
|
|
557
|
+
}
|
|
558
|
+
finally {
|
|
559
|
+
this._isResolvingEmbedding = false;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
// 2. Bulkhead check
|
|
563
|
+
if (this.activeCalls >= this.maxConcurrency) {
|
|
564
|
+
this.emit('bulkheadRejected');
|
|
565
|
+
throw new BulkheadRejectionError();
|
|
566
|
+
}
|
|
567
|
+
this.activeCalls++;
|
|
568
|
+
const accumulatedChunks = [];
|
|
569
|
+
let lastError;
|
|
570
|
+
try {
|
|
571
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
572
|
+
const key = provider ? this.getKeyByProvider(provider) : this.getKey();
|
|
573
|
+
if (!key) {
|
|
574
|
+
if (this.fallbackFn) {
|
|
575
|
+
this.emit('fallback', 'all keys exhausted (stream)');
|
|
576
|
+
yield this.fallbackFn();
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
579
|
+
throw new AllKeysExhaustedError();
|
|
580
|
+
}
|
|
581
|
+
let iterator;
|
|
582
|
+
try {
|
|
583
|
+
// Start the generator
|
|
584
|
+
iterator = fn(key);
|
|
585
|
+
// PRIME THE ITERATOR:
|
|
586
|
+
// Try to get the first chunk. This forces the generator body to
|
|
587
|
+
// execute until the first 'yield' or until it throws.
|
|
588
|
+
const firstResult = await iterator.next();
|
|
589
|
+
// If we got here, the INITIAL connection/logic succeeded!
|
|
590
|
+
if (firstResult.done)
|
|
591
|
+
return;
|
|
592
|
+
// Yield first chunk
|
|
593
|
+
yield firstResult.value;
|
|
594
|
+
if (this.semanticCache && prompt)
|
|
595
|
+
accumulatedChunks.push(firstResult.value);
|
|
596
|
+
// Yield rest of chunks
|
|
597
|
+
for await (const chunk of iterator) {
|
|
598
|
+
yield chunk;
|
|
599
|
+
if (this.semanticCache && prompt)
|
|
600
|
+
accumulatedChunks.push(chunk);
|
|
601
|
+
}
|
|
602
|
+
// Success! Store in cache
|
|
603
|
+
if (this.semanticCache && prompt && currentPromptVector && accumulatedChunks.length > 0) {
|
|
604
|
+
this.semanticCache.set(prompt, currentPromptVector, accumulatedChunks);
|
|
605
|
+
}
|
|
606
|
+
return; // Full success, exit retry loop
|
|
607
|
+
}
|
|
608
|
+
catch (error) {
|
|
609
|
+
lastError = error;
|
|
610
|
+
const classification = this.classifyError(error, finishReason);
|
|
611
|
+
this.markFailed(key, classification);
|
|
612
|
+
this.emit('executeFailed', key, error);
|
|
613
|
+
// Note: If we already yielded the FIRST chunk, we CANNOT retry the connection
|
|
614
|
+
// because the user has already received data. Mid-stream failures propagate.
|
|
615
|
+
if (accumulatedChunks.length > 0) {
|
|
616
|
+
throw error;
|
|
617
|
+
}
|
|
618
|
+
if (!classification.retryable || attempt >= maxRetries) {
|
|
619
|
+
if (this.fallbackFn && attempt >= maxRetries) {
|
|
620
|
+
this.emit('fallback', 'max retries exceeded (stream)');
|
|
621
|
+
yield this.fallbackFn();
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
throw error;
|
|
625
|
+
}
|
|
626
|
+
const delay = this.calculateBackoff(attempt);
|
|
627
|
+
this.emit('retry', key, attempt + 1, delay);
|
|
628
|
+
await this._sleep(delay);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
finally {
|
|
633
|
+
this.activeCalls--;
|
|
634
|
+
}
|
|
635
|
+
throw lastError;
|
|
636
|
+
}
|
|
504
637
|
async _executeWithRetry(fn, maxRetries, timeoutMs, finishReason, provider) {
|
|
505
638
|
let lastError;
|
|
506
639
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
@@ -568,7 +701,11 @@ class ApiKeyManager extends events_1.EventEmitter {
|
|
|
568
701
|
}
|
|
569
702
|
}
|
|
570
703
|
_sleep(ms) {
|
|
571
|
-
return new Promise(resolve =>
|
|
704
|
+
return new Promise(resolve => {
|
|
705
|
+
const timer = setTimeout(resolve, ms);
|
|
706
|
+
if (timer.unref)
|
|
707
|
+
timer.unref();
|
|
708
|
+
});
|
|
572
709
|
}
|
|
573
710
|
// ─── Health Checks ───────────────────────────────────────────────────────
|
|
574
711
|
/**
|
|
@@ -584,6 +721,8 @@ class ApiKeyManager extends events_1.EventEmitter {
|
|
|
584
721
|
startHealthChecks(intervalMs = 60_000) {
|
|
585
722
|
this.stopHealthChecks(); // Clear any existing interval
|
|
586
723
|
this.healthCheckInterval = setInterval(() => this._runHealthChecks(), intervalMs);
|
|
724
|
+
if (this.healthCheckInterval.unref)
|
|
725
|
+
this.healthCheckInterval.unref();
|
|
587
726
|
}
|
|
588
727
|
/**
|
|
589
728
|
* Stop periodic health checks
|