@probeo/anymodel 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -10
- package/dist/batch/index.d.ts +4 -0
- package/dist/batch/index.d.ts.map +1 -0
- package/dist/batch/index.js +3 -0
- package/dist/batch/index.js.map +1 -0
- package/dist/batch/manager.d.ts +72 -0
- package/dist/batch/manager.d.ts.map +1 -0
- package/dist/batch/manager.js +328 -0
- package/dist/batch/manager.js.map +1 -0
- package/dist/batch/store.d.ts +54 -0
- package/dist/batch/store.d.ts.map +1 -0
- package/dist/batch/store.js +109 -0
- package/dist/batch/store.js.map +1 -0
- package/dist/cli.cjs +112 -19
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +28 -1993
- package/dist/cli.js.map +1 -1
- package/dist/client.d.ts +42 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +181 -0
- package/dist/client.js.map +1 -0
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +120 -0
- package/dist/config.js.map +1 -0
- package/dist/index.cjs +112 -19
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +10 -0
- package/dist/index.d.ts +14 -538
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -1972
- package/dist/index.js.map +1 -1
- package/dist/providers/adapter.d.ts +33 -0
- package/dist/providers/adapter.d.ts.map +1 -0
- package/dist/providers/adapter.js +2 -0
- package/dist/providers/adapter.js.map +1 -0
- package/dist/providers/anthropic-batch.d.ts +3 -0
- package/dist/providers/anthropic-batch.d.ts.map +1 -0
- package/dist/providers/anthropic-batch.js +228 -0
- package/dist/providers/anthropic-batch.js.map +1 -0
- package/dist/providers/anthropic.d.ts +3 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +358 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/custom.d.ts +8 -0
- package/dist/providers/custom.d.ts.map +1 -0
- package/dist/providers/custom.js +61 -0
- package/dist/providers/custom.js.map +1 -0
- package/dist/providers/google.d.ts +3 -0
- package/dist/providers/google.d.ts.map +1 -0
- package/dist/providers/google.js +331 -0
- package/dist/providers/google.js.map +1 -0
- package/dist/providers/index.d.ts +6 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +5 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/openai-batch.d.ts +3 -0
- package/dist/providers/openai-batch.d.ts.map +1 -0
- package/dist/providers/openai-batch.js +208 -0
- package/dist/providers/openai-batch.js.map +1 -0
- package/dist/providers/openai.d.ts +3 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +221 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/registry.d.ts +10 -0
- package/dist/providers/registry.d.ts.map +1 -0
- package/dist/providers/registry.js +27 -0
- package/dist/providers/registry.js.map +1 -0
- package/dist/router.d.ts +29 -0
- package/dist/router.d.ts.map +1 -0
- package/dist/router.js +212 -0
- package/dist/router.js.map +1 -0
- package/dist/server.d.ts +10 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +149 -0
- package/dist/server.js.map +1 -0
- package/dist/types.d.ts +283 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +21 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/fs-io.d.ts +40 -0
- package/dist/utils/fs-io.d.ts.map +1 -0
- package/dist/utils/fs-io.js +203 -0
- package/dist/utils/fs-io.js.map +1 -0
- package/dist/utils/generation-stats.d.ts +25 -0
- package/dist/utils/generation-stats.d.ts.map +1 -0
- package/dist/utils/generation-stats.js +46 -0
- package/dist/utils/generation-stats.js.map +1 -0
- package/dist/utils/id.d.ts +2 -0
- package/dist/utils/id.d.ts.map +1 -0
- package/dist/utils/id.js +6 -0
- package/dist/utils/id.js.map +1 -0
- package/dist/utils/model-parser.d.ts +6 -0
- package/dist/utils/model-parser.d.ts.map +1 -0
- package/dist/utils/model-parser.js +21 -0
- package/dist/utils/model-parser.js.map +1 -0
- package/dist/utils/rate-limiter.d.ts +36 -0
- package/dist/utils/rate-limiter.d.ts.map +1 -0
- package/dist/utils/rate-limiter.js +80 -0
- package/dist/utils/rate-limiter.js.map +1 -0
- package/dist/utils/retry.d.ts +7 -0
- package/dist/utils/retry.d.ts.map +1 -0
- package/dist/utils/retry.js +54 -0
- package/dist/utils/retry.js.map +1 -0
- package/dist/utils/transforms.d.ts +7 -0
- package/dist/utils/transforms.d.ts.map +1 -0
- package/dist/utils/transforms.js +66 -0
- package/dist/utils/transforms.js.map +1 -0
- package/dist/utils/validate.d.ts +3 -0
- package/dist/utils/validate.d.ts.map +1 -0
- package/dist/utils/validate.js +31 -0
- package/dist/utils/validate.js.map +1 -0
- package/package.json +9 -2
package/README.md
CHANGED
|
@@ -49,18 +49,39 @@ for await (const chunk of stream) {
|
|
|
49
49
|
}
|
|
50
50
|
```
|
|
51
51
|
|
|
52
|
+
## Supported Providers
|
|
53
|
+
|
|
54
|
+
Set the env var and go. Models are auto-discovered from each provider's API.
|
|
55
|
+
|
|
56
|
+
| Provider | Env Var | Example Model |
|
|
57
|
+
|----------|---------|---------------|
|
|
58
|
+
| OpenAI | `OPENAI_API_KEY` | `openai/gpt-4o` |
|
|
59
|
+
| Anthropic | `ANTHROPIC_API_KEY` | `anthropic/claude-sonnet-4-6` |
|
|
60
|
+
| Google | `GOOGLE_API_KEY` | `google/gemini-2.5-pro` |
|
|
61
|
+
| Mistral | `MISTRAL_API_KEY` | `mistral/mistral-large-latest` |
|
|
62
|
+
| Groq | `GROQ_API_KEY` | `groq/llama-3.3-70b-versatile` |
|
|
63
|
+
| DeepSeek | `DEEPSEEK_API_KEY` | `deepseek/deepseek-chat` |
|
|
64
|
+
| xAI | `XAI_API_KEY` | `xai/grok-3` |
|
|
65
|
+
| Together | `TOGETHER_API_KEY` | `together/meta-llama/Llama-3.3-70B-Instruct-Turbo` |
|
|
66
|
+
| Fireworks | `FIREWORKS_API_KEY` | `fireworks/accounts/fireworks/models/llama-v3p3-70b-instruct` |
|
|
67
|
+
| Perplexity | `PERPLEXITY_API_KEY` | `perplexity/sonar-pro` |
|
|
68
|
+
| Ollama | `OLLAMA_BASE_URL` | `ollama/llama3.3` |
|
|
69
|
+
|
|
70
|
+
Ollama runs locally with no API key — just set `OLLAMA_BASE_URL` (defaults to `http://localhost:11434/v1`).
|
|
71
|
+
|
|
52
72
|
## Model Naming
|
|
53
73
|
|
|
54
74
|
Models use `provider/model` format:
|
|
55
75
|
|
|
56
76
|
```
|
|
57
77
|
anthropic/claude-sonnet-4-6
|
|
58
|
-
anthropic/claude-opus-4-6
|
|
59
|
-
anthropic/claude-haiku-4-5
|
|
60
78
|
openai/gpt-4o
|
|
61
|
-
openai/o3
|
|
62
79
|
google/gemini-2.5-pro
|
|
63
|
-
|
|
80
|
+
mistral/mistral-large-latest
|
|
81
|
+
groq/llama-3.3-70b-versatile
|
|
82
|
+
deepseek/deepseek-chat
|
|
83
|
+
xai/grok-3
|
|
84
|
+
ollama/llama3.3
|
|
64
85
|
```
|
|
65
86
|
|
|
66
87
|
## Fallback Routing
|
|
@@ -126,11 +147,13 @@ const response = await client.chat.completions.create({
|
|
|
126
147
|
|
|
127
148
|
## Batch Processing
|
|
128
149
|
|
|
129
|
-
Process many requests
|
|
150
|
+
Process many requests with native provider batch APIs or concurrent fallback. OpenAI and Anthropic batches are processed server-side — OpenAI at 50% cost, Anthropic with async processing for up to 10K requests. Other providers fall back to concurrent execution automatically.
|
|
151
|
+
|
|
152
|
+
### Submit and wait
|
|
130
153
|
|
|
131
154
|
```typescript
|
|
132
155
|
const results = await client.batches.createAndPoll({
|
|
133
|
-
model: "
|
|
156
|
+
model: "openai/gpt-4o-mini",
|
|
134
157
|
requests: [
|
|
135
158
|
{ custom_id: "req-1", messages: [{ role: "user", content: "Summarize AI" }] },
|
|
136
159
|
{ custom_id: "req-2", messages: [{ role: "user", content: "Summarize ML" }] },
|
|
@@ -143,14 +166,71 @@ for (const result of results.results) {
|
|
|
143
166
|
}
|
|
144
167
|
```
|
|
145
168
|
|
|
146
|
-
|
|
169
|
+
### Submit now, check later
|
|
170
|
+
|
|
171
|
+
Submit a batch and get back an ID immediately — no need to keep the process running for native batches (OpenAI, Anthropic):
|
|
147
172
|
|
|
148
173
|
```typescript
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
174
|
+
// Submit and get the batch ID
|
|
175
|
+
const batch = await client.batches.create({
|
|
176
|
+
model: "anthropic/claude-haiku-4-5",
|
|
177
|
+
requests: [
|
|
178
|
+
{ custom_id: "req-1", messages: [{ role: "user", content: "Summarize AI" }] },
|
|
179
|
+
{ custom_id: "req-2", messages: [{ role: "user", content: "Summarize ML" }] },
|
|
180
|
+
],
|
|
181
|
+
});
|
|
182
|
+
console.log(batch.id); // "batch-abc123"
|
|
183
|
+
console.log(batch.batch_mode); // "native" or "concurrent"
|
|
184
|
+
|
|
185
|
+
// Check status any time — even after a process restart
|
|
186
|
+
const status = client.batches.get("batch-abc123");
|
|
187
|
+
console.log(status.status); // "pending", "processing", "completed", "failed"
|
|
188
|
+
|
|
189
|
+
// Wait for results when you're ready (reconnects to provider API)
|
|
190
|
+
const results = await client.batches.poll("batch-abc123");
|
|
191
|
+
|
|
192
|
+
// Or get results directly if already completed
|
|
193
|
+
const results = client.batches.results("batch-abc123");
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### List and cancel
|
|
197
|
+
|
|
198
|
+
```typescript
|
|
199
|
+
// List all batches on disk
|
|
200
|
+
const all = client.batches.list();
|
|
201
|
+
for (const b of all) {
|
|
202
|
+
console.log(b.id, b.batch_mode, b.status, b.provider_name);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Cancel a running batch (also cancels at the provider for native batches)
|
|
206
|
+
await client.batches.cancel("batch-abc123");
|
|
152
207
|
```
|
|
153
208
|
|
|
209
|
+
### Batch configuration
|
|
210
|
+
|
|
211
|
+
```typescript
|
|
212
|
+
const client = new AnyModel({
|
|
213
|
+
batch: {
|
|
214
|
+
pollInterval: 10000, // default poll interval in ms (default: 5000)
|
|
215
|
+
concurrencyFallback: 10, // concurrent request limit for non-native providers (default: 5)
|
|
216
|
+
},
|
|
217
|
+
io: {
|
|
218
|
+
readConcurrency: 30, // concurrent file reads (default: 20)
|
|
219
|
+
writeConcurrency: 15, // concurrent file writes (default: 10)
|
|
220
|
+
},
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
// Override poll interval per call
|
|
224
|
+
const results = await client.batches.createAndPoll(request, {
|
|
225
|
+
interval: 3000, // poll every 3s for this batch
|
|
226
|
+
onProgress: (batch) => {
|
|
227
|
+
console.log(`${batch.completed}/${batch.total} done`);
|
|
228
|
+
},
|
|
229
|
+
});
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Batches are persisted to `./.anymodel/batches/` in the current working directory and survive process restarts.
|
|
233
|
+
|
|
154
234
|
## Models Endpoint
|
|
155
235
|
|
|
156
236
|
```typescript
|
|
@@ -210,6 +290,14 @@ Create `anymodel.config.json` in your project root:
|
|
|
210
290
|
"defaults": {
|
|
211
291
|
"temperature": 0.7,
|
|
212
292
|
"max_tokens": 4096
|
|
293
|
+
},
|
|
294
|
+
"batch": {
|
|
295
|
+
"pollInterval": 5000,
|
|
296
|
+
"concurrencyFallback": 5
|
|
297
|
+
},
|
|
298
|
+
"io": {
|
|
299
|
+
"readConcurrency": 20,
|
|
300
|
+
"writeConcurrency": 10
|
|
213
301
|
}
|
|
214
302
|
}
|
|
215
303
|
```
|
|
@@ -337,6 +425,25 @@ npx tsx examples/basic.ts batch
|
|
|
337
425
|
- **Retries**: Automatic retry with exponential backoff on 429/502/503 errors (configurable via `defaults.retries`)
|
|
338
426
|
- **Rate limit tracking**: Per-provider rate limit state, automatically skips rate-limited providers during fallback routing
|
|
339
427
|
- **Parameter stripping**: Unsupported parameters are automatically removed before forwarding to providers
|
|
428
|
+
- **High-volume IO**: All batch file operations use concurrency-limited async queues with atomic durable writes (temp file + fsync + rename) to prevent corruption on crash. Defaults: 20 concurrent reads, 10 concurrent writes — configurable via `io.readConcurrency` and `io.writeConcurrency`
|
|
429
|
+
|
|
430
|
+
## Roadmap
|
|
431
|
+
|
|
432
|
+
- [ ] **A/B testing** — split routing (% traffic to each model) and compare mode (same request to multiple models, return all responses with stats)
|
|
433
|
+
- [ ] **Cost tracking** — per-request and aggregate cost calculation from provider pricing
|
|
434
|
+
- [ ] **Caching** — response caching with configurable TTL for identical requests
|
|
435
|
+
- [x] **Native batch APIs** — OpenAI Batch API (JSONL upload, 50% cost) and Anthropic Message Batches (10K requests, async). Auto-detects provider and routes to native API, falls back to concurrent for other providers
|
|
436
|
+
- [ ] **Webhooks** — notify on batch completion
|
|
437
|
+
- [ ] **Prompt logging** — optional request/response logging for debugging and evaluation
|
|
438
|
+
|
|
439
|
+
## Roadmap
|
|
440
|
+
|
|
441
|
+
- [ ] **A/B testing** — split routing (% traffic to each model) and compare mode (same request to multiple models, return all responses with stats)
|
|
442
|
+
- [ ] **Cost tracking** — per-request and aggregate cost calculation from provider pricing
|
|
443
|
+
- [ ] **Caching** — response caching with configurable TTL for identical requests
|
|
444
|
+
- [ ] **Native batch APIs** — use Anthropic Message Batches and OpenAI Batch API instead of concurrent requests
|
|
445
|
+
- [ ] **Webhooks** — notify on batch completion
|
|
446
|
+
- [ ] **Prompt logging** — optional request/response logging for debugging and evaluation
|
|
340
447
|
|
|
341
448
|
## License
|
|
342
449
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/batch/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/batch/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { BatchCreateRequest, BatchObject, BatchResults } from '../types.js';
|
|
2
|
+
import type { Router } from '../router.js';
|
|
3
|
+
import type { BatchAdapter } from '../providers/adapter.js';
|
|
4
|
+
export interface BatchPollOptions {
|
|
5
|
+
/** Poll interval in ms. Default: 5000 */
|
|
6
|
+
interval?: number;
|
|
7
|
+
/** Timeout in ms. 0 = indefinite. Default: 0 */
|
|
8
|
+
timeout?: number;
|
|
9
|
+
/** Progress callback */
|
|
10
|
+
onProgress?: (batch: BatchObject) => void;
|
|
11
|
+
}
|
|
12
|
+
export declare class BatchManager {
|
|
13
|
+
private store;
|
|
14
|
+
private router;
|
|
15
|
+
private concurrencyLimit;
|
|
16
|
+
private defaultPollInterval;
|
|
17
|
+
private batchAdapters;
|
|
18
|
+
constructor(router: Router, options?: {
|
|
19
|
+
dir?: string;
|
|
20
|
+
concurrency?: number;
|
|
21
|
+
pollInterval?: number;
|
|
22
|
+
});
|
|
23
|
+
/**
|
|
24
|
+
* Register a native batch adapter for a provider.
|
|
25
|
+
*/
|
|
26
|
+
registerBatchAdapter(providerName: string, adapter: BatchAdapter): void;
|
|
27
|
+
/**
|
|
28
|
+
* Check if a provider has native batch support.
|
|
29
|
+
*/
|
|
30
|
+
private getNativeBatchAdapter;
|
|
31
|
+
/**
|
|
32
|
+
* Create a batch and return immediately (no polling).
|
|
33
|
+
*/
|
|
34
|
+
create(request: BatchCreateRequest): Promise<BatchObject>;
|
|
35
|
+
/**
|
|
36
|
+
* Create a batch and poll until completion.
|
|
37
|
+
*/
|
|
38
|
+
createAndPoll(request: BatchCreateRequest, options?: BatchPollOptions): Promise<BatchResults>;
|
|
39
|
+
/**
|
|
40
|
+
* Poll an existing batch until completion.
|
|
41
|
+
*/
|
|
42
|
+
poll(id: string, options?: BatchPollOptions): Promise<BatchResults>;
|
|
43
|
+
/**
|
|
44
|
+
* Get the current status of a batch.
|
|
45
|
+
*/
|
|
46
|
+
get(id: string): Promise<BatchObject | null>;
|
|
47
|
+
/**
|
|
48
|
+
* Get results for a completed batch.
|
|
49
|
+
*/
|
|
50
|
+
getResults(id: string): Promise<BatchResults>;
|
|
51
|
+
/**
|
|
52
|
+
* List all batches.
|
|
53
|
+
*/
|
|
54
|
+
list(): Promise<BatchObject[]>;
|
|
55
|
+
/**
|
|
56
|
+
* Cancel a batch.
|
|
57
|
+
*/
|
|
58
|
+
cancel(id: string): Promise<BatchObject>;
|
|
59
|
+
/**
|
|
60
|
+
* Process batch via native provider batch API.
|
|
61
|
+
*/
|
|
62
|
+
private processNativeBatch;
|
|
63
|
+
/**
|
|
64
|
+
* Sync native batch status from provider.
|
|
65
|
+
*/
|
|
66
|
+
private syncNativeBatchStatus;
|
|
67
|
+
/**
|
|
68
|
+
* Process batch requests concurrently (fallback path).
|
|
69
|
+
*/
|
|
70
|
+
private processConcurrentBatch;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../src/batch/manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAClB,WAAW,EACX,YAAY,EAIb,MAAM,aAAa,CAAC;AAIrB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;CAC3C;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,aAAa,CAAmC;gBAE5C,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;IAOnG;;OAEG;IACH,oBAAoB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,IAAI;IAIvE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAM7B;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;IAkC/D;;OAEG;IACG,aAAa,CACjB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,YAAY,CAAC;IAKxB;;OAEG;IACG,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC;IAkC7E;;OAEG;IACG,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAIlD;;OAEG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA6BnD;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAUpC;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IA4B9C;;OAEG;YACW,kBAAkB;IAmChC;;OAEG;YACW,qBAAqB;IA6CnC;;OAEG;YACW,sBAAsB;CAiFrC"}
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
import { AnyModelError } from '../types.js';
|
|
2
|
+
import { generateId } from '../utils/id.js';
|
|
3
|
+
import { BatchStore } from './store.js';
|
|
4
|
+
export class BatchManager {
|
|
5
|
+
store;
|
|
6
|
+
router;
|
|
7
|
+
concurrencyLimit;
|
|
8
|
+
defaultPollInterval;
|
|
9
|
+
batchAdapters = new Map();
|
|
10
|
+
constructor(router, options) {
|
|
11
|
+
this.store = new BatchStore(options?.dir);
|
|
12
|
+
this.router = router;
|
|
13
|
+
this.concurrencyLimit = options?.concurrency ?? 5;
|
|
14
|
+
this.defaultPollInterval = options?.pollInterval ?? 5000;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Register a native batch adapter for a provider.
|
|
18
|
+
*/
|
|
19
|
+
registerBatchAdapter(providerName, adapter) {
|
|
20
|
+
this.batchAdapters.set(providerName, adapter);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Check if a provider has native batch support.
|
|
24
|
+
*/
|
|
25
|
+
getNativeBatchAdapter(model) {
|
|
26
|
+
const providerName = model.split('/')[0];
|
|
27
|
+
const adapter = this.batchAdapters.get(providerName);
|
|
28
|
+
return adapter ? { adapter, providerName } : null;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Create a batch and return immediately (no polling).
|
|
32
|
+
*/
|
|
33
|
+
async create(request) {
|
|
34
|
+
const id = generateId('batch');
|
|
35
|
+
const now = new Date().toISOString();
|
|
36
|
+
const providerName = request.model.split('/')[0] || 'unknown';
|
|
37
|
+
const native = this.getNativeBatchAdapter(request.model);
|
|
38
|
+
const batchMode = native ? 'native' : 'concurrent';
|
|
39
|
+
const batch = {
|
|
40
|
+
id,
|
|
41
|
+
object: 'batch',
|
|
42
|
+
status: 'pending',
|
|
43
|
+
model: request.model,
|
|
44
|
+
provider_name: providerName,
|
|
45
|
+
batch_mode: batchMode,
|
|
46
|
+
total: request.requests.length,
|
|
47
|
+
completed: 0,
|
|
48
|
+
failed: 0,
|
|
49
|
+
created_at: now,
|
|
50
|
+
completed_at: null,
|
|
51
|
+
expires_at: null,
|
|
52
|
+
};
|
|
53
|
+
await this.store.create(batch);
|
|
54
|
+
await this.store.saveRequests(id, request.requests);
|
|
55
|
+
if (native) {
|
|
56
|
+
this.processNativeBatch(id, request, native.adapter).catch(() => { });
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
this.processConcurrentBatch(id, request).catch(() => { });
|
|
60
|
+
}
|
|
61
|
+
return batch;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Create a batch and poll until completion.
|
|
65
|
+
*/
|
|
66
|
+
async createAndPoll(request, options = {}) {
|
|
67
|
+
const batch = await this.create(request);
|
|
68
|
+
return this.poll(batch.id, options);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Poll an existing batch until completion.
|
|
72
|
+
*/
|
|
73
|
+
async poll(id, options = {}) {
|
|
74
|
+
const interval = options.interval ?? this.defaultPollInterval;
|
|
75
|
+
const timeout = options.timeout ?? 0;
|
|
76
|
+
const startTime = Date.now();
|
|
77
|
+
while (true) {
|
|
78
|
+
let batch = await this.store.getMeta(id);
|
|
79
|
+
if (!batch) {
|
|
80
|
+
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
81
|
+
}
|
|
82
|
+
// For native batches, sync status from provider
|
|
83
|
+
if (batch.batch_mode === 'native' && batch.status === 'processing') {
|
|
84
|
+
await this.syncNativeBatchStatus(id);
|
|
85
|
+
batch = await this.store.getMeta(id);
|
|
86
|
+
if (!batch)
|
|
87
|
+
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
88
|
+
}
|
|
89
|
+
if (options.onProgress) {
|
|
90
|
+
options.onProgress(batch);
|
|
91
|
+
}
|
|
92
|
+
if (batch.status === 'completed' || batch.status === 'failed' || batch.status === 'cancelled') {
|
|
93
|
+
return this.getResults(id);
|
|
94
|
+
}
|
|
95
|
+
if (timeout > 0 && Date.now() - startTime > timeout) {
|
|
96
|
+
throw new AnyModelError(408, `Batch ${id} timed out after ${timeout}ms`);
|
|
97
|
+
}
|
|
98
|
+
await new Promise(resolve => setTimeout(resolve, interval));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Get the current status of a batch.
|
|
103
|
+
*/
|
|
104
|
+
async get(id) {
|
|
105
|
+
return this.store.getMeta(id);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Get results for a completed batch.
|
|
109
|
+
*/
|
|
110
|
+
async getResults(id) {
|
|
111
|
+
const batch = await this.store.getMeta(id);
|
|
112
|
+
if (!batch) {
|
|
113
|
+
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
114
|
+
}
|
|
115
|
+
const results = await this.store.getResults(id);
|
|
116
|
+
const usage = {
|
|
117
|
+
total_prompt_tokens: 0,
|
|
118
|
+
total_completion_tokens: 0,
|
|
119
|
+
estimated_cost: 0,
|
|
120
|
+
};
|
|
121
|
+
for (const result of results) {
|
|
122
|
+
if (result.response) {
|
|
123
|
+
usage.total_prompt_tokens += result.response.usage.prompt_tokens;
|
|
124
|
+
usage.total_completion_tokens += result.response.usage.completion_tokens;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return {
|
|
128
|
+
id: batch.id,
|
|
129
|
+
status: batch.status,
|
|
130
|
+
results,
|
|
131
|
+
usage_summary: usage,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* List all batches.
|
|
136
|
+
*/
|
|
137
|
+
async list() {
|
|
138
|
+
const ids = await this.store.listBatches();
|
|
139
|
+
const batches = [];
|
|
140
|
+
for (const id of ids) {
|
|
141
|
+
const meta = await this.store.getMeta(id);
|
|
142
|
+
if (meta)
|
|
143
|
+
batches.push(meta);
|
|
144
|
+
}
|
|
145
|
+
return batches;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Cancel a batch.
|
|
149
|
+
*/
|
|
150
|
+
async cancel(id) {
|
|
151
|
+
const batch = await this.store.getMeta(id);
|
|
152
|
+
if (!batch) {
|
|
153
|
+
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
154
|
+
}
|
|
155
|
+
if (batch.status === 'completed' || batch.status === 'cancelled') {
|
|
156
|
+
return batch;
|
|
157
|
+
}
|
|
158
|
+
// If native batch, cancel at provider too
|
|
159
|
+
if (batch.batch_mode === 'native') {
|
|
160
|
+
const providerState = await this.store.loadProviderState(id);
|
|
161
|
+
const adapter = this.batchAdapters.get(batch.provider_name);
|
|
162
|
+
if (adapter && providerState?.providerBatchId) {
|
|
163
|
+
try {
|
|
164
|
+
await adapter.cancelBatch(providerState.providerBatchId);
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
// Best-effort cancellation
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
batch.status = 'cancelled';
|
|
172
|
+
batch.completed_at = new Date().toISOString();
|
|
173
|
+
await this.store.updateMeta(batch);
|
|
174
|
+
return batch;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Process batch via native provider batch API.
|
|
178
|
+
*/
|
|
179
|
+
async processNativeBatch(batchId, request, adapter) {
|
|
180
|
+
const batch = await this.store.getMeta(batchId);
|
|
181
|
+
if (!batch)
|
|
182
|
+
return;
|
|
183
|
+
try {
|
|
184
|
+
const model = request.model.includes('/')
|
|
185
|
+
? request.model.split('/').slice(1).join('/')
|
|
186
|
+
: request.model;
|
|
187
|
+
const { providerBatchId, metadata } = await adapter.createBatch(model, request.requests, request.options);
|
|
188
|
+
await this.store.saveProviderState(batchId, {
|
|
189
|
+
providerBatchId,
|
|
190
|
+
providerName: batch.provider_name,
|
|
191
|
+
...metadata,
|
|
192
|
+
});
|
|
193
|
+
batch.status = 'processing';
|
|
194
|
+
await this.store.updateMeta(batch);
|
|
195
|
+
}
|
|
196
|
+
catch (err) {
|
|
197
|
+
batch.status = 'failed';
|
|
198
|
+
batch.completed_at = new Date().toISOString();
|
|
199
|
+
await this.store.updateMeta(batch);
|
|
200
|
+
throw err;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Sync native batch status from provider.
|
|
205
|
+
*/
|
|
206
|
+
async syncNativeBatchStatus(batchId) {
|
|
207
|
+
const batch = await this.store.getMeta(batchId);
|
|
208
|
+
if (!batch)
|
|
209
|
+
return;
|
|
210
|
+
const providerState = await this.store.loadProviderState(batchId);
|
|
211
|
+
if (!providerState?.providerBatchId)
|
|
212
|
+
return;
|
|
213
|
+
const adapter = this.batchAdapters.get(batch.provider_name);
|
|
214
|
+
if (!adapter)
|
|
215
|
+
return;
|
|
216
|
+
try {
|
|
217
|
+
const status = await adapter.pollBatch(providerState.providerBatchId);
|
|
218
|
+
batch.total = status.total || batch.total;
|
|
219
|
+
batch.completed = status.completed;
|
|
220
|
+
batch.failed = status.failed;
|
|
221
|
+
if (status.status === 'completed' || status.status === 'failed' || status.status === 'cancelled') {
|
|
222
|
+
batch.status = status.status;
|
|
223
|
+
batch.completed_at = new Date().toISOString();
|
|
224
|
+
if (status.status === 'completed' || status.status === 'failed') {
|
|
225
|
+
try {
|
|
226
|
+
const results = await adapter.getBatchResults(providerState.providerBatchId);
|
|
227
|
+
for (const result of results) {
|
|
228
|
+
await this.store.appendResult(batchId, result);
|
|
229
|
+
}
|
|
230
|
+
batch.completed = results.filter(r => r.status === 'success').length;
|
|
231
|
+
batch.failed = results.filter(r => r.status === 'error').length;
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
if (batch.status !== 'failed') {
|
|
235
|
+
batch.status = 'failed';
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
batch.status = 'processing';
|
|
242
|
+
}
|
|
243
|
+
await this.store.updateMeta(batch);
|
|
244
|
+
}
|
|
245
|
+
catch {
|
|
246
|
+
// Provider API error during poll — don't change status
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Process batch requests concurrently (fallback path).
|
|
251
|
+
*/
|
|
252
|
+
async processConcurrentBatch(batchId, request) {
|
|
253
|
+
const batch = await this.store.getMeta(batchId);
|
|
254
|
+
if (!batch)
|
|
255
|
+
return;
|
|
256
|
+
batch.status = 'processing';
|
|
257
|
+
await this.store.updateMeta(batch);
|
|
258
|
+
const items = request.requests;
|
|
259
|
+
const active = new Set();
|
|
260
|
+
const processItem = async (item) => {
|
|
261
|
+
const current = await this.store.getMeta(batchId);
|
|
262
|
+
if (current?.status === 'cancelled')
|
|
263
|
+
return;
|
|
264
|
+
const chatRequest = {
|
|
265
|
+
model: request.model,
|
|
266
|
+
messages: item.messages,
|
|
267
|
+
max_tokens: item.max_tokens ?? request.options?.max_tokens,
|
|
268
|
+
temperature: item.temperature ?? request.options?.temperature,
|
|
269
|
+
top_p: item.top_p ?? request.options?.top_p,
|
|
270
|
+
top_k: item.top_k ?? request.options?.top_k,
|
|
271
|
+
stop: item.stop ?? request.options?.stop,
|
|
272
|
+
response_format: item.response_format ?? request.options?.response_format,
|
|
273
|
+
tools: item.tools ?? request.options?.tools,
|
|
274
|
+
tool_choice: item.tool_choice ?? request.options?.tool_choice,
|
|
275
|
+
};
|
|
276
|
+
let result;
|
|
277
|
+
try {
|
|
278
|
+
const response = await this.router.complete(chatRequest);
|
|
279
|
+
result = {
|
|
280
|
+
custom_id: item.custom_id,
|
|
281
|
+
status: 'success',
|
|
282
|
+
response,
|
|
283
|
+
error: null,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
catch (err) {
|
|
287
|
+
const error = err instanceof AnyModelError ? err : new AnyModelError(500, String(err));
|
|
288
|
+
result = {
|
|
289
|
+
custom_id: item.custom_id,
|
|
290
|
+
status: 'error',
|
|
291
|
+
response: null,
|
|
292
|
+
error: { code: error.code, message: error.message },
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
await this.store.appendResult(batchId, result);
|
|
296
|
+
const meta = await this.store.getMeta(batchId);
|
|
297
|
+
if (meta) {
|
|
298
|
+
if (result.status === 'success') {
|
|
299
|
+
meta.completed++;
|
|
300
|
+
}
|
|
301
|
+
else {
|
|
302
|
+
meta.failed++;
|
|
303
|
+
}
|
|
304
|
+
await this.store.updateMeta(meta);
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
for (const item of items) {
|
|
308
|
+
const current = await this.store.getMeta(batchId);
|
|
309
|
+
if (current?.status === 'cancelled')
|
|
310
|
+
break;
|
|
311
|
+
if (active.size >= this.concurrencyLimit) {
|
|
312
|
+
await Promise.race(active);
|
|
313
|
+
}
|
|
314
|
+
const promise = processItem(item).then(() => {
|
|
315
|
+
active.delete(promise);
|
|
316
|
+
});
|
|
317
|
+
active.add(promise);
|
|
318
|
+
}
|
|
319
|
+
await Promise.all(active);
|
|
320
|
+
const finalMeta = await this.store.getMeta(batchId);
|
|
321
|
+
if (finalMeta && finalMeta.status !== 'cancelled') {
|
|
322
|
+
finalMeta.status = finalMeta.failed === finalMeta.total ? 'failed' : 'completed';
|
|
323
|
+
finalMeta.completed_at = new Date().toISOString();
|
|
324
|
+
await this.store.updateMeta(finalMeta);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
//# sourceMappingURL=manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/batch/manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAaxC,MAAM,OAAO,YAAY;IACf,KAAK,CAAa;IAClB,MAAM,CAAS;IACf,gBAAgB,CAAS;IACzB,mBAAmB,CAAS;IAC5B,aAAa,GAAG,IAAI,GAAG,EAAwB,CAAC;IAExD,YAAY,MAAc,EAAE,OAAuE;QACjG,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,WAAW,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,YAAY,IAAI,IAAI,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,YAAoB,EAAE,OAAqB;QAC9D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,KAAa;QACzC,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAA2B;QACtC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,QAAiB,CAAC,CAAC,CAAC,YAAqB,CAAC;QAErE,MAAM,KAAK,GAAgB;YACzB,EAAE;YACF,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,aAAa,EAAE,YAAY;YAC3B,UAAU,EAAE,SAAS;YACrB,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;YAC9B,SAAS,EAAE,CAAC;YACZ,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,GAAG;YACf,YAAY,EAAE,IAAI;YAClB,UAAU,EAAE,IAAI;SACjB,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEpD,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,sBAAsB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,OAA2B,EAC3B,UAA4B,EAAE;QAE9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,EAAU,EAAE,UAA4B,EAAE;QACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,mBAAmB,CAAC;QAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;YACxD,CAAC;YAED,gDAAgD;YAChD,IAAI,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBACnE,MAAM,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;gBACrC,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBACrC,IAAI,CAAC,KAAK;oBAAE,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;YACpE,CAAC;YAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAC9F,OAAO,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,oBAAoB,OAAO,IAAI,CAAC,CAAC;YAC3E,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU;QACzB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAEhD,MAAM,KAAK,GAAsB;YAC/B,mBAAmB,EAAE,CAAC;YACtB,uBAAuB,EAAE,CAAC;YAC1B,cAAc,EAAE,CAAC;SAClB,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACpB,KAAK,CAAC,mBAAmB,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;gBACjE,KAAK,CAAC,uBAAuB,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO;YACP,aAAa,EAAE,KAAK;SACrB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC1C,IAAI,IAAI;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACjE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,0CAA0C;QAC1C,IAAI,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC5D,IAAI,OAAO,IAAI,aAAa,EAAE,eAAe,EAAE,CAAC;gBAC9C,IAAI,CAAC;oBACH,MAAM,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,eAAyB,CAAC,CAAC;gBACrE,CAAC;gBAAC,MAAM,CAAC;oBACP,2BAA2B;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC;QAC3B,KAAK,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC9B,OAAe,EACf,OAA2B,EAC3B,OAAqB;QAErB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;gBACvC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBAC7C,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;YAElB,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,WAAW,CAC7D,KAAK,EACL,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,OAA8C,CACvD,CAAC;YAEF,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,OAAO,EAAE;gBAC1C,eAAe;gBACf,YAAY,EAAE,KAAK,CAAC,aAAa;gBACjC,GAAG,QAAQ;aACZ,CAAC,CAAC;YAEH,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC;YAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;YACxB,KAAK,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAC,OAAe;QACjD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,aAAa,EAAE,eAAe;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5D,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,eAAyB,CAAC,CAAC;YAEhF,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;YAC1C,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YACnC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAE7B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACjG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC7B,KAAK,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAE9C,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBAChE,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,aAAa,CAAC,eAAyB,CAAC,CAAC;wBACvF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;4BAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;wBACjD,CAAC;wBACD,KAAK,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;wBACrE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;oBAClE,CAAC;oBAAC,MAAM,CAAC;wBACP,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;4BAC9B,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;wBAC1B,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC;YAC9B,CAAC;YAED,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;QACzD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,sBAAsB,CAAC,OAAe,EAAE,OAA2B;QAC/E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC;QAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAEnC,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAC;QAExC,MAAM,WAAW,GAAG,KAAK,EAAE,IAAqB,EAAiB,EAAE;YACjE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,OAAO,EAAE,MAAM,KAAK,WAAW;gBAAE,OAAO;YAE5C,MAAM,WAAW,GAA0B;gBACzC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,EAAE,UAAU;gBAC1D,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,OAAO,EAAE,WAAW;gBAC7D,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,KAAK;gBAC3C,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,KAAK;gBAC3C,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,IAAI;gBACxC,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,OAAO,CAAC,OAAO,EAAE,eAAe;gBACzE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,KAAK;gBAC3C,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,OAAO,EAAE,WAAW;aAC9D,CAAC;YAEF,IAAI,MAAuB,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBACzD,MAAM,GAAG;oBACP,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,MAAM,EAAE,SAAS;oBACjB,QAAQ;oBACR,KAAK,EAAE,IAAI;iBACZ,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,GAAG,YAAY,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBACvF,MAAM,GAAG;oBACP,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,MAAM,EAAE,OAAO;oBACf,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE;iBACpD,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;oBAChC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,CAAC;gBACD,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,OAAO,EAAE,MAAM,KAAK,WAAW;gBAAE,MAAM;YAE3C,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACzC,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;YAED,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBAC1C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAE1B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClD,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;YACjF,SAAS,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAClD,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { BatchObject, BatchResultItem } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Disk-based batch persistence store.
|
|
4
|
+
* Uses queued, concurrency-limited IO for high-volume operations.
|
|
5
|
+
* Structure: {dir}/{batchId}/meta.json, requests.jsonl, results.jsonl, provider.json
|
|
6
|
+
*/
|
|
7
|
+
export declare class BatchStore {
|
|
8
|
+
private dir;
|
|
9
|
+
private initialized;
|
|
10
|
+
constructor(dir?: string);
|
|
11
|
+
private init;
|
|
12
|
+
private batchDir;
|
|
13
|
+
/**
|
|
14
|
+
* Create a new batch directory and save initial metadata.
|
|
15
|
+
*/
|
|
16
|
+
create(batch: BatchObject): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Update batch metadata (atomic write).
|
|
19
|
+
*/
|
|
20
|
+
updateMeta(batch: BatchObject): Promise<void>;
|
|
21
|
+
/**
|
|
22
|
+
* Save requests as JSONL.
|
|
23
|
+
*/
|
|
24
|
+
saveRequests(id: string, requests: unknown[]): Promise<void>;
|
|
25
|
+
/**
|
|
26
|
+
* Append a result to results.jsonl.
|
|
27
|
+
*/
|
|
28
|
+
appendResult(id: string, result: BatchResultItem): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Save provider-specific state (e.g., provider batch ID).
|
|
31
|
+
*/
|
|
32
|
+
saveProviderState(id: string, state: Record<string, unknown>): Promise<void>;
|
|
33
|
+
/**
|
|
34
|
+
* Load provider state.
|
|
35
|
+
*/
|
|
36
|
+
loadProviderState(id: string): Promise<Record<string, unknown> | null>;
|
|
37
|
+
/**
|
|
38
|
+
* Get batch metadata.
|
|
39
|
+
*/
|
|
40
|
+
getMeta(id: string): Promise<BatchObject | null>;
|
|
41
|
+
/**
|
|
42
|
+
* Get all results for a batch.
|
|
43
|
+
*/
|
|
44
|
+
getResults(id: string): Promise<BatchResultItem[]>;
|
|
45
|
+
/**
|
|
46
|
+
* List all batch IDs.
|
|
47
|
+
*/
|
|
48
|
+
listBatches(): Promise<string[]>;
|
|
49
|
+
/**
|
|
50
|
+
* Check if a batch exists.
|
|
51
|
+
*/
|
|
52
|
+
exists(id: string): Promise<boolean>;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=store.d.ts.map
|