@probeo/anymodel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -6
- package/dist/batch/index.d.ts +4 -0
- package/dist/batch/index.d.ts.map +1 -0
- package/dist/batch/index.js +3 -0
- package/dist/batch/index.js.map +1 -0
- package/dist/batch/manager.d.ts +72 -0
- package/dist/batch/manager.d.ts.map +1 -0
- package/dist/batch/manager.js +328 -0
- package/dist/batch/manager.js.map +1 -0
- package/dist/batch/store.d.ts +54 -0
- package/dist/batch/store.d.ts.map +1 -0
- package/dist/batch/store.js +109 -0
- package/dist/batch/store.js.map +1 -0
- package/dist/cli.cjs +4 -28
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +28 -2110
- package/dist/cli.js.map +1 -1
- package/dist/client.d.ts +42 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +181 -0
- package/dist/client.js.map +1 -0
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +120 -0
- package/dist/config.js.map +1 -0
- package/dist/index.cjs +4 -28
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +14 -548
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -2089
- package/dist/index.js.map +1 -1
- package/dist/providers/adapter.d.ts +33 -0
- package/dist/providers/adapter.d.ts.map +1 -0
- package/dist/providers/adapter.js +2 -0
- package/dist/providers/adapter.js.map +1 -0
- package/dist/providers/anthropic-batch.d.ts +3 -0
- package/dist/providers/anthropic-batch.d.ts.map +1 -0
- package/dist/providers/anthropic-batch.js +228 -0
- package/dist/providers/anthropic-batch.js.map +1 -0
- package/dist/providers/anthropic.d.ts +3 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +358 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/custom.d.ts +8 -0
- package/dist/providers/custom.d.ts.map +1 -0
- package/dist/providers/custom.js +61 -0
- package/dist/providers/custom.js.map +1 -0
- package/dist/providers/google.d.ts +3 -0
- package/dist/providers/google.d.ts.map +1 -0
- package/dist/providers/google.js +331 -0
- package/dist/providers/google.js.map +1 -0
- package/dist/providers/index.d.ts +6 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +5 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/openai-batch.d.ts +3 -0
- package/dist/providers/openai-batch.d.ts.map +1 -0
- package/dist/providers/openai-batch.js +208 -0
- package/dist/providers/openai-batch.js.map +1 -0
- package/dist/providers/openai.d.ts +3 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +221 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/registry.d.ts +10 -0
- package/dist/providers/registry.d.ts.map +1 -0
- package/dist/providers/registry.js +27 -0
- package/dist/providers/registry.js.map +1 -0
- package/dist/router.d.ts +29 -0
- package/dist/router.d.ts.map +1 -0
- package/dist/router.js +212 -0
- package/dist/router.js.map +1 -0
- package/dist/server.d.ts +10 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +149 -0
- package/dist/server.js.map +1 -0
- package/dist/types.d.ts +283 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +21 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/fs-io.d.ts +40 -0
- package/dist/utils/fs-io.d.ts.map +1 -0
- package/dist/utils/fs-io.js +203 -0
- package/dist/utils/fs-io.js.map +1 -0
- package/dist/utils/generation-stats.d.ts +25 -0
- package/dist/utils/generation-stats.d.ts.map +1 -0
- package/dist/utils/generation-stats.js +46 -0
- package/dist/utils/generation-stats.js.map +1 -0
- package/dist/utils/id.d.ts +2 -0
- package/dist/utils/id.d.ts.map +1 -0
- package/dist/utils/id.js +6 -0
- package/dist/utils/id.js.map +1 -0
- package/dist/utils/model-parser.d.ts +6 -0
- package/dist/utils/model-parser.d.ts.map +1 -0
- package/dist/utils/model-parser.js +21 -0
- package/dist/utils/model-parser.js.map +1 -0
- package/dist/utils/rate-limiter.d.ts +36 -0
- package/dist/utils/rate-limiter.d.ts.map +1 -0
- package/dist/utils/rate-limiter.js +80 -0
- package/dist/utils/rate-limiter.js.map +1 -0
- package/dist/utils/retry.d.ts +7 -0
- package/dist/utils/retry.d.ts.map +1 -0
- package/dist/utils/retry.js +54 -0
- package/dist/utils/retry.js.map +1 -0
- package/dist/utils/transforms.d.ts +7 -0
- package/dist/utils/transforms.d.ts.map +1 -0
- package/dist/utils/transforms.js +66 -0
- package/dist/utils/transforms.js.map +1 -0
- package/dist/utils/validate.d.ts +3 -0
- package/dist/utils/validate.d.ts.map +1 -0
- package/dist/utils/validate.js +31 -0
- package/dist/utils/validate.js.map +1 -0
- package/package.json +4 -1
package/README.md
CHANGED
|
@@ -147,11 +147,13 @@ const response = await client.chat.completions.create({
|
|
|
147
147
|
|
|
148
148
|
## Batch Processing
|
|
149
149
|
|
|
150
|
-
Process many requests
|
|
150
|
+
Process many requests with native provider batch APIs or concurrent fallback. OpenAI and Anthropic batches are processed server-side — OpenAI at 50% cost, Anthropic with async processing for up to 10K requests. Other providers fall back to concurrent execution automatically.
|
|
151
|
+
|
|
152
|
+
### Submit and wait
|
|
151
153
|
|
|
152
154
|
```typescript
|
|
153
155
|
const results = await client.batches.createAndPoll({
|
|
154
|
-
model: "
|
|
156
|
+
model: "openai/gpt-4o-mini",
|
|
155
157
|
requests: [
|
|
156
158
|
{ custom_id: "req-1", messages: [{ role: "user", content: "Summarize AI" }] },
|
|
157
159
|
{ custom_id: "req-2", messages: [{ role: "user", content: "Summarize ML" }] },
|
|
@@ -164,14 +166,71 @@ for (const result of results.results) {
|
|
|
164
166
|
}
|
|
165
167
|
```
|
|
166
168
|
|
|
167
|
-
|
|
169
|
+
### Submit now, check later
|
|
170
|
+
|
|
171
|
+
Submit a batch and get back an ID immediately — no need to keep the process running for native batches (OpenAI, Anthropic):
|
|
172
|
+
|
|
173
|
+
```typescript
|
|
174
|
+
// Submit and get the batch ID
|
|
175
|
+
const batch = await client.batches.create({
|
|
176
|
+
model: "anthropic/claude-haiku-4-5",
|
|
177
|
+
requests: [
|
|
178
|
+
{ custom_id: "req-1", messages: [{ role: "user", content: "Summarize AI" }] },
|
|
179
|
+
{ custom_id: "req-2", messages: [{ role: "user", content: "Summarize ML" }] },
|
|
180
|
+
],
|
|
181
|
+
});
|
|
182
|
+
console.log(batch.id); // "batch-abc123"
|
|
183
|
+
console.log(batch.batch_mode); // "native" or "concurrent"
|
|
184
|
+
|
|
185
|
+
// Check status any time — even after a process restart
|
|
186
|
+
const status = client.batches.get("batch-abc123");
|
|
187
|
+
console.log(status.status); // "pending", "processing", "completed", "failed"
|
|
188
|
+
|
|
189
|
+
// Wait for results when you're ready (reconnects to provider API)
|
|
190
|
+
const results = await client.batches.poll("batch-abc123");
|
|
191
|
+
|
|
192
|
+
// Or get results directly if already completed
|
|
193
|
+
const results = client.batches.results("batch-abc123");
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### List and cancel
|
|
197
|
+
|
|
198
|
+
```typescript
|
|
199
|
+
// List all batches on disk
|
|
200
|
+
const all = client.batches.list();
|
|
201
|
+
for (const b of all) {
|
|
202
|
+
console.log(b.id, b.batch_mode, b.status, b.provider_name);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Cancel a running batch (also cancels at the provider for native batches)
|
|
206
|
+
await client.batches.cancel("batch-abc123");
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Batch configuration
|
|
168
210
|
|
|
169
211
|
```typescript
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
212
|
+
const client = new AnyModel({
|
|
213
|
+
batch: {
|
|
214
|
+
pollInterval: 10000, // default poll interval in ms (default: 5000)
|
|
215
|
+
concurrencyFallback: 10, // concurrent request limit for non-native providers (default: 5)
|
|
216
|
+
},
|
|
217
|
+
io: {
|
|
218
|
+
readConcurrency: 30, // concurrent file reads (default: 20)
|
|
219
|
+
writeConcurrency: 15, // concurrent file writes (default: 10)
|
|
220
|
+
},
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
// Override poll interval per call
|
|
224
|
+
const results = await client.batches.createAndPoll(request, {
|
|
225
|
+
interval: 3000, // poll every 3s for this batch
|
|
226
|
+
onProgress: (batch) => {
|
|
227
|
+
console.log(`${batch.completed}/${batch.total} done`);
|
|
228
|
+
},
|
|
229
|
+
});
|
|
173
230
|
```
|
|
174
231
|
|
|
232
|
+
Batches are persisted to `./.anymodel/batches/` in the current working directory and survive process restarts.
|
|
233
|
+
|
|
175
234
|
## Models Endpoint
|
|
176
235
|
|
|
177
236
|
```typescript
|
|
@@ -231,6 +290,14 @@ Create `anymodel.config.json` in your project root:
|
|
|
231
290
|
"defaults": {
|
|
232
291
|
"temperature": 0.7,
|
|
233
292
|
"max_tokens": 4096
|
|
293
|
+
},
|
|
294
|
+
"batch": {
|
|
295
|
+
"pollInterval": 5000,
|
|
296
|
+
"concurrencyFallback": 5
|
|
297
|
+
},
|
|
298
|
+
"io": {
|
|
299
|
+
"readConcurrency": 20,
|
|
300
|
+
"writeConcurrency": 10
|
|
234
301
|
}
|
|
235
302
|
}
|
|
236
303
|
```
|
|
@@ -358,6 +425,25 @@ npx tsx examples/basic.ts batch
|
|
|
358
425
|
- **Retries**: Automatic retry with exponential backoff on 429/502/503 errors (configurable via `defaults.retries`)
|
|
359
426
|
- **Rate limit tracking**: Per-provider rate limit state, automatically skips rate-limited providers during fallback routing
|
|
360
427
|
- **Parameter stripping**: Unsupported parameters are automatically removed before forwarding to providers
|
|
428
|
+
- **High-volume IO**: All batch file operations use concurrency-limited async queues with atomic durable writes (temp file + fsync + rename) to prevent corruption on crash. Defaults: 20 concurrent reads, 10 concurrent writes — configurable via `io.readConcurrency` and `io.writeConcurrency`
|
|
429
|
+
|
|
430
|
+
## Roadmap
|
|
431
|
+
|
|
432
|
+
- [ ] **A/B testing** — split routing (% traffic to each model) and compare mode (same request to multiple models, return all responses with stats)
|
|
433
|
+
- [ ] **Cost tracking** — per-request and aggregate cost calculation from provider pricing
|
|
434
|
+
- [ ] **Caching** — response caching with configurable TTL for identical requests
|
|
435
|
+
- [x] **Native batch APIs** — OpenAI Batch API (JSONL upload, 50% cost) and Anthropic Message Batches (10K requests, async). Auto-detects provider and routes to native API, falls back to concurrent for other providers
|
|
436
|
+
- [ ] **Webhooks** — notify on batch completion
|
|
437
|
+
- [ ] **Prompt logging** — optional request/response logging for debugging and evaluation
|
|
438
|
+
|
|
439
|
+
## Roadmap
|
|
440
|
+
|
|
441
|
+
- [ ] **A/B testing** — split routing (% traffic to each model) and compare mode (same request to multiple models, return all responses with stats)
|
|
442
|
+
- [ ] **Cost tracking** — per-request and aggregate cost calculation from provider pricing
|
|
443
|
+
- [ ] **Caching** — response caching with configurable TTL for identical requests
|
|
444
|
+
- [ ] **Native batch APIs** — use Anthropic Message Batches and OpenAI Batch API instead of concurrent requests
|
|
445
|
+
- [ ] **Webhooks** — notify on batch completion
|
|
446
|
+
- [ ] **Prompt logging** — optional request/response logging for debugging and evaluation
|
|
361
447
|
|
|
362
448
|
## License
|
|
363
449
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/batch/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/batch/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { BatchCreateRequest, BatchObject, BatchResults } from '../types.js';
|
|
2
|
+
import type { Router } from '../router.js';
|
|
3
|
+
import type { BatchAdapter } from '../providers/adapter.js';
|
|
4
|
+
export interface BatchPollOptions {
|
|
5
|
+
/** Poll interval in ms. Default: 5000 */
|
|
6
|
+
interval?: number;
|
|
7
|
+
/** Timeout in ms. 0 = indefinite. Default: 0 */
|
|
8
|
+
timeout?: number;
|
|
9
|
+
/** Progress callback */
|
|
10
|
+
onProgress?: (batch: BatchObject) => void;
|
|
11
|
+
}
|
|
12
|
+
export declare class BatchManager {
|
|
13
|
+
private store;
|
|
14
|
+
private router;
|
|
15
|
+
private concurrencyLimit;
|
|
16
|
+
private defaultPollInterval;
|
|
17
|
+
private batchAdapters;
|
|
18
|
+
constructor(router: Router, options?: {
|
|
19
|
+
dir?: string;
|
|
20
|
+
concurrency?: number;
|
|
21
|
+
pollInterval?: number;
|
|
22
|
+
});
|
|
23
|
+
/**
|
|
24
|
+
* Register a native batch adapter for a provider.
|
|
25
|
+
*/
|
|
26
|
+
registerBatchAdapter(providerName: string, adapter: BatchAdapter): void;
|
|
27
|
+
/**
|
|
28
|
+
* Check if a provider has native batch support.
|
|
29
|
+
*/
|
|
30
|
+
private getNativeBatchAdapter;
|
|
31
|
+
/**
|
|
32
|
+
* Create a batch and return immediately (no polling).
|
|
33
|
+
*/
|
|
34
|
+
create(request: BatchCreateRequest): Promise<BatchObject>;
|
|
35
|
+
/**
|
|
36
|
+
* Create a batch and poll until completion.
|
|
37
|
+
*/
|
|
38
|
+
createAndPoll(request: BatchCreateRequest, options?: BatchPollOptions): Promise<BatchResults>;
|
|
39
|
+
/**
|
|
40
|
+
* Poll an existing batch until completion.
|
|
41
|
+
*/
|
|
42
|
+
poll(id: string, options?: BatchPollOptions): Promise<BatchResults>;
|
|
43
|
+
/**
|
|
44
|
+
* Get the current status of a batch.
|
|
45
|
+
*/
|
|
46
|
+
get(id: string): Promise<BatchObject | null>;
|
|
47
|
+
/**
|
|
48
|
+
* Get results for a completed batch.
|
|
49
|
+
*/
|
|
50
|
+
getResults(id: string): Promise<BatchResults>;
|
|
51
|
+
/**
|
|
52
|
+
* List all batches.
|
|
53
|
+
*/
|
|
54
|
+
list(): Promise<BatchObject[]>;
|
|
55
|
+
/**
|
|
56
|
+
* Cancel a batch.
|
|
57
|
+
*/
|
|
58
|
+
cancel(id: string): Promise<BatchObject>;
|
|
59
|
+
/**
|
|
60
|
+
* Process batch via native provider batch API.
|
|
61
|
+
*/
|
|
62
|
+
private processNativeBatch;
|
|
63
|
+
/**
|
|
64
|
+
* Sync native batch status from provider.
|
|
65
|
+
*/
|
|
66
|
+
private syncNativeBatchStatus;
|
|
67
|
+
/**
|
|
68
|
+
* Process batch requests concurrently (fallback path).
|
|
69
|
+
*/
|
|
70
|
+
private processConcurrentBatch;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../src/batch/manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAClB,WAAW,EACX,YAAY,EAIb,MAAM,aAAa,CAAC;AAIrB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;CAC3C;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,aAAa,CAAmC;gBAE5C,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;IAOnG;;OAEG;IACH,oBAAoB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,IAAI;IAIvE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAM7B;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;IAkC/D;;OAEG;IACG,aAAa,CACjB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,YAAY,CAAC;IAKxB;;OAEG;IACG,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC;IAkC7E;;OAEG;IACG,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAIlD;;OAEG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA6BnD;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAUpC;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IA4B9C;;OAEG;YACW,kBAAkB;IAmChC;;OAEG;YACW,qBAAqB;IA6CnC;;OAEG;YACW,sBAAsB;CAiFrC"}
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
import { AnyModelError } from '../types.js';
|
|
2
|
+
import { generateId } from '../utils/id.js';
|
|
3
|
+
import { BatchStore } from './store.js';
|
|
4
|
+
export class BatchManager {
|
|
5
|
+
store;
|
|
6
|
+
router;
|
|
7
|
+
concurrencyLimit;
|
|
8
|
+
defaultPollInterval;
|
|
9
|
+
batchAdapters = new Map();
|
|
10
|
+
constructor(router, options) {
|
|
11
|
+
this.store = new BatchStore(options?.dir);
|
|
12
|
+
this.router = router;
|
|
13
|
+
this.concurrencyLimit = options?.concurrency ?? 5;
|
|
14
|
+
this.defaultPollInterval = options?.pollInterval ?? 5000;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Register a native batch adapter for a provider.
|
|
18
|
+
*/
|
|
19
|
+
registerBatchAdapter(providerName, adapter) {
|
|
20
|
+
this.batchAdapters.set(providerName, adapter);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Check if a provider has native batch support.
|
|
24
|
+
*/
|
|
25
|
+
getNativeBatchAdapter(model) {
|
|
26
|
+
const providerName = model.split('/')[0];
|
|
27
|
+
const adapter = this.batchAdapters.get(providerName);
|
|
28
|
+
return adapter ? { adapter, providerName } : null;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Create a batch and return immediately (no polling).
|
|
32
|
+
*/
|
|
33
|
+
async create(request) {
|
|
34
|
+
const id = generateId('batch');
|
|
35
|
+
const now = new Date().toISOString();
|
|
36
|
+
const providerName = request.model.split('/')[0] || 'unknown';
|
|
37
|
+
const native = this.getNativeBatchAdapter(request.model);
|
|
38
|
+
const batchMode = native ? 'native' : 'concurrent';
|
|
39
|
+
const batch = {
|
|
40
|
+
id,
|
|
41
|
+
object: 'batch',
|
|
42
|
+
status: 'pending',
|
|
43
|
+
model: request.model,
|
|
44
|
+
provider_name: providerName,
|
|
45
|
+
batch_mode: batchMode,
|
|
46
|
+
total: request.requests.length,
|
|
47
|
+
completed: 0,
|
|
48
|
+
failed: 0,
|
|
49
|
+
created_at: now,
|
|
50
|
+
completed_at: null,
|
|
51
|
+
expires_at: null,
|
|
52
|
+
};
|
|
53
|
+
await this.store.create(batch);
|
|
54
|
+
await this.store.saveRequests(id, request.requests);
|
|
55
|
+
if (native) {
|
|
56
|
+
this.processNativeBatch(id, request, native.adapter).catch(() => { });
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
this.processConcurrentBatch(id, request).catch(() => { });
|
|
60
|
+
}
|
|
61
|
+
return batch;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Create a batch and poll until completion.
|
|
65
|
+
*/
|
|
66
|
+
async createAndPoll(request, options = {}) {
|
|
67
|
+
const batch = await this.create(request);
|
|
68
|
+
return this.poll(batch.id, options);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Poll an existing batch until completion.
|
|
72
|
+
*/
|
|
73
|
+
async poll(id, options = {}) {
|
|
74
|
+
const interval = options.interval ?? this.defaultPollInterval;
|
|
75
|
+
const timeout = options.timeout ?? 0;
|
|
76
|
+
const startTime = Date.now();
|
|
77
|
+
while (true) {
|
|
78
|
+
let batch = await this.store.getMeta(id);
|
|
79
|
+
if (!batch) {
|
|
80
|
+
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
81
|
+
}
|
|
82
|
+
// For native batches, sync status from provider
|
|
83
|
+
if (batch.batch_mode === 'native' && batch.status === 'processing') {
|
|
84
|
+
await this.syncNativeBatchStatus(id);
|
|
85
|
+
batch = await this.store.getMeta(id);
|
|
86
|
+
if (!batch)
|
|
87
|
+
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
88
|
+
}
|
|
89
|
+
if (options.onProgress) {
|
|
90
|
+
options.onProgress(batch);
|
|
91
|
+
}
|
|
92
|
+
if (batch.status === 'completed' || batch.status === 'failed' || batch.status === 'cancelled') {
|
|
93
|
+
return this.getResults(id);
|
|
94
|
+
}
|
|
95
|
+
if (timeout > 0 && Date.now() - startTime > timeout) {
|
|
96
|
+
throw new AnyModelError(408, `Batch ${id} timed out after ${timeout}ms`);
|
|
97
|
+
}
|
|
98
|
+
await new Promise(resolve => setTimeout(resolve, interval));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Get the current status of a batch.
|
|
103
|
+
*/
|
|
104
|
+
async get(id) {
|
|
105
|
+
return this.store.getMeta(id);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Get results for a completed batch.
|
|
109
|
+
*/
|
|
110
|
+
async getResults(id) {
|
|
111
|
+
const batch = await this.store.getMeta(id);
|
|
112
|
+
if (!batch) {
|
|
113
|
+
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
114
|
+
}
|
|
115
|
+
const results = await this.store.getResults(id);
|
|
116
|
+
const usage = {
|
|
117
|
+
total_prompt_tokens: 0,
|
|
118
|
+
total_completion_tokens: 0,
|
|
119
|
+
estimated_cost: 0,
|
|
120
|
+
};
|
|
121
|
+
for (const result of results) {
|
|
122
|
+
if (result.response) {
|
|
123
|
+
usage.total_prompt_tokens += result.response.usage.prompt_tokens;
|
|
124
|
+
usage.total_completion_tokens += result.response.usage.completion_tokens;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return {
|
|
128
|
+
id: batch.id,
|
|
129
|
+
status: batch.status,
|
|
130
|
+
results,
|
|
131
|
+
usage_summary: usage,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* List all batches.
|
|
136
|
+
*/
|
|
137
|
+
async list() {
|
|
138
|
+
const ids = await this.store.listBatches();
|
|
139
|
+
const batches = [];
|
|
140
|
+
for (const id of ids) {
|
|
141
|
+
const meta = await this.store.getMeta(id);
|
|
142
|
+
if (meta)
|
|
143
|
+
batches.push(meta);
|
|
144
|
+
}
|
|
145
|
+
return batches;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Cancel a batch.
|
|
149
|
+
*/
|
|
150
|
+
async cancel(id) {
|
|
151
|
+
const batch = await this.store.getMeta(id);
|
|
152
|
+
if (!batch) {
|
|
153
|
+
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
154
|
+
}
|
|
155
|
+
if (batch.status === 'completed' || batch.status === 'cancelled') {
|
|
156
|
+
return batch;
|
|
157
|
+
}
|
|
158
|
+
// If native batch, cancel at provider too
|
|
159
|
+
if (batch.batch_mode === 'native') {
|
|
160
|
+
const providerState = await this.store.loadProviderState(id);
|
|
161
|
+
const adapter = this.batchAdapters.get(batch.provider_name);
|
|
162
|
+
if (adapter && providerState?.providerBatchId) {
|
|
163
|
+
try {
|
|
164
|
+
await adapter.cancelBatch(providerState.providerBatchId);
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
// Best-effort cancellation
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
batch.status = 'cancelled';
|
|
172
|
+
batch.completed_at = new Date().toISOString();
|
|
173
|
+
await this.store.updateMeta(batch);
|
|
174
|
+
return batch;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Process batch via native provider batch API.
|
|
178
|
+
*/
|
|
179
|
+
async processNativeBatch(batchId, request, adapter) {
|
|
180
|
+
const batch = await this.store.getMeta(batchId);
|
|
181
|
+
if (!batch)
|
|
182
|
+
return;
|
|
183
|
+
try {
|
|
184
|
+
const model = request.model.includes('/')
|
|
185
|
+
? request.model.split('/').slice(1).join('/')
|
|
186
|
+
: request.model;
|
|
187
|
+
const { providerBatchId, metadata } = await adapter.createBatch(model, request.requests, request.options);
|
|
188
|
+
await this.store.saveProviderState(batchId, {
|
|
189
|
+
providerBatchId,
|
|
190
|
+
providerName: batch.provider_name,
|
|
191
|
+
...metadata,
|
|
192
|
+
});
|
|
193
|
+
batch.status = 'processing';
|
|
194
|
+
await this.store.updateMeta(batch);
|
|
195
|
+
}
|
|
196
|
+
catch (err) {
|
|
197
|
+
batch.status = 'failed';
|
|
198
|
+
batch.completed_at = new Date().toISOString();
|
|
199
|
+
await this.store.updateMeta(batch);
|
|
200
|
+
throw err;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Sync native batch status from provider.
|
|
205
|
+
*/
|
|
206
|
+
async syncNativeBatchStatus(batchId) {
|
|
207
|
+
const batch = await this.store.getMeta(batchId);
|
|
208
|
+
if (!batch)
|
|
209
|
+
return;
|
|
210
|
+
const providerState = await this.store.loadProviderState(batchId);
|
|
211
|
+
if (!providerState?.providerBatchId)
|
|
212
|
+
return;
|
|
213
|
+
const adapter = this.batchAdapters.get(batch.provider_name);
|
|
214
|
+
if (!adapter)
|
|
215
|
+
return;
|
|
216
|
+
try {
|
|
217
|
+
const status = await adapter.pollBatch(providerState.providerBatchId);
|
|
218
|
+
batch.total = status.total || batch.total;
|
|
219
|
+
batch.completed = status.completed;
|
|
220
|
+
batch.failed = status.failed;
|
|
221
|
+
if (status.status === 'completed' || status.status === 'failed' || status.status === 'cancelled') {
|
|
222
|
+
batch.status = status.status;
|
|
223
|
+
batch.completed_at = new Date().toISOString();
|
|
224
|
+
if (status.status === 'completed' || status.status === 'failed') {
|
|
225
|
+
try {
|
|
226
|
+
const results = await adapter.getBatchResults(providerState.providerBatchId);
|
|
227
|
+
for (const result of results) {
|
|
228
|
+
await this.store.appendResult(batchId, result);
|
|
229
|
+
}
|
|
230
|
+
batch.completed = results.filter(r => r.status === 'success').length;
|
|
231
|
+
batch.failed = results.filter(r => r.status === 'error').length;
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
if (batch.status !== 'failed') {
|
|
235
|
+
batch.status = 'failed';
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
batch.status = 'processing';
|
|
242
|
+
}
|
|
243
|
+
await this.store.updateMeta(batch);
|
|
244
|
+
}
|
|
245
|
+
catch {
|
|
246
|
+
// Provider API error during poll — don't change status
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Process batch requests concurrently (fallback path).
|
|
251
|
+
*/
|
|
252
|
+
async processConcurrentBatch(batchId, request) {
|
|
253
|
+
const batch = await this.store.getMeta(batchId);
|
|
254
|
+
if (!batch)
|
|
255
|
+
return;
|
|
256
|
+
batch.status = 'processing';
|
|
257
|
+
await this.store.updateMeta(batch);
|
|
258
|
+
const items = request.requests;
|
|
259
|
+
const active = new Set();
|
|
260
|
+
const processItem = async (item) => {
|
|
261
|
+
const current = await this.store.getMeta(batchId);
|
|
262
|
+
if (current?.status === 'cancelled')
|
|
263
|
+
return;
|
|
264
|
+
const chatRequest = {
|
|
265
|
+
model: request.model,
|
|
266
|
+
messages: item.messages,
|
|
267
|
+
max_tokens: item.max_tokens ?? request.options?.max_tokens,
|
|
268
|
+
temperature: item.temperature ?? request.options?.temperature,
|
|
269
|
+
top_p: item.top_p ?? request.options?.top_p,
|
|
270
|
+
top_k: item.top_k ?? request.options?.top_k,
|
|
271
|
+
stop: item.stop ?? request.options?.stop,
|
|
272
|
+
response_format: item.response_format ?? request.options?.response_format,
|
|
273
|
+
tools: item.tools ?? request.options?.tools,
|
|
274
|
+
tool_choice: item.tool_choice ?? request.options?.tool_choice,
|
|
275
|
+
};
|
|
276
|
+
let result;
|
|
277
|
+
try {
|
|
278
|
+
const response = await this.router.complete(chatRequest);
|
|
279
|
+
result = {
|
|
280
|
+
custom_id: item.custom_id,
|
|
281
|
+
status: 'success',
|
|
282
|
+
response,
|
|
283
|
+
error: null,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
catch (err) {
|
|
287
|
+
const error = err instanceof AnyModelError ? err : new AnyModelError(500, String(err));
|
|
288
|
+
result = {
|
|
289
|
+
custom_id: item.custom_id,
|
|
290
|
+
status: 'error',
|
|
291
|
+
response: null,
|
|
292
|
+
error: { code: error.code, message: error.message },
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
await this.store.appendResult(batchId, result);
|
|
296
|
+
const meta = await this.store.getMeta(batchId);
|
|
297
|
+
if (meta) {
|
|
298
|
+
if (result.status === 'success') {
|
|
299
|
+
meta.completed++;
|
|
300
|
+
}
|
|
301
|
+
else {
|
|
302
|
+
meta.failed++;
|
|
303
|
+
}
|
|
304
|
+
await this.store.updateMeta(meta);
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
for (const item of items) {
|
|
308
|
+
const current = await this.store.getMeta(batchId);
|
|
309
|
+
if (current?.status === 'cancelled')
|
|
310
|
+
break;
|
|
311
|
+
if (active.size >= this.concurrencyLimit) {
|
|
312
|
+
await Promise.race(active);
|
|
313
|
+
}
|
|
314
|
+
const promise = processItem(item).then(() => {
|
|
315
|
+
active.delete(promise);
|
|
316
|
+
});
|
|
317
|
+
active.add(promise);
|
|
318
|
+
}
|
|
319
|
+
await Promise.all(active);
|
|
320
|
+
const finalMeta = await this.store.getMeta(batchId);
|
|
321
|
+
if (finalMeta && finalMeta.status !== 'cancelled') {
|
|
322
|
+
finalMeta.status = finalMeta.failed === finalMeta.total ? 'failed' : 'completed';
|
|
323
|
+
finalMeta.completed_at = new Date().toISOString();
|
|
324
|
+
await this.store.updateMeta(finalMeta);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
//# sourceMappingURL=manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/batch/manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAaxC,MAAM,OAAO,YAAY;IACf,KAAK,CAAa;IAClB,MAAM,CAAS;IACf,gBAAgB,CAAS;IACzB,mBAAmB,CAAS;IAC5B,aAAa,GAAG,IAAI,GAAG,EAAwB,CAAC;IAExD,YAAY,MAAc,EAAE,OAAuE;QACjG,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,WAAW,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,YAAY,IAAI,IAAI,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,YAAoB,EAAE,OAAqB;QAC9D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,KAAa;QACzC,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAA2B;QACtC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,QAAiB,CAAC,CAAC,CAAC,YAAqB,CAAC;QAErE,MAAM,KAAK,GAAgB;YACzB,EAAE;YACF,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,aAAa,EAAE,YAAY;YAC3B,UAAU,EAAE,SAAS;YACrB,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;YAC9B,SAAS,EAAE,CAAC;YACZ,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,GAAG;YACf,YAAY,EAAE,IAAI;YAClB,UAAU,EAAE,IAAI;SACjB,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEpD,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,sBAAsB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,OAA2B,EAC3B,UAA4B,EAAE;QAE9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,EAAU,EAAE,UAA4B,EAAE;QACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,mBAAmB,CAAC;QAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;YACxD,CAAC;YAED,gDAAgD;YAChD,IAAI,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBACnE,MAAM,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;gBACrC,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBACrC,IAAI,CAAC,KAAK;oBAAE,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;YACpE,CAAC;YAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAC9F,OAAO,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,oBAAoB,OAAO,IAAI,CAAC,CAAC;YAC3E,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU;QACzB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAEhD,MAAM,KAAK,GAAsB;YAC/B,mBAAmB,EAAE,CAAC;YACtB,uBAAuB,EAAE,CAAC;YAC1B,cAAc,EAAE,CAAC;SAClB,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACpB,KAAK,CAAC,mBAAmB,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;gBACjE,KAAK,CAAC,uBAAuB,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO;YACP,aAAa,EAAE,KAAK;SACrB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC1C,IAAI,IAAI;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACjE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,0CAA0C;QAC1C,IAAI,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC5D,IAAI,OAAO,IAAI,aAAa,EAAE,eAAe,EAAE,CAAC;gBAC9C,IAAI,CAAC;oBACH,MAAM,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,eAAyB,CAAC,CAAC;gBACrE,CAAC;gBAAC,MAAM,CAAC;oBACP,2BAA2B;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC;QAC3B,KAAK,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC9B,OAAe,EACf,OAA2B,EAC3B,OAAqB;QAErB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;gBACvC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBAC7C,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;YAElB,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,WAAW,CAC7D,KAAK,EACL,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,OAA8C,CACvD,CAAC;YAEF,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,OAAO,EAAE;gBAC1C,eAAe;gBACf,YAAY,EAAE,KAAK,CAAC,aAAa;gBACjC,GAAG,QAAQ;aACZ,CAAC,CAAC;YAEH,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC;YAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;YACxB,KAAK,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAC,OAAe;QACjD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,aAAa,EAAE,eAAe;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5D,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,eAAyB,CAAC,CAAC;YAEhF,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;YAC1C,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YACnC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAE7B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACjG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC7B,KAAK,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAE9C,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBAChE,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,aAAa,CAAC,eAAyB,CAAC,CAAC;wBACvF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;4BAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;wBACjD,CAAC;wBACD,KAAK,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;wBACrE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;oBAClE,CAAC;oBAAC,MAAM,CAAC;wBACP,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;4BAC9B,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;wBAC1B,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC;YAC9B,CAAC;YAED,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;QACzD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,sBAAsB,CAAC,OAAe,EAAE,OAA2B;QAC/E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC;QAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAEnC,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAC;QAExC,MAAM,WAAW,GAAG,KAAK,EAAE,IAAqB,EAAiB,EAAE;YACjE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,OAAO,EAAE,MAAM,KAAK,WAAW;gBAAE,OAAO;YAE5C,MAAM,WAAW,GAA0B;gBACzC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,EAAE,UAAU;gBAC1D,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,OAAO,EAAE,WAAW;gBAC7D,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,KAAK;gBAC3C,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,KAAK;gBAC3C,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,IAAI;gBACxC,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,OAAO,CAAC,OAAO,EAAE,eAAe;gBACzE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,KAAK;gBAC3C,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,OAAO,EAAE,WAAW;aAC9D,CAAC;YAEF,IAAI,MAAuB,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBACzD,MAAM,GAAG;oBACP,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,MAAM,EAAE,SAAS;oBACjB,QAAQ;oBACR,KAAK,EAAE,IAAI;iBACZ,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,GAAG,YAAY,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBACvF,MAAM,GAAG;oBACP,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,MAAM,EAAE,OAAO;oBACf,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE;iBACpD,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;oBAChC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,CAAC;gBACD,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,OAAO,EAAE,MAAM,KAAK,WAAW;gBAAE,MAAM;YAE3C,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACzC,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;YAED,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBAC1C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAE1B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClD,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;YACjF,SAAS,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAClD,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { BatchObject, BatchResultItem } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Disk-based batch persistence store.
|
|
4
|
+
* Uses queued, concurrency-limited IO for high-volume operations.
|
|
5
|
+
* Structure: {dir}/{batchId}/meta.json, requests.jsonl, results.jsonl, provider.json
|
|
6
|
+
*/
|
|
7
|
+
export declare class BatchStore {
|
|
8
|
+
private dir;
|
|
9
|
+
private initialized;
|
|
10
|
+
constructor(dir?: string);
|
|
11
|
+
private init;
|
|
12
|
+
private batchDir;
|
|
13
|
+
/**
|
|
14
|
+
* Create a new batch directory and save initial metadata.
|
|
15
|
+
*/
|
|
16
|
+
create(batch: BatchObject): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Update batch metadata (atomic write).
|
|
19
|
+
*/
|
|
20
|
+
updateMeta(batch: BatchObject): Promise<void>;
|
|
21
|
+
/**
|
|
22
|
+
* Save requests as JSONL.
|
|
23
|
+
*/
|
|
24
|
+
saveRequests(id: string, requests: unknown[]): Promise<void>;
|
|
25
|
+
/**
|
|
26
|
+
* Append a result to results.jsonl.
|
|
27
|
+
*/
|
|
28
|
+
appendResult(id: string, result: BatchResultItem): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Save provider-specific state (e.g., provider batch ID).
|
|
31
|
+
*/
|
|
32
|
+
saveProviderState(id: string, state: Record<string, unknown>): Promise<void>;
|
|
33
|
+
/**
|
|
34
|
+
* Load provider state.
|
|
35
|
+
*/
|
|
36
|
+
loadProviderState(id: string): Promise<Record<string, unknown> | null>;
|
|
37
|
+
/**
|
|
38
|
+
* Get batch metadata.
|
|
39
|
+
*/
|
|
40
|
+
getMeta(id: string): Promise<BatchObject | null>;
|
|
41
|
+
/**
|
|
42
|
+
* Get all results for a batch.
|
|
43
|
+
*/
|
|
44
|
+
getResults(id: string): Promise<BatchResultItem[]>;
|
|
45
|
+
/**
|
|
46
|
+
* List all batch IDs.
|
|
47
|
+
*/
|
|
48
|
+
listBatches(): Promise<string[]>;
|
|
49
|
+
/**
|
|
50
|
+
* Check if a batch exists.
|
|
51
|
+
*/
|
|
52
|
+
exists(id: string): Promise<boolean>;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/batch/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAiBhE;;;;GAIG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAS;gBAEhB,GAAG,CAAC,EAAE,MAAM;YAIV,IAAI;IAMlB,OAAO,CAAC,QAAQ;IAIhB;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/C;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAOnD;;OAEG;IACG,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlE;;OAEG;IACG,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtE;;OAEG;IACG,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlF;;OAEG;IACG,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAM5E;;OAEG;IACG,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAMtD;;OAEG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAWxD;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAUtC;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAG3C"}
|