@probeo/anymodel 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -10
- package/dist/cli.cjs +827 -108
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.ts +0 -2
- package/dist/cli.js +2785 -30
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +848 -110
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +106 -22
- package/dist/index.d.ts +632 -14
- package/dist/index.js +2782 -15
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/batch/index.d.ts +0 -4
- package/dist/batch/index.d.ts.map +0 -1
- package/dist/batch/index.js +0 -3
- package/dist/batch/index.js.map +0 -1
- package/dist/batch/manager.d.ts +0 -72
- package/dist/batch/manager.d.ts.map +0 -1
- package/dist/batch/manager.js +0 -328
- package/dist/batch/manager.js.map +0 -1
- package/dist/batch/store.d.ts +0 -54
- package/dist/batch/store.d.ts.map +0 -1
- package/dist/batch/store.js +0 -109
- package/dist/batch/store.js.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/client.d.ts +0 -42
- package/dist/client.d.ts.map +0 -1
- package/dist/client.js +0 -181
- package/dist/client.js.map +0 -1
- package/dist/config.d.ts +0 -6
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -120
- package/dist/config.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/providers/adapter.d.ts +0 -33
- package/dist/providers/adapter.d.ts.map +0 -1
- package/dist/providers/adapter.js +0 -2
- package/dist/providers/adapter.js.map +0 -1
- package/dist/providers/anthropic-batch.d.ts +0 -3
- package/dist/providers/anthropic-batch.d.ts.map +0 -1
- package/dist/providers/anthropic-batch.js +0 -228
- package/dist/providers/anthropic-batch.js.map +0 -1
- package/dist/providers/anthropic.d.ts +0 -3
- package/dist/providers/anthropic.d.ts.map +0 -1
- package/dist/providers/anthropic.js +0 -358
- package/dist/providers/anthropic.js.map +0 -1
- package/dist/providers/custom.d.ts +0 -8
- package/dist/providers/custom.d.ts.map +0 -1
- package/dist/providers/custom.js +0 -61
- package/dist/providers/custom.js.map +0 -1
- package/dist/providers/google.d.ts +0 -3
- package/dist/providers/google.d.ts.map +0 -1
- package/dist/providers/google.js +0 -331
- package/dist/providers/google.js.map +0 -1
- package/dist/providers/index.d.ts +0 -6
- package/dist/providers/index.d.ts.map +0 -1
- package/dist/providers/index.js +0 -5
- package/dist/providers/index.js.map +0 -1
- package/dist/providers/openai-batch.d.ts +0 -3
- package/dist/providers/openai-batch.d.ts.map +0 -1
- package/dist/providers/openai-batch.js +0 -208
- package/dist/providers/openai-batch.js.map +0 -1
- package/dist/providers/openai.d.ts +0 -3
- package/dist/providers/openai.d.ts.map +0 -1
- package/dist/providers/openai.js +0 -221
- package/dist/providers/openai.js.map +0 -1
- package/dist/providers/registry.d.ts +0 -10
- package/dist/providers/registry.d.ts.map +0 -1
- package/dist/providers/registry.js +0 -27
- package/dist/providers/registry.js.map +0 -1
- package/dist/router.d.ts +0 -29
- package/dist/router.d.ts.map +0 -1
- package/dist/router.js +0 -212
- package/dist/router.js.map +0 -1
- package/dist/server.d.ts +0 -10
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js +0 -149
- package/dist/server.js.map +0 -1
- package/dist/types.d.ts +0 -283
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -21
- package/dist/types.js.map +0 -1
- package/dist/utils/fs-io.d.ts +0 -40
- package/dist/utils/fs-io.d.ts.map +0 -1
- package/dist/utils/fs-io.js +0 -203
- package/dist/utils/fs-io.js.map +0 -1
- package/dist/utils/generation-stats.d.ts +0 -25
- package/dist/utils/generation-stats.d.ts.map +0 -1
- package/dist/utils/generation-stats.js +0 -46
- package/dist/utils/generation-stats.js.map +0 -1
- package/dist/utils/id.d.ts +0 -2
- package/dist/utils/id.d.ts.map +0 -1
- package/dist/utils/id.js +0 -6
- package/dist/utils/id.js.map +0 -1
- package/dist/utils/model-parser.d.ts +0 -6
- package/dist/utils/model-parser.d.ts.map +0 -1
- package/dist/utils/model-parser.js +0 -21
- package/dist/utils/model-parser.js.map +0 -1
- package/dist/utils/rate-limiter.d.ts +0 -36
- package/dist/utils/rate-limiter.d.ts.map +0 -1
- package/dist/utils/rate-limiter.js +0 -80
- package/dist/utils/rate-limiter.js.map +0 -1
- package/dist/utils/retry.d.ts +0 -7
- package/dist/utils/retry.d.ts.map +0 -1
- package/dist/utils/retry.js +0 -54
- package/dist/utils/retry.js.map +0 -1
- package/dist/utils/transforms.d.ts +0 -7
- package/dist/utils/transforms.d.ts.map +0 -1
- package/dist/utils/transforms.js +0 -66
- package/dist/utils/transforms.js.map +0 -1
- package/dist/utils/validate.d.ts +0 -3
- package/dist/utils/validate.d.ts.map +0 -1
- package/dist/utils/validate.js +0 -31
- package/dist/utils/validate.js.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
2
3
|
var __defProp = Object.defineProperty;
|
|
3
4
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
5
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
8
|
var __export = (target, all) => {
|
|
7
9
|
for (var name in all)
|
|
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
15
17
|
}
|
|
16
18
|
return to;
|
|
17
19
|
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
18
28
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
29
|
|
|
20
30
|
// src/index.ts
|
|
@@ -25,9 +35,20 @@ __export(src_exports, {
|
|
|
25
35
|
BatchManager: () => BatchManager,
|
|
26
36
|
BatchStore: () => BatchStore,
|
|
27
37
|
GenerationStatsStore: () => GenerationStatsStore,
|
|
38
|
+
appendFileQueued: () => appendFileQueued,
|
|
39
|
+
configureFsIO: () => configureFsIO,
|
|
40
|
+
createAnthropicBatchAdapter: () => createAnthropicBatchAdapter,
|
|
28
41
|
createAnyModelServer: () => createAnyModelServer,
|
|
42
|
+
createOpenAIBatchAdapter: () => createOpenAIBatchAdapter,
|
|
43
|
+
ensureDir: () => ensureDir,
|
|
44
|
+
getFsQueueStatus: () => getFsQueueStatus,
|
|
45
|
+
joinPath: () => joinPath,
|
|
46
|
+
readFileQueued: () => readFileQueued,
|
|
29
47
|
resolveConfig: () => resolveConfig,
|
|
30
|
-
startServer: () => startServer
|
|
48
|
+
startServer: () => startServer,
|
|
49
|
+
waitForFsQueuesIdle: () => waitForFsQueuesIdle,
|
|
50
|
+
writeFileFlushedQueued: () => writeFileFlushedQueued,
|
|
51
|
+
writeFileQueued: () => writeFileQueued
|
|
31
52
|
});
|
|
32
53
|
module.exports = __toCommonJS(src_exports);
|
|
33
54
|
|
|
@@ -175,7 +196,7 @@ async function withRetry(fn, options = {}) {
|
|
|
175
196
|
throw error;
|
|
176
197
|
}
|
|
177
198
|
const delay = computeDelay(attempt, opts, error);
|
|
178
|
-
await new Promise((
|
|
199
|
+
await new Promise((resolve2) => setTimeout(resolve2, delay));
|
|
179
200
|
}
|
|
180
201
|
}
|
|
181
202
|
throw lastError;
|
|
@@ -534,8 +555,8 @@ var SUPPORTED_PARAMS = /* @__PURE__ */ new Set([
|
|
|
534
555
|
]);
|
|
535
556
|
function createOpenAIAdapter(apiKey, baseURL) {
|
|
536
557
|
const base = baseURL || OPENAI_API_BASE;
|
|
537
|
-
async function makeRequest(
|
|
538
|
-
const res = await fetch(`${base}${
|
|
558
|
+
async function makeRequest(path2, body, method = "POST") {
|
|
559
|
+
const res = await fetch(`${base}${path2}`, {
|
|
539
560
|
method,
|
|
540
561
|
headers: {
|
|
541
562
|
"Content-Type": "application/json",
|
|
@@ -650,7 +671,19 @@ function createOpenAIAdapter(apiKey, baseURL) {
|
|
|
650
671
|
async listModels() {
|
|
651
672
|
const res = await makeRequest("/models", void 0, "GET");
|
|
652
673
|
const data = await res.json();
|
|
653
|
-
return (data.data || []).filter((m) =>
|
|
674
|
+
return (data.data || []).filter((m) => {
|
|
675
|
+
const id = m.id;
|
|
676
|
+
if (id.includes("embedding")) return false;
|
|
677
|
+
if (id.includes("whisper")) return false;
|
|
678
|
+
if (id.includes("tts")) return false;
|
|
679
|
+
if (id.includes("dall-e")) return false;
|
|
680
|
+
if (id.includes("davinci")) return false;
|
|
681
|
+
if (id.includes("babbage")) return false;
|
|
682
|
+
if (id.includes("moderation")) return false;
|
|
683
|
+
if (id.includes("realtime")) return false;
|
|
684
|
+
if (id.startsWith("ft:")) return false;
|
|
685
|
+
return id.startsWith("gpt-") || id.startsWith("o1") || id.startsWith("o3") || id.startsWith("o4") || id.startsWith("chatgpt-");
|
|
686
|
+
}).map((m) => ({
|
|
654
687
|
id: `openai/${m.id}`,
|
|
655
688
|
name: m.id,
|
|
656
689
|
created: m.created,
|
|
@@ -674,6 +707,9 @@ function createOpenAIAdapter(apiKey, baseURL) {
|
|
|
674
707
|
supportsParameter(param) {
|
|
675
708
|
return SUPPORTED_PARAMS.has(param);
|
|
676
709
|
},
|
|
710
|
+
supportsBatch() {
|
|
711
|
+
return true;
|
|
712
|
+
},
|
|
677
713
|
async sendRequest(request) {
|
|
678
714
|
const body = buildRequestBody(request);
|
|
679
715
|
const res = await makeRequest("/chat/completions", body);
|
|
@@ -717,14 +753,19 @@ var SUPPORTED_PARAMS2 = /* @__PURE__ */ new Set([
|
|
|
717
753
|
"response_format"
|
|
718
754
|
]);
|
|
719
755
|
var FALLBACK_MODELS = [
|
|
720
|
-
|
|
756
|
+
// Claude 4.6
|
|
757
|
+
{ id: "anthropic/claude-opus-4-6", name: "Claude Opus 4.6", created: 0, description: "Most capable model", context_length: 2e5, pricing: { prompt: "0.000015", completion: "0.000075" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image"], output_modalities: ["text"], tokenizer: "claude" }, top_provider: { context_length: 2e5, max_completion_tokens: 32768, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS2) },
|
|
721
758
|
{ id: "anthropic/claude-sonnet-4-6", name: "Claude Sonnet 4.6", created: 0, description: "Best balance of speed and capability", context_length: 2e5, pricing: { prompt: "0.000003", completion: "0.000015" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image"], output_modalities: ["text"], tokenizer: "claude" }, top_provider: { context_length: 2e5, max_completion_tokens: 16384, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS2) },
|
|
759
|
+
// Claude 4.5
|
|
722
760
|
{ id: "anthropic/claude-sonnet-4-5-20251022", name: "Claude Sonnet 4.5", created: 0, description: "Previous generation balanced model", context_length: 2e5, pricing: { prompt: "0.000003", completion: "0.000015" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image"], output_modalities: ["text"], tokenizer: "claude" }, top_provider: { context_length: 2e5, max_completion_tokens: 16384, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS2) },
|
|
723
|
-
{ id: "anthropic/claude-haiku-4-5", name: "Claude Haiku 4.5", created: 0, description: "
|
|
761
|
+
{ id: "anthropic/claude-haiku-4-5", name: "Claude Haiku 4.5", created: 0, description: "Fast and compact", context_length: 2e5, pricing: { prompt: "0.000001", completion: "0.000005" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image"], output_modalities: ["text"], tokenizer: "claude" }, top_provider: { context_length: 2e5, max_completion_tokens: 8192, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS2) },
|
|
762
|
+
// Claude 3.5
|
|
763
|
+
{ id: "anthropic/claude-3-5-sonnet-20241022", name: "Claude 3.5 Sonnet", created: 0, description: "Legacy balanced model", context_length: 2e5, pricing: { prompt: "0.000003", completion: "0.000015" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image"], output_modalities: ["text"], tokenizer: "claude" }, top_provider: { context_length: 2e5, max_completion_tokens: 8192, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS2) },
|
|
764
|
+
{ id: "anthropic/claude-3-5-haiku-20241022", name: "Claude 3.5 Haiku", created: 0, description: "Legacy fast model", context_length: 2e5, pricing: { prompt: "0.0000008", completion: "0.000004" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image"], output_modalities: ["text"], tokenizer: "claude" }, top_provider: { context_length: 2e5, max_completion_tokens: 8192, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS2) }
|
|
724
765
|
];
|
|
725
766
|
function createAnthropicAdapter(apiKey) {
|
|
726
|
-
async function makeRequest(
|
|
727
|
-
const res = await fetch(`${ANTHROPIC_API_BASE}${
|
|
767
|
+
async function makeRequest(path2, body, stream = false) {
|
|
768
|
+
const res = await fetch(`${ANTHROPIC_API_BASE}${path2}`, {
|
|
728
769
|
method: "POST",
|
|
729
770
|
headers: {
|
|
730
771
|
"Content-Type": "application/json",
|
|
@@ -1018,6 +1059,9 @@ ${body.system}` : jsonInstruction;
|
|
|
1018
1059
|
supportsParameter(param) {
|
|
1019
1060
|
return SUPPORTED_PARAMS2.has(param);
|
|
1020
1061
|
},
|
|
1062
|
+
supportsBatch() {
|
|
1063
|
+
return true;
|
|
1064
|
+
},
|
|
1021
1065
|
async sendRequest(request) {
|
|
1022
1066
|
const body = translateRequest(request);
|
|
1023
1067
|
const res = await makeRequest("/messages", body);
|
|
@@ -1052,8 +1096,15 @@ var SUPPORTED_PARAMS3 = /* @__PURE__ */ new Set([
|
|
|
1052
1096
|
"response_format"
|
|
1053
1097
|
]);
|
|
1054
1098
|
var FALLBACK_MODELS2 = [
|
|
1099
|
+
// Gemini 2.5
|
|
1055
1100
|
{ id: "google/gemini-2.5-pro", name: "Gemini 2.5 Pro", created: 0, description: "Most capable Gemini model", context_length: 1048576, pricing: { prompt: "0.00000125", completion: "0.000005" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image", "video", "audio"], output_modalities: ["text"], tokenizer: "gemini" }, top_provider: { context_length: 1048576, max_completion_tokens: 65536, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS3) },
|
|
1056
|
-
{ id: "google/gemini-2.5-flash", name: "Gemini 2.5 Flash", created: 0, description: "Fast and efficient", context_length: 1048576, pricing: { prompt: "0.00000015", completion: "0.0000006" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image", "video", "audio"], output_modalities: ["text"], tokenizer: "gemini" }, top_provider: { context_length: 1048576, max_completion_tokens: 65536, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS3) }
|
|
1101
|
+
{ id: "google/gemini-2.5-flash", name: "Gemini 2.5 Flash", created: 0, description: "Fast and efficient", context_length: 1048576, pricing: { prompt: "0.00000015", completion: "0.0000006" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image", "video", "audio"], output_modalities: ["text"], tokenizer: "gemini" }, top_provider: { context_length: 1048576, max_completion_tokens: 65536, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS3) },
|
|
1102
|
+
// Gemini 2.0
|
|
1103
|
+
{ id: "google/gemini-2.0-flash", name: "Gemini 2.0 Flash", created: 0, description: "Fast multimodal model", context_length: 1048576, pricing: { prompt: "0.0000001", completion: "0.0000004" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image", "video", "audio"], output_modalities: ["text"], tokenizer: "gemini" }, top_provider: { context_length: 1048576, max_completion_tokens: 65536, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS3) },
|
|
1104
|
+
{ id: "google/gemini-2.0-flash-lite", name: "Gemini 2.0 Flash Lite", created: 0, description: "Lightweight and fast", context_length: 1048576, pricing: { prompt: "0.00000005", completion: "0.0000002" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image", "video", "audio"], output_modalities: ["text"], tokenizer: "gemini" }, top_provider: { context_length: 1048576, max_completion_tokens: 65536, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS3) },
|
|
1105
|
+
// Gemini 1.5
|
|
1106
|
+
{ id: "google/gemini-1.5-pro", name: "Gemini 1.5 Pro", created: 0, description: "Previous generation pro model", context_length: 2097152, pricing: { prompt: "0.00000125", completion: "0.000005" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image", "video", "audio"], output_modalities: ["text"], tokenizer: "gemini" }, top_provider: { context_length: 2097152, max_completion_tokens: 8192, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS3) },
|
|
1107
|
+
{ id: "google/gemini-1.5-flash", name: "Gemini 1.5 Flash", created: 0, description: "Previous generation flash model", context_length: 1048576, pricing: { prompt: "0.000000075", completion: "0.0000003" }, architecture: { modality: "text+image->text", input_modalities: ["text", "image", "video", "audio"], output_modalities: ["text"], tokenizer: "gemini" }, top_provider: { context_length: 1048576, max_completion_tokens: 8192, is_moderated: false }, supported_parameters: Array.from(SUPPORTED_PARAMS3) }
|
|
1057
1108
|
];
|
|
1058
1109
|
function createGoogleAdapter(apiKey) {
|
|
1059
1110
|
function getModelEndpoint(model, stream) {
|
|
@@ -1290,6 +1341,9 @@ function createGoogleAdapter(apiKey) {
|
|
|
1290
1341
|
supportsParameter(param) {
|
|
1291
1342
|
return SUPPORTED_PARAMS3.has(param);
|
|
1292
1343
|
},
|
|
1344
|
+
supportsBatch() {
|
|
1345
|
+
return false;
|
|
1346
|
+
},
|
|
1293
1347
|
async sendRequest(request) {
|
|
1294
1348
|
const body = translateRequest(request);
|
|
1295
1349
|
const url = getModelEndpoint(request.model, false);
|
|
@@ -1348,6 +1402,9 @@ function createCustomAdapter(name, config) {
|
|
|
1348
1402
|
return {
|
|
1349
1403
|
...openaiAdapter,
|
|
1350
1404
|
name,
|
|
1405
|
+
supportsBatch() {
|
|
1406
|
+
return false;
|
|
1407
|
+
},
|
|
1351
1408
|
async listModels() {
|
|
1352
1409
|
if (config.models && config.models.length > 0) {
|
|
1353
1410
|
return config.models.map((modelId) => ({
|
|
@@ -1413,10 +1470,10 @@ function interpolateDeep(obj) {
|
|
|
1413
1470
|
}
|
|
1414
1471
|
return obj;
|
|
1415
1472
|
}
|
|
1416
|
-
function loadJsonFile(
|
|
1417
|
-
if (!(0, import_node_fs.existsSync)(
|
|
1473
|
+
function loadJsonFile(path2) {
|
|
1474
|
+
if (!(0, import_node_fs.existsSync)(path2)) return null;
|
|
1418
1475
|
try {
|
|
1419
|
-
const raw = (0, import_node_fs.readFileSync)(
|
|
1476
|
+
const raw = (0, import_node_fs.readFileSync)(path2, "utf-8");
|
|
1420
1477
|
const parsed = JSON.parse(raw);
|
|
1421
1478
|
return interpolateDeep(parsed);
|
|
1422
1479
|
} catch {
|
|
@@ -1517,93 +1574,237 @@ var GenerationStatsStore = class {
|
|
|
1517
1574
|
}
|
|
1518
1575
|
};
|
|
1519
1576
|
|
|
1520
|
-
// src/
|
|
1577
|
+
// src/utils/fs-io.ts
|
|
1578
|
+
var import_promises = require("fs/promises");
|
|
1521
1579
|
var import_node_fs2 = require("fs");
|
|
1522
|
-
var import_node_path2 = require("path");
|
|
1523
|
-
var
|
|
1524
|
-
var
|
|
1580
|
+
var import_node_path2 = __toESM(require("path"), 1);
|
|
1581
|
+
var import_p_queue = __toESM(require("p-queue"), 1);
|
|
1582
|
+
var writeQueue = new import_p_queue.default({ concurrency: 10 });
|
|
1583
|
+
var readQueue = new import_p_queue.default({ concurrency: 20 });
|
|
1584
|
+
function configureFsIO(options) {
|
|
1585
|
+
if (options.readConcurrency !== void 0) {
|
|
1586
|
+
readQueue.concurrency = options.readConcurrency;
|
|
1587
|
+
}
|
|
1588
|
+
if (options.writeConcurrency !== void 0) {
|
|
1589
|
+
writeQueue.concurrency = options.writeConcurrency;
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
var ensuredDirs = /* @__PURE__ */ new Set();
|
|
1593
|
+
var joinPathCache = /* @__PURE__ */ new Map();
|
|
1594
|
+
var dirnameCache = /* @__PURE__ */ new Map();
|
|
1595
|
+
var resolvePathCache = /* @__PURE__ */ new Map();
|
|
1596
|
+
async function ensureDir(dir) {
|
|
1597
|
+
if (!dir) return;
|
|
1598
|
+
if (ensuredDirs.has(dir)) return;
|
|
1599
|
+
await (0, import_promises.mkdir)(dir, { recursive: true });
|
|
1600
|
+
ensuredDirs.add(dir);
|
|
1601
|
+
}
|
|
1602
|
+
async function readFileQueued(filePath, encoding = "utf8") {
|
|
1603
|
+
return readQueue.add(async () => {
|
|
1604
|
+
return (0, import_promises.readFile)(filePath, encoding);
|
|
1605
|
+
});
|
|
1606
|
+
}
|
|
1607
|
+
async function readJsonQueued(filePath) {
|
|
1608
|
+
const raw = await readFileQueued(filePath, "utf8");
|
|
1609
|
+
return JSON.parse(raw);
|
|
1610
|
+
}
|
|
1611
|
+
async function readDirQueued(dirPath) {
|
|
1612
|
+
return readQueue.add(async () => {
|
|
1613
|
+
return (0, import_promises.readdir)(dirPath, { withFileTypes: true });
|
|
1614
|
+
});
|
|
1615
|
+
}
|
|
1616
|
+
async function pathExistsQueued(p) {
|
|
1617
|
+
return readQueue.add(async () => {
|
|
1618
|
+
try {
|
|
1619
|
+
await (0, import_promises.stat)(p);
|
|
1620
|
+
return true;
|
|
1621
|
+
} catch {
|
|
1622
|
+
return false;
|
|
1623
|
+
}
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1626
|
+
async function fileExistsQueued(filePath) {
|
|
1627
|
+
return readQueue.add(async () => {
|
|
1628
|
+
try {
|
|
1629
|
+
const s = await (0, import_promises.stat)(filePath);
|
|
1630
|
+
return s.isFile();
|
|
1631
|
+
} catch {
|
|
1632
|
+
return false;
|
|
1633
|
+
}
|
|
1634
|
+
});
|
|
1635
|
+
}
|
|
1636
|
+
async function writeFileQueued(filePath, data) {
|
|
1637
|
+
await writeQueue.add(async () => {
|
|
1638
|
+
const dir = dirnameOf(filePath);
|
|
1639
|
+
await ensureDir(dir);
|
|
1640
|
+
await (0, import_promises.writeFile)(filePath, data);
|
|
1641
|
+
});
|
|
1642
|
+
}
|
|
1643
|
+
async function appendFileQueued(filePath, data) {
|
|
1644
|
+
await writeQueue.add(async () => {
|
|
1645
|
+
const dir = dirnameOf(filePath);
|
|
1646
|
+
await ensureDir(dir);
|
|
1647
|
+
await (0, import_promises.writeFile)(filePath, data, { flag: "a" });
|
|
1648
|
+
});
|
|
1649
|
+
}
|
|
1650
|
+
async function writeFileFlushedQueued(filePath, data) {
|
|
1651
|
+
await writeQueue.add(async () => {
|
|
1652
|
+
const dir = dirnameOf(filePath);
|
|
1653
|
+
await ensureDir(dir);
|
|
1654
|
+
const tmpPath = joinPath(
|
|
1655
|
+
dir,
|
|
1656
|
+
`.${import_node_path2.default.basename(filePath)}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`
|
|
1657
|
+
);
|
|
1658
|
+
const fh = await (0, import_promises.open)(tmpPath, "w");
|
|
1659
|
+
try {
|
|
1660
|
+
await fh.writeFile(data);
|
|
1661
|
+
await fh.sync();
|
|
1662
|
+
} finally {
|
|
1663
|
+
await fh.close();
|
|
1664
|
+
}
|
|
1665
|
+
await (0, import_promises.rename)(tmpPath, filePath);
|
|
1666
|
+
try {
|
|
1667
|
+
const dh = await (0, import_promises.open)(dir, "r");
|
|
1668
|
+
try {
|
|
1669
|
+
await dh.sync();
|
|
1670
|
+
} finally {
|
|
1671
|
+
await dh.close();
|
|
1672
|
+
}
|
|
1673
|
+
} catch {
|
|
1674
|
+
}
|
|
1675
|
+
});
|
|
1676
|
+
}
|
|
1677
|
+
function joinPath(...segments) {
|
|
1678
|
+
const key = segments.join("\0");
|
|
1679
|
+
const cached = joinPathCache.get(key);
|
|
1680
|
+
if (cached !== void 0) return cached;
|
|
1681
|
+
const out = import_node_path2.default.join(...segments);
|
|
1682
|
+
joinPathCache.set(key, out);
|
|
1683
|
+
return out;
|
|
1684
|
+
}
|
|
1685
|
+
function dirnameOf(p) {
|
|
1686
|
+
const cached = dirnameCache.get(p);
|
|
1687
|
+
if (cached !== void 0) return cached;
|
|
1688
|
+
const out = import_node_path2.default.dirname(p);
|
|
1689
|
+
dirnameCache.set(p, out);
|
|
1690
|
+
return out;
|
|
1691
|
+
}
|
|
1692
|
+
function resolvePath(...segments) {
|
|
1693
|
+
const key = segments.join("\0");
|
|
1694
|
+
const cached = resolvePathCache.get(key);
|
|
1695
|
+
if (cached !== void 0) return cached;
|
|
1696
|
+
const out = import_node_path2.default.resolve(...segments);
|
|
1697
|
+
resolvePathCache.set(key, out);
|
|
1698
|
+
return out;
|
|
1699
|
+
}
|
|
1700
|
+
function getFsQueueStatus() {
|
|
1701
|
+
return {
|
|
1702
|
+
read: { size: readQueue.size, pending: readQueue.pending },
|
|
1703
|
+
write: { size: writeQueue.size, pending: writeQueue.pending }
|
|
1704
|
+
};
|
|
1705
|
+
}
|
|
1706
|
+
async function waitForFsQueuesIdle() {
|
|
1707
|
+
await Promise.all([writeQueue.onIdle(), readQueue.onIdle()]);
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1710
|
+
// src/batch/store.ts
|
|
1711
|
+
var DEFAULT_BATCH_DIR = joinPath(process.cwd(), ".anymodel", "batches");
|
|
1525
1712
|
var BatchStore = class {
|
|
1526
1713
|
dir;
|
|
1714
|
+
initialized = false;
|
|
1527
1715
|
constructor(dir) {
|
|
1528
|
-
this.dir = (
|
|
1529
|
-
|
|
1716
|
+
this.dir = resolvePath(dir || DEFAULT_BATCH_DIR);
|
|
1717
|
+
}
|
|
1718
|
+
async init() {
|
|
1719
|
+
if (this.initialized) return;
|
|
1720
|
+
await ensureDir(this.dir);
|
|
1721
|
+
this.initialized = true;
|
|
1530
1722
|
}
|
|
1531
1723
|
batchDir(id) {
|
|
1532
|
-
return (
|
|
1724
|
+
return joinPath(this.dir, id);
|
|
1533
1725
|
}
|
|
1534
1726
|
/**
|
|
1535
1727
|
* Create a new batch directory and save initial metadata.
|
|
1536
1728
|
*/
|
|
1537
|
-
create(batch) {
|
|
1729
|
+
async create(batch) {
|
|
1730
|
+
await this.init();
|
|
1538
1731
|
const dir = this.batchDir(batch.id);
|
|
1539
|
-
|
|
1540
|
-
|
|
1732
|
+
await ensureDir(dir);
|
|
1733
|
+
await writeFileFlushedQueued(joinPath(dir, "meta.json"), JSON.stringify(batch, null, 2));
|
|
1541
1734
|
}
|
|
1542
1735
|
/**
|
|
1543
|
-
* Update batch metadata.
|
|
1736
|
+
* Update batch metadata (atomic write).
|
|
1544
1737
|
*/
|
|
1545
|
-
updateMeta(batch) {
|
|
1546
|
-
|
|
1547
|
-
|
|
1738
|
+
async updateMeta(batch) {
|
|
1739
|
+
await writeFileFlushedQueued(
|
|
1740
|
+
joinPath(this.batchDir(batch.id), "meta.json"),
|
|
1741
|
+
JSON.stringify(batch, null, 2)
|
|
1742
|
+
);
|
|
1548
1743
|
}
|
|
1549
1744
|
/**
|
|
1550
1745
|
* Save requests as JSONL.
|
|
1551
1746
|
*/
|
|
1552
|
-
saveRequests(id, requests) {
|
|
1553
|
-
const dir = this.batchDir(id);
|
|
1747
|
+
async saveRequests(id, requests) {
|
|
1554
1748
|
const lines = requests.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
1555
|
-
|
|
1749
|
+
await writeFileQueued(joinPath(this.batchDir(id), "requests.jsonl"), lines);
|
|
1556
1750
|
}
|
|
1557
1751
|
/**
|
|
1558
1752
|
* Append a result to results.jsonl.
|
|
1559
1753
|
*/
|
|
1560
|
-
appendResult(id, result) {
|
|
1561
|
-
|
|
1562
|
-
|
|
1754
|
+
async appendResult(id, result) {
|
|
1755
|
+
await appendFileQueued(
|
|
1756
|
+
joinPath(this.batchDir(id), "results.jsonl"),
|
|
1757
|
+
JSON.stringify(result) + "\n"
|
|
1758
|
+
);
|
|
1563
1759
|
}
|
|
1564
1760
|
/**
|
|
1565
1761
|
* Save provider-specific state (e.g., provider batch ID).
|
|
1566
1762
|
*/
|
|
1567
|
-
saveProviderState(id, state) {
|
|
1568
|
-
|
|
1569
|
-
|
|
1763
|
+
async saveProviderState(id, state) {
|
|
1764
|
+
await writeFileFlushedQueued(
|
|
1765
|
+
joinPath(this.batchDir(id), "provider.json"),
|
|
1766
|
+
JSON.stringify(state, null, 2)
|
|
1767
|
+
);
|
|
1570
1768
|
}
|
|
1571
1769
|
/**
|
|
1572
1770
|
* Load provider state.
|
|
1573
1771
|
*/
|
|
1574
|
-
loadProviderState(id) {
|
|
1575
|
-
const
|
|
1576
|
-
if (!
|
|
1577
|
-
return
|
|
1772
|
+
async loadProviderState(id) {
|
|
1773
|
+
const p = joinPath(this.batchDir(id), "provider.json");
|
|
1774
|
+
if (!await fileExistsQueued(p)) return null;
|
|
1775
|
+
return readJsonQueued(p);
|
|
1578
1776
|
}
|
|
1579
1777
|
/**
|
|
1580
1778
|
* Get batch metadata.
|
|
1581
1779
|
*/
|
|
1582
|
-
getMeta(id) {
|
|
1583
|
-
const
|
|
1584
|
-
if (!
|
|
1585
|
-
return
|
|
1780
|
+
async getMeta(id) {
|
|
1781
|
+
const p = joinPath(this.batchDir(id), "meta.json");
|
|
1782
|
+
if (!await fileExistsQueued(p)) return null;
|
|
1783
|
+
return readJsonQueued(p);
|
|
1586
1784
|
}
|
|
1587
1785
|
/**
|
|
1588
1786
|
* Get all results for a batch.
|
|
1589
1787
|
*/
|
|
1590
|
-
getResults(id) {
|
|
1591
|
-
const
|
|
1592
|
-
if (!
|
|
1593
|
-
|
|
1788
|
+
async getResults(id) {
|
|
1789
|
+
const p = joinPath(this.batchDir(id), "results.jsonl");
|
|
1790
|
+
if (!await fileExistsQueued(p)) return [];
|
|
1791
|
+
const raw = await readFileQueued(p, "utf8");
|
|
1792
|
+
return raw.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
1594
1793
|
}
|
|
1595
1794
|
/**
|
|
1596
1795
|
* List all batch IDs.
|
|
1597
1796
|
*/
|
|
1598
|
-
listBatches() {
|
|
1599
|
-
|
|
1600
|
-
|
|
1797
|
+
async listBatches() {
|
|
1798
|
+
await this.init();
|
|
1799
|
+
if (!await pathExistsQueued(this.dir)) return [];
|
|
1800
|
+
const entries = await readDirQueued(this.dir);
|
|
1801
|
+
return entries.filter((d) => d.isDirectory()).map((d) => d.name).sort();
|
|
1601
1802
|
}
|
|
1602
1803
|
/**
|
|
1603
1804
|
* Check if a batch exists.
|
|
1604
1805
|
*/
|
|
1605
|
-
exists(id) {
|
|
1606
|
-
return (
|
|
1806
|
+
async exists(id) {
|
|
1807
|
+
return fileExistsQueued(joinPath(this.batchDir(id), "meta.json"));
|
|
1607
1808
|
}
|
|
1608
1809
|
};
|
|
1609
1810
|
|
|
@@ -1612,10 +1813,27 @@ var BatchManager = class {
|
|
|
1612
1813
|
store;
|
|
1613
1814
|
router;
|
|
1614
1815
|
concurrencyLimit;
|
|
1816
|
+
defaultPollInterval;
|
|
1817
|
+
batchAdapters = /* @__PURE__ */ new Map();
|
|
1615
1818
|
constructor(router, options) {
|
|
1616
1819
|
this.store = new BatchStore(options?.dir);
|
|
1617
1820
|
this.router = router;
|
|
1618
1821
|
this.concurrencyLimit = options?.concurrency ?? 5;
|
|
1822
|
+
this.defaultPollInterval = options?.pollInterval ?? 5e3;
|
|
1823
|
+
}
|
|
1824
|
+
/**
|
|
1825
|
+
* Register a native batch adapter for a provider.
|
|
1826
|
+
*/
|
|
1827
|
+
registerBatchAdapter(providerName, adapter) {
|
|
1828
|
+
this.batchAdapters.set(providerName, adapter);
|
|
1829
|
+
}
|
|
1830
|
+
/**
|
|
1831
|
+
* Check if a provider has native batch support.
|
|
1832
|
+
*/
|
|
1833
|
+
getNativeBatchAdapter(model) {
|
|
1834
|
+
const providerName = model.split("/")[0];
|
|
1835
|
+
const adapter = this.batchAdapters.get(providerName);
|
|
1836
|
+
return adapter ? { adapter, providerName } : null;
|
|
1619
1837
|
}
|
|
1620
1838
|
/**
|
|
1621
1839
|
* Create a batch and return immediately (no polling).
|
|
@@ -1623,13 +1841,16 @@ var BatchManager = class {
|
|
|
1623
1841
|
async create(request) {
|
|
1624
1842
|
const id = generateId("batch");
|
|
1625
1843
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1844
|
+
const providerName = request.model.split("/")[0] || "unknown";
|
|
1845
|
+
const native = this.getNativeBatchAdapter(request.model);
|
|
1846
|
+
const batchMode = native ? "native" : "concurrent";
|
|
1626
1847
|
const batch = {
|
|
1627
1848
|
id,
|
|
1628
1849
|
object: "batch",
|
|
1629
1850
|
status: "pending",
|
|
1630
1851
|
model: request.model,
|
|
1631
|
-
provider_name:
|
|
1632
|
-
batch_mode:
|
|
1852
|
+
provider_name: providerName,
|
|
1853
|
+
batch_mode: batchMode,
|
|
1633
1854
|
total: request.requests.length,
|
|
1634
1855
|
completed: 0,
|
|
1635
1856
|
failed: 0,
|
|
@@ -1637,10 +1858,15 @@ var BatchManager = class {
|
|
|
1637
1858
|
completed_at: null,
|
|
1638
1859
|
expires_at: null
|
|
1639
1860
|
};
|
|
1640
|
-
this.store.create(batch);
|
|
1641
|
-
this.store.saveRequests(id, request.requests);
|
|
1642
|
-
|
|
1643
|
-
|
|
1861
|
+
await this.store.create(batch);
|
|
1862
|
+
await this.store.saveRequests(id, request.requests);
|
|
1863
|
+
if (native) {
|
|
1864
|
+
this.processNativeBatch(id, request, native.adapter).catch(() => {
|
|
1865
|
+
});
|
|
1866
|
+
} else {
|
|
1867
|
+
this.processConcurrentBatch(id, request).catch(() => {
|
|
1868
|
+
});
|
|
1869
|
+
}
|
|
1644
1870
|
return batch;
|
|
1645
1871
|
}
|
|
1646
1872
|
/**
|
|
@@ -1654,14 +1880,19 @@ var BatchManager = class {
|
|
|
1654
1880
|
* Poll an existing batch until completion.
|
|
1655
1881
|
*/
|
|
1656
1882
|
async poll(id, options = {}) {
|
|
1657
|
-
const interval = options.interval ??
|
|
1883
|
+
const interval = options.interval ?? this.defaultPollInterval;
|
|
1658
1884
|
const timeout = options.timeout ?? 0;
|
|
1659
1885
|
const startTime = Date.now();
|
|
1660
1886
|
while (true) {
|
|
1661
|
-
|
|
1887
|
+
let batch = await this.store.getMeta(id);
|
|
1662
1888
|
if (!batch) {
|
|
1663
1889
|
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
1664
1890
|
}
|
|
1891
|
+
if (batch.batch_mode === "native" && batch.status === "processing") {
|
|
1892
|
+
await this.syncNativeBatchStatus(id);
|
|
1893
|
+
batch = await this.store.getMeta(id);
|
|
1894
|
+
if (!batch) throw new AnyModelError(404, `Batch ${id} not found`);
|
|
1895
|
+
}
|
|
1665
1896
|
if (options.onProgress) {
|
|
1666
1897
|
options.onProgress(batch);
|
|
1667
1898
|
}
|
|
@@ -1671,24 +1902,24 @@ var BatchManager = class {
|
|
|
1671
1902
|
if (timeout > 0 && Date.now() - startTime > timeout) {
|
|
1672
1903
|
throw new AnyModelError(408, `Batch ${id} timed out after ${timeout}ms`);
|
|
1673
1904
|
}
|
|
1674
|
-
await new Promise((
|
|
1905
|
+
await new Promise((resolve2) => setTimeout(resolve2, interval));
|
|
1675
1906
|
}
|
|
1676
1907
|
}
|
|
1677
1908
|
/**
|
|
1678
1909
|
* Get the current status of a batch.
|
|
1679
1910
|
*/
|
|
1680
|
-
get(id) {
|
|
1911
|
+
async get(id) {
|
|
1681
1912
|
return this.store.getMeta(id);
|
|
1682
1913
|
}
|
|
1683
1914
|
/**
|
|
1684
1915
|
* Get results for a completed batch.
|
|
1685
1916
|
*/
|
|
1686
|
-
getResults(id) {
|
|
1687
|
-
const batch = this.store.getMeta(id);
|
|
1917
|
+
async getResults(id) {
|
|
1918
|
+
const batch = await this.store.getMeta(id);
|
|
1688
1919
|
if (!batch) {
|
|
1689
1920
|
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
1690
1921
|
}
|
|
1691
|
-
const results = this.store.getResults(id);
|
|
1922
|
+
const results = await this.store.getResults(id);
|
|
1692
1923
|
const usage = {
|
|
1693
1924
|
total_prompt_tokens: 0,
|
|
1694
1925
|
total_completion_tokens: 0,
|
|
@@ -1710,37 +1941,119 @@ var BatchManager = class {
|
|
|
1710
1941
|
/**
|
|
1711
1942
|
* List all batches.
|
|
1712
1943
|
*/
|
|
1713
|
-
list() {
|
|
1714
|
-
|
|
1944
|
+
async list() {
|
|
1945
|
+
const ids = await this.store.listBatches();
|
|
1946
|
+
const batches = [];
|
|
1947
|
+
for (const id of ids) {
|
|
1948
|
+
const meta = await this.store.getMeta(id);
|
|
1949
|
+
if (meta) batches.push(meta);
|
|
1950
|
+
}
|
|
1951
|
+
return batches;
|
|
1715
1952
|
}
|
|
1716
1953
|
/**
|
|
1717
1954
|
* Cancel a batch.
|
|
1718
1955
|
*/
|
|
1719
|
-
cancel(id) {
|
|
1720
|
-
const batch = this.store.getMeta(id);
|
|
1956
|
+
async cancel(id) {
|
|
1957
|
+
const batch = await this.store.getMeta(id);
|
|
1721
1958
|
if (!batch) {
|
|
1722
1959
|
throw new AnyModelError(404, `Batch ${id} not found`);
|
|
1723
1960
|
}
|
|
1724
1961
|
if (batch.status === "completed" || batch.status === "cancelled") {
|
|
1725
1962
|
return batch;
|
|
1726
1963
|
}
|
|
1964
|
+
if (batch.batch_mode === "native") {
|
|
1965
|
+
const providerState = await this.store.loadProviderState(id);
|
|
1966
|
+
const adapter = this.batchAdapters.get(batch.provider_name);
|
|
1967
|
+
if (adapter && providerState?.providerBatchId) {
|
|
1968
|
+
try {
|
|
1969
|
+
await adapter.cancelBatch(providerState.providerBatchId);
|
|
1970
|
+
} catch {
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1727
1974
|
batch.status = "cancelled";
|
|
1728
1975
|
batch.completed_at = (/* @__PURE__ */ new Date()).toISOString();
|
|
1729
|
-
this.store.updateMeta(batch);
|
|
1976
|
+
await this.store.updateMeta(batch);
|
|
1730
1977
|
return batch;
|
|
1731
1978
|
}
|
|
1732
1979
|
/**
|
|
1733
|
-
* Process batch
|
|
1980
|
+
* Process batch via native provider batch API.
|
|
1734
1981
|
*/
|
|
1735
|
-
async
|
|
1736
|
-
const batch = this.store.getMeta(batchId);
|
|
1982
|
+
async processNativeBatch(batchId, request, adapter) {
|
|
1983
|
+
const batch = await this.store.getMeta(batchId);
|
|
1984
|
+
if (!batch) return;
|
|
1985
|
+
try {
|
|
1986
|
+
const model = request.model.includes("/") ? request.model.split("/").slice(1).join("/") : request.model;
|
|
1987
|
+
const { providerBatchId, metadata } = await adapter.createBatch(
|
|
1988
|
+
model,
|
|
1989
|
+
request.requests,
|
|
1990
|
+
request.options
|
|
1991
|
+
);
|
|
1992
|
+
await this.store.saveProviderState(batchId, {
|
|
1993
|
+
providerBatchId,
|
|
1994
|
+
providerName: batch.provider_name,
|
|
1995
|
+
...metadata
|
|
1996
|
+
});
|
|
1997
|
+
batch.status = "processing";
|
|
1998
|
+
await this.store.updateMeta(batch);
|
|
1999
|
+
} catch (err) {
|
|
2000
|
+
batch.status = "failed";
|
|
2001
|
+
batch.completed_at = (/* @__PURE__ */ new Date()).toISOString();
|
|
2002
|
+
await this.store.updateMeta(batch);
|
|
2003
|
+
throw err;
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
/**
|
|
2007
|
+
* Sync native batch status from provider.
|
|
2008
|
+
*/
|
|
2009
|
+
async syncNativeBatchStatus(batchId) {
|
|
2010
|
+
const batch = await this.store.getMeta(batchId);
|
|
2011
|
+
if (!batch) return;
|
|
2012
|
+
const providerState = await this.store.loadProviderState(batchId);
|
|
2013
|
+
if (!providerState?.providerBatchId) return;
|
|
2014
|
+
const adapter = this.batchAdapters.get(batch.provider_name);
|
|
2015
|
+
if (!adapter) return;
|
|
2016
|
+
try {
|
|
2017
|
+
const status = await adapter.pollBatch(providerState.providerBatchId);
|
|
2018
|
+
batch.total = status.total || batch.total;
|
|
2019
|
+
batch.completed = status.completed;
|
|
2020
|
+
batch.failed = status.failed;
|
|
2021
|
+
if (status.status === "completed" || status.status === "failed" || status.status === "cancelled") {
|
|
2022
|
+
batch.status = status.status;
|
|
2023
|
+
batch.completed_at = (/* @__PURE__ */ new Date()).toISOString();
|
|
2024
|
+
if (status.status === "completed" || status.status === "failed") {
|
|
2025
|
+
try {
|
|
2026
|
+
const results = await adapter.getBatchResults(providerState.providerBatchId);
|
|
2027
|
+
for (const result of results) {
|
|
2028
|
+
await this.store.appendResult(batchId, result);
|
|
2029
|
+
}
|
|
2030
|
+
batch.completed = results.filter((r) => r.status === "success").length;
|
|
2031
|
+
batch.failed = results.filter((r) => r.status === "error").length;
|
|
2032
|
+
} catch {
|
|
2033
|
+
if (batch.status !== "failed") {
|
|
2034
|
+
batch.status = "failed";
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
} else {
|
|
2039
|
+
batch.status = "processing";
|
|
2040
|
+
}
|
|
2041
|
+
await this.store.updateMeta(batch);
|
|
2042
|
+
} catch {
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
/**
|
|
2046
|
+
* Process batch requests concurrently (fallback path).
|
|
2047
|
+
*/
|
|
2048
|
+
async processConcurrentBatch(batchId, request) {
|
|
2049
|
+
const batch = await this.store.getMeta(batchId);
|
|
2050
|
+
if (!batch) return;
|
|
1737
2051
|
batch.status = "processing";
|
|
1738
|
-
this.store.updateMeta(batch);
|
|
2052
|
+
await this.store.updateMeta(batch);
|
|
1739
2053
|
const items = request.requests;
|
|
1740
|
-
const queue = [...items];
|
|
1741
2054
|
const active = /* @__PURE__ */ new Set();
|
|
1742
2055
|
const processItem = async (item) => {
|
|
1743
|
-
const current = this.store.getMeta(batchId);
|
|
2056
|
+
const current = await this.store.getMeta(batchId);
|
|
1744
2057
|
if (current?.status === "cancelled") return;
|
|
1745
2058
|
const chatRequest = {
|
|
1746
2059
|
model: request.model,
|
|
@@ -1772,17 +2085,19 @@ var BatchManager = class {
|
|
|
1772
2085
|
error: { code: error.code, message: error.message }
|
|
1773
2086
|
};
|
|
1774
2087
|
}
|
|
1775
|
-
this.store.appendResult(batchId, result);
|
|
1776
|
-
const meta = this.store.getMeta(batchId);
|
|
1777
|
-
if (
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
2088
|
+
await this.store.appendResult(batchId, result);
|
|
2089
|
+
const meta = await this.store.getMeta(batchId);
|
|
2090
|
+
if (meta) {
|
|
2091
|
+
if (result.status === "success") {
|
|
2092
|
+
meta.completed++;
|
|
2093
|
+
} else {
|
|
2094
|
+
meta.failed++;
|
|
2095
|
+
}
|
|
2096
|
+
await this.store.updateMeta(meta);
|
|
1781
2097
|
}
|
|
1782
|
-
this.store.updateMeta(meta);
|
|
1783
2098
|
};
|
|
1784
|
-
for (const item of
|
|
1785
|
-
const current = this.store.getMeta(batchId);
|
|
2099
|
+
for (const item of items) {
|
|
2100
|
+
const current = await this.store.getMeta(batchId);
|
|
1786
2101
|
if (current?.status === "cancelled") break;
|
|
1787
2102
|
if (active.size >= this.concurrencyLimit) {
|
|
1788
2103
|
await Promise.race(active);
|
|
@@ -1793,15 +2108,411 @@ var BatchManager = class {
|
|
|
1793
2108
|
active.add(promise);
|
|
1794
2109
|
}
|
|
1795
2110
|
await Promise.all(active);
|
|
1796
|
-
const finalMeta = this.store.getMeta(batchId);
|
|
1797
|
-
if (finalMeta.status !== "cancelled") {
|
|
2111
|
+
const finalMeta = await this.store.getMeta(batchId);
|
|
2112
|
+
if (finalMeta && finalMeta.status !== "cancelled") {
|
|
1798
2113
|
finalMeta.status = finalMeta.failed === finalMeta.total ? "failed" : "completed";
|
|
1799
2114
|
finalMeta.completed_at = (/* @__PURE__ */ new Date()).toISOString();
|
|
1800
|
-
this.store.updateMeta(finalMeta);
|
|
2115
|
+
await this.store.updateMeta(finalMeta);
|
|
1801
2116
|
}
|
|
1802
2117
|
}
|
|
1803
2118
|
};
|
|
1804
2119
|
|
|
2120
|
+
// src/providers/openai-batch.ts
|
|
2121
|
+
var OPENAI_API_BASE2 = "https://api.openai.com/v1";
|
|
2122
|
+
function createOpenAIBatchAdapter(apiKey) {
|
|
2123
|
+
async function apiRequest(path2, options = {}) {
|
|
2124
|
+
const headers = {
|
|
2125
|
+
"Authorization": `Bearer ${apiKey}`
|
|
2126
|
+
};
|
|
2127
|
+
let fetchBody;
|
|
2128
|
+
if (options.formData) {
|
|
2129
|
+
fetchBody = options.formData;
|
|
2130
|
+
} else if (options.body) {
|
|
2131
|
+
headers["Content-Type"] = "application/json";
|
|
2132
|
+
fetchBody = JSON.stringify(options.body);
|
|
2133
|
+
}
|
|
2134
|
+
const res = await fetch(`${OPENAI_API_BASE2}${path2}`, {
|
|
2135
|
+
method: options.method || "GET",
|
|
2136
|
+
headers,
|
|
2137
|
+
body: fetchBody
|
|
2138
|
+
});
|
|
2139
|
+
if (!res.ok) {
|
|
2140
|
+
let errorBody;
|
|
2141
|
+
try {
|
|
2142
|
+
errorBody = await res.json();
|
|
2143
|
+
} catch {
|
|
2144
|
+
errorBody = { message: res.statusText };
|
|
2145
|
+
}
|
|
2146
|
+
const msg = errorBody?.error?.message || errorBody?.message || res.statusText;
|
|
2147
|
+
throw new AnyModelError(res.status >= 500 ? 502 : res.status, msg, {
|
|
2148
|
+
provider_name: "openai",
|
|
2149
|
+
raw: errorBody
|
|
2150
|
+
});
|
|
2151
|
+
}
|
|
2152
|
+
return res;
|
|
2153
|
+
}
|
|
2154
|
+
function buildJSONL(model, requests) {
|
|
2155
|
+
return requests.map((req) => {
|
|
2156
|
+
const body = {
|
|
2157
|
+
model,
|
|
2158
|
+
messages: req.messages
|
|
2159
|
+
};
|
|
2160
|
+
if (req.max_tokens !== void 0) body.max_tokens = req.max_tokens;
|
|
2161
|
+
if (req.temperature !== void 0) body.temperature = req.temperature;
|
|
2162
|
+
if (req.top_p !== void 0) body.top_p = req.top_p;
|
|
2163
|
+
if (req.stop !== void 0) body.stop = req.stop;
|
|
2164
|
+
if (req.response_format !== void 0) body.response_format = req.response_format;
|
|
2165
|
+
if (req.tools !== void 0) body.tools = req.tools;
|
|
2166
|
+
if (req.tool_choice !== void 0) body.tool_choice = req.tool_choice;
|
|
2167
|
+
return JSON.stringify({
|
|
2168
|
+
custom_id: req.custom_id,
|
|
2169
|
+
method: "POST",
|
|
2170
|
+
url: "/v1/chat/completions",
|
|
2171
|
+
body
|
|
2172
|
+
});
|
|
2173
|
+
}).join("\n");
|
|
2174
|
+
}
|
|
2175
|
+
function rePrefixId(id) {
|
|
2176
|
+
if (id && id.startsWith("chatcmpl-")) {
|
|
2177
|
+
return `gen-${id.substring(9)}`;
|
|
2178
|
+
}
|
|
2179
|
+
return id.startsWith("gen-") ? id : `gen-${id}`;
|
|
2180
|
+
}
|
|
2181
|
+
function translateOpenAIResponse(body) {
|
|
2182
|
+
return {
|
|
2183
|
+
id: rePrefixId(body.id || generateId()),
|
|
2184
|
+
object: "chat.completion",
|
|
2185
|
+
created: body.created || Math.floor(Date.now() / 1e3),
|
|
2186
|
+
model: `openai/${body.model}`,
|
|
2187
|
+
choices: body.choices,
|
|
2188
|
+
usage: body.usage
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2191
|
+
function mapStatus(openaiStatus) {
|
|
2192
|
+
switch (openaiStatus) {
|
|
2193
|
+
case "validating":
|
|
2194
|
+
case "finalizing":
|
|
2195
|
+
return "processing";
|
|
2196
|
+
case "in_progress":
|
|
2197
|
+
return "processing";
|
|
2198
|
+
case "completed":
|
|
2199
|
+
return "completed";
|
|
2200
|
+
case "failed":
|
|
2201
|
+
return "failed";
|
|
2202
|
+
case "expired":
|
|
2203
|
+
return "failed";
|
|
2204
|
+
case "cancelled":
|
|
2205
|
+
case "cancelling":
|
|
2206
|
+
return "cancelled";
|
|
2207
|
+
default:
|
|
2208
|
+
return "pending";
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
return {
|
|
2212
|
+
async createBatch(model, requests, options) {
|
|
2213
|
+
const jsonlContent = buildJSONL(model, requests);
|
|
2214
|
+
const blob = new Blob([jsonlContent], { type: "application/jsonl" });
|
|
2215
|
+
const formData = new FormData();
|
|
2216
|
+
formData.append("purpose", "batch");
|
|
2217
|
+
formData.append("file", blob, "batch_input.jsonl");
|
|
2218
|
+
const uploadRes = await apiRequest("/files", { method: "POST", formData });
|
|
2219
|
+
const fileData = await uploadRes.json();
|
|
2220
|
+
const inputFileId = fileData.id;
|
|
2221
|
+
const batchRes = await apiRequest("/batches", {
|
|
2222
|
+
method: "POST",
|
|
2223
|
+
body: {
|
|
2224
|
+
input_file_id: inputFileId,
|
|
2225
|
+
endpoint: "/v1/chat/completions",
|
|
2226
|
+
completion_window: "24h",
|
|
2227
|
+
metadata: options?.metadata
|
|
2228
|
+
}
|
|
2229
|
+
});
|
|
2230
|
+
const batchData = await batchRes.json();
|
|
2231
|
+
return {
|
|
2232
|
+
providerBatchId: batchData.id,
|
|
2233
|
+
metadata: {
|
|
2234
|
+
input_file_id: inputFileId,
|
|
2235
|
+
openai_status: batchData.status
|
|
2236
|
+
}
|
|
2237
|
+
};
|
|
2238
|
+
},
|
|
2239
|
+
async pollBatch(providerBatchId) {
|
|
2240
|
+
const res = await apiRequest(`/batches/${providerBatchId}`);
|
|
2241
|
+
const data = await res.json();
|
|
2242
|
+
const requestCounts = data.request_counts || {};
|
|
2243
|
+
return {
|
|
2244
|
+
status: mapStatus(data.status),
|
|
2245
|
+
total: requestCounts.total || 0,
|
|
2246
|
+
completed: requestCounts.completed || 0,
|
|
2247
|
+
failed: requestCounts.failed || 0
|
|
2248
|
+
};
|
|
2249
|
+
},
|
|
2250
|
+
async getBatchResults(providerBatchId) {
|
|
2251
|
+
const batchRes = await apiRequest(`/batches/${providerBatchId}`);
|
|
2252
|
+
const batchData = await batchRes.json();
|
|
2253
|
+
const results = [];
|
|
2254
|
+
if (batchData.output_file_id) {
|
|
2255
|
+
const outputRes = await apiRequest(`/files/${batchData.output_file_id}/content`);
|
|
2256
|
+
const outputText = await outputRes.text();
|
|
2257
|
+
for (const line of outputText.trim().split("\n")) {
|
|
2258
|
+
if (!line) continue;
|
|
2259
|
+
const item = JSON.parse(line);
|
|
2260
|
+
if (item.response?.status_code === 200) {
|
|
2261
|
+
results.push({
|
|
2262
|
+
custom_id: item.custom_id,
|
|
2263
|
+
status: "success",
|
|
2264
|
+
response: translateOpenAIResponse(item.response.body),
|
|
2265
|
+
error: null
|
|
2266
|
+
});
|
|
2267
|
+
} else {
|
|
2268
|
+
results.push({
|
|
2269
|
+
custom_id: item.custom_id,
|
|
2270
|
+
status: "error",
|
|
2271
|
+
response: null,
|
|
2272
|
+
error: {
|
|
2273
|
+
code: item.response?.status_code || 500,
|
|
2274
|
+
message: item.error?.message || item.response?.body?.error?.message || "Unknown error"
|
|
2275
|
+
}
|
|
2276
|
+
});
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
if (batchData.error_file_id) {
|
|
2281
|
+
const errorRes = await apiRequest(`/files/${batchData.error_file_id}/content`);
|
|
2282
|
+
const errorText = await errorRes.text();
|
|
2283
|
+
for (const line of errorText.trim().split("\n")) {
|
|
2284
|
+
if (!line) continue;
|
|
2285
|
+
const item = JSON.parse(line);
|
|
2286
|
+
const existing = results.find((r) => r.custom_id === item.custom_id);
|
|
2287
|
+
if (!existing) {
|
|
2288
|
+
results.push({
|
|
2289
|
+
custom_id: item.custom_id,
|
|
2290
|
+
status: "error",
|
|
2291
|
+
response: null,
|
|
2292
|
+
error: {
|
|
2293
|
+
code: item.response?.status_code || 500,
|
|
2294
|
+
message: item.error?.message || "Batch item error"
|
|
2295
|
+
}
|
|
2296
|
+
});
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
2300
|
+
return results;
|
|
2301
|
+
},
|
|
2302
|
+
async cancelBatch(providerBatchId) {
|
|
2303
|
+
await apiRequest(`/batches/${providerBatchId}/cancel`, { method: "POST" });
|
|
2304
|
+
}
|
|
2305
|
+
};
|
|
2306
|
+
}
|
|
2307
|
+
|
|
2308
|
+
// src/providers/anthropic-batch.ts
|
|
2309
|
+
var ANTHROPIC_API_BASE2 = "https://api.anthropic.com/v1";
|
|
2310
|
+
var ANTHROPIC_VERSION2 = "2023-06-01";
|
|
2311
|
+
var DEFAULT_MAX_TOKENS2 = 4096;
|
|
2312
|
+
function createAnthropicBatchAdapter(apiKey) {
|
|
2313
|
+
async function apiRequest(path2, options = {}) {
|
|
2314
|
+
const headers = {
|
|
2315
|
+
"x-api-key": apiKey,
|
|
2316
|
+
"anthropic-version": ANTHROPIC_VERSION2,
|
|
2317
|
+
"Content-Type": "application/json"
|
|
2318
|
+
};
|
|
2319
|
+
const res = await fetch(`${ANTHROPIC_API_BASE2}${path2}`, {
|
|
2320
|
+
method: options.method || "GET",
|
|
2321
|
+
headers,
|
|
2322
|
+
body: options.body ? JSON.stringify(options.body) : void 0
|
|
2323
|
+
});
|
|
2324
|
+
if (!res.ok) {
|
|
2325
|
+
let errorBody;
|
|
2326
|
+
try {
|
|
2327
|
+
errorBody = await res.json();
|
|
2328
|
+
} catch {
|
|
2329
|
+
errorBody = { message: res.statusText };
|
|
2330
|
+
}
|
|
2331
|
+
const msg = errorBody?.error?.message || errorBody?.message || res.statusText;
|
|
2332
|
+
throw new AnyModelError(res.status >= 500 ? 502 : res.status, msg, {
|
|
2333
|
+
provider_name: "anthropic",
|
|
2334
|
+
raw: errorBody
|
|
2335
|
+
});
|
|
2336
|
+
}
|
|
2337
|
+
return res;
|
|
2338
|
+
}
|
|
2339
|
+
function translateToAnthropicParams(model, req) {
|
|
2340
|
+
const params = {
|
|
2341
|
+
model,
|
|
2342
|
+
max_tokens: req.max_tokens || DEFAULT_MAX_TOKENS2
|
|
2343
|
+
};
|
|
2344
|
+
const systemMessages = req.messages.filter((m) => m.role === "system");
|
|
2345
|
+
const nonSystemMessages = req.messages.filter((m) => m.role !== "system");
|
|
2346
|
+
if (systemMessages.length > 0) {
|
|
2347
|
+
params.system = systemMessages.map((m) => typeof m.content === "string" ? m.content : "").join("\n");
|
|
2348
|
+
}
|
|
2349
|
+
params.messages = nonSystemMessages.map((m) => ({
|
|
2350
|
+
role: m.role === "tool" ? "user" : m.role,
|
|
2351
|
+
content: m.tool_call_id ? [{ type: "tool_result", tool_use_id: m.tool_call_id, content: typeof m.content === "string" ? m.content : "" }] : m.content
|
|
2352
|
+
}));
|
|
2353
|
+
if (req.temperature !== void 0) params.temperature = req.temperature;
|
|
2354
|
+
if (req.top_p !== void 0) params.top_p = req.top_p;
|
|
2355
|
+
if (req.top_k !== void 0) params.top_k = req.top_k;
|
|
2356
|
+
if (req.stop !== void 0) params.stop_sequences = Array.isArray(req.stop) ? req.stop : [req.stop];
|
|
2357
|
+
if (req.tools && req.tools.length > 0) {
|
|
2358
|
+
params.tools = req.tools.map((t) => ({
|
|
2359
|
+
name: t.function.name,
|
|
2360
|
+
description: t.function.description || "",
|
|
2361
|
+
input_schema: t.function.parameters || { type: "object", properties: {} }
|
|
2362
|
+
}));
|
|
2363
|
+
if (req.tool_choice) {
|
|
2364
|
+
if (req.tool_choice === "auto") {
|
|
2365
|
+
params.tool_choice = { type: "auto" };
|
|
2366
|
+
} else if (req.tool_choice === "required") {
|
|
2367
|
+
params.tool_choice = { type: "any" };
|
|
2368
|
+
} else if (req.tool_choice === "none") {
|
|
2369
|
+
delete params.tools;
|
|
2370
|
+
} else if (typeof req.tool_choice === "object") {
|
|
2371
|
+
params.tool_choice = { type: "tool", name: req.tool_choice.function.name };
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
}
|
|
2375
|
+
if (req.response_format) {
|
|
2376
|
+
if (req.response_format.type === "json_object" || req.response_format.type === "json_schema") {
|
|
2377
|
+
const jsonInstruction = "Respond with valid JSON only. Do not include any text outside the JSON object.";
|
|
2378
|
+
params.system = params.system ? `${jsonInstruction}
|
|
2379
|
+
|
|
2380
|
+
${params.system}` : jsonInstruction;
|
|
2381
|
+
}
|
|
2382
|
+
}
|
|
2383
|
+
return params;
|
|
2384
|
+
}
|
|
2385
|
+
function mapStopReason(reason) {
|
|
2386
|
+
switch (reason) {
|
|
2387
|
+
case "end_turn":
|
|
2388
|
+
return "stop";
|
|
2389
|
+
case "max_tokens":
|
|
2390
|
+
return "length";
|
|
2391
|
+
case "tool_use":
|
|
2392
|
+
return "tool_calls";
|
|
2393
|
+
case "stop_sequence":
|
|
2394
|
+
return "stop";
|
|
2395
|
+
default:
|
|
2396
|
+
return "stop";
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
function translateAnthropicMessage(msg) {
|
|
2400
|
+
let content = "";
|
|
2401
|
+
const toolCalls = [];
|
|
2402
|
+
for (const block of msg.content || []) {
|
|
2403
|
+
if (block.type === "text") {
|
|
2404
|
+
content += block.text;
|
|
2405
|
+
} else if (block.type === "tool_use") {
|
|
2406
|
+
toolCalls.push({
|
|
2407
|
+
id: block.id,
|
|
2408
|
+
type: "function",
|
|
2409
|
+
function: {
|
|
2410
|
+
name: block.name,
|
|
2411
|
+
arguments: JSON.stringify(block.input)
|
|
2412
|
+
}
|
|
2413
|
+
});
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
const message = { role: "assistant", content };
|
|
2417
|
+
if (toolCalls.length > 0) {
|
|
2418
|
+
message.tool_calls = toolCalls;
|
|
2419
|
+
}
|
|
2420
|
+
return {
|
|
2421
|
+
id: generateId(),
|
|
2422
|
+
object: "chat.completion",
|
|
2423
|
+
created: Math.floor(Date.now() / 1e3),
|
|
2424
|
+
model: `anthropic/${msg.model}`,
|
|
2425
|
+
choices: [{
|
|
2426
|
+
index: 0,
|
|
2427
|
+
message,
|
|
2428
|
+
finish_reason: mapStopReason(msg.stop_reason)
|
|
2429
|
+
}],
|
|
2430
|
+
usage: {
|
|
2431
|
+
prompt_tokens: msg.usage?.input_tokens || 0,
|
|
2432
|
+
completion_tokens: msg.usage?.output_tokens || 0,
|
|
2433
|
+
total_tokens: (msg.usage?.input_tokens || 0) + (msg.usage?.output_tokens || 0)
|
|
2434
|
+
}
|
|
2435
|
+
};
|
|
2436
|
+
}
|
|
2437
|
+
return {
|
|
2438
|
+
async createBatch(model, requests, _options) {
|
|
2439
|
+
const batchRequests = requests.map((req) => ({
|
|
2440
|
+
custom_id: req.custom_id,
|
|
2441
|
+
params: translateToAnthropicParams(model, req)
|
|
2442
|
+
}));
|
|
2443
|
+
const res = await apiRequest("/messages/batches", {
|
|
2444
|
+
method: "POST",
|
|
2445
|
+
body: { requests: batchRequests }
|
|
2446
|
+
});
|
|
2447
|
+
const data = await res.json();
|
|
2448
|
+
return {
|
|
2449
|
+
providerBatchId: data.id,
|
|
2450
|
+
metadata: {
|
|
2451
|
+
anthropic_type: data.type,
|
|
2452
|
+
created_at: data.created_at
|
|
2453
|
+
}
|
|
2454
|
+
};
|
|
2455
|
+
},
|
|
2456
|
+
async pollBatch(providerBatchId) {
|
|
2457
|
+
const res = await apiRequest(`/messages/batches/${providerBatchId}`);
|
|
2458
|
+
const data = await res.json();
|
|
2459
|
+
const counts = data.request_counts || {};
|
|
2460
|
+
const total = (counts.processing || 0) + (counts.succeeded || 0) + (counts.errored || 0) + (counts.canceled || 0) + (counts.expired || 0);
|
|
2461
|
+
let status;
|
|
2462
|
+
if (data.processing_status === "ended") {
|
|
2463
|
+
if (counts.succeeded === 0 && (counts.errored > 0 || counts.expired > 0 || counts.canceled > 0)) {
|
|
2464
|
+
status = "failed";
|
|
2465
|
+
} else if (data.cancel_initiated_at) {
|
|
2466
|
+
status = "cancelled";
|
|
2467
|
+
} else {
|
|
2468
|
+
status = "completed";
|
|
2469
|
+
}
|
|
2470
|
+
} else {
|
|
2471
|
+
status = "processing";
|
|
2472
|
+
}
|
|
2473
|
+
return {
|
|
2474
|
+
status,
|
|
2475
|
+
total,
|
|
2476
|
+
completed: counts.succeeded || 0,
|
|
2477
|
+
failed: (counts.errored || 0) + (counts.expired || 0) + (counts.canceled || 0)
|
|
2478
|
+
};
|
|
2479
|
+
},
|
|
2480
|
+
async getBatchResults(providerBatchId) {
|
|
2481
|
+
const res = await apiRequest(`/messages/batches/${providerBatchId}/results`);
|
|
2482
|
+
const text = await res.text();
|
|
2483
|
+
const results = [];
|
|
2484
|
+
for (const line of text.trim().split("\n")) {
|
|
2485
|
+
if (!line) continue;
|
|
2486
|
+
const item = JSON.parse(line);
|
|
2487
|
+
if (item.result?.type === "succeeded") {
|
|
2488
|
+
results.push({
|
|
2489
|
+
custom_id: item.custom_id,
|
|
2490
|
+
status: "success",
|
|
2491
|
+
response: translateAnthropicMessage(item.result.message),
|
|
2492
|
+
error: null
|
|
2493
|
+
});
|
|
2494
|
+
} else {
|
|
2495
|
+
const errorType = item.result?.type || "unknown";
|
|
2496
|
+
const errorMsg = item.result?.error?.message || `Batch item ${errorType}`;
|
|
2497
|
+
results.push({
|
|
2498
|
+
custom_id: item.custom_id,
|
|
2499
|
+
status: "error",
|
|
2500
|
+
response: null,
|
|
2501
|
+
error: {
|
|
2502
|
+
code: errorType === "expired" ? 408 : 500,
|
|
2503
|
+
message: errorMsg
|
|
2504
|
+
}
|
|
2505
|
+
});
|
|
2506
|
+
}
|
|
2507
|
+
}
|
|
2508
|
+
return results;
|
|
2509
|
+
},
|
|
2510
|
+
async cancelBatch(providerBatchId) {
|
|
2511
|
+
await apiRequest(`/messages/batches/${providerBatchId}/cancel`, { method: "POST" });
|
|
2512
|
+
}
|
|
2513
|
+
};
|
|
2514
|
+
}
|
|
2515
|
+
|
|
1805
2516
|
// src/client.ts
|
|
1806
2517
|
var AnyModel = class {
|
|
1807
2518
|
registry;
|
|
@@ -1817,6 +2528,9 @@ var AnyModel = class {
|
|
|
1817
2528
|
constructor(config = {}) {
|
|
1818
2529
|
this.config = resolveConfig(config);
|
|
1819
2530
|
this.registry = new ProviderRegistry();
|
|
2531
|
+
if (this.config.io) {
|
|
2532
|
+
configureFsIO(this.config.io);
|
|
2533
|
+
}
|
|
1820
2534
|
this.registerProviders();
|
|
1821
2535
|
this.router = new Router(this.registry, this.config.aliases, this.config);
|
|
1822
2536
|
this.chat = {
|
|
@@ -1866,8 +2580,10 @@ var AnyModel = class {
|
|
|
1866
2580
|
};
|
|
1867
2581
|
this.batchManager = new BatchManager(this.router, {
|
|
1868
2582
|
dir: this.config.batch?.dir,
|
|
1869
|
-
concurrency: this.config.batch?.concurrencyFallback
|
|
2583
|
+
concurrency: this.config.batch?.concurrencyFallback,
|
|
2584
|
+
pollInterval: this.config.batch?.pollInterval
|
|
1870
2585
|
});
|
|
2586
|
+
this.registerBatchAdapters();
|
|
1871
2587
|
this.batches = {
|
|
1872
2588
|
create: (request) => this.batchManager.create(request),
|
|
1873
2589
|
createAndPoll: (request, options) => this.batchManager.createAndPoll(request, options),
|
|
@@ -1919,6 +2635,17 @@ var AnyModel = class {
|
|
|
1919
2635
|
}
|
|
1920
2636
|
}
|
|
1921
2637
|
}
|
|
2638
|
+
registerBatchAdapters() {
|
|
2639
|
+
const config = this.config;
|
|
2640
|
+
const openaiKey = config.openai?.apiKey || process.env.OPENAI_API_KEY;
|
|
2641
|
+
if (openaiKey) {
|
|
2642
|
+
this.batchManager.registerBatchAdapter("openai", createOpenAIBatchAdapter(openaiKey));
|
|
2643
|
+
}
|
|
2644
|
+
const anthropicKey = config.anthropic?.apiKey || process.env.ANTHROPIC_API_KEY;
|
|
2645
|
+
if (anthropicKey) {
|
|
2646
|
+
this.batchManager.registerBatchAdapter("anthropic", createAnthropicBatchAdapter(anthropicKey));
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
1922
2649
|
applyDefaults(request) {
|
|
1923
2650
|
const defaults = this.config.defaults;
|
|
1924
2651
|
if (!defaults) return request;
|
|
@@ -1947,10 +2674,10 @@ var AnyModel = class {
|
|
|
1947
2674
|
// src/server.ts
|
|
1948
2675
|
var import_node_http = require("http");
|
|
1949
2676
|
function parseBody(req) {
|
|
1950
|
-
return new Promise((
|
|
2677
|
+
return new Promise((resolve2, reject) => {
|
|
1951
2678
|
const chunks = [];
|
|
1952
2679
|
req.on("data", (chunk) => chunks.push(chunk));
|
|
1953
|
-
req.on("end", () =>
|
|
2680
|
+
req.on("end", () => resolve2(Buffer.concat(chunks).toString()));
|
|
1954
2681
|
req.on("error", reject);
|
|
1955
2682
|
});
|
|
1956
2683
|
}
|
|
@@ -1980,7 +2707,7 @@ function createAnyModelServer(options = {}) {
|
|
|
1980
2707
|
const basePath = "/api/v1";
|
|
1981
2708
|
const server = (0, import_node_http.createServer)(async (req, res) => {
|
|
1982
2709
|
const url = new URL(req.url || "/", `http://${req.headers.host}`);
|
|
1983
|
-
const
|
|
2710
|
+
const path2 = url.pathname;
|
|
1984
2711
|
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
1985
2712
|
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
1986
2713
|
res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization");
|
|
@@ -1990,11 +2717,11 @@ function createAnyModelServer(options = {}) {
|
|
|
1990
2717
|
return;
|
|
1991
2718
|
}
|
|
1992
2719
|
try {
|
|
1993
|
-
if (
|
|
2720
|
+
if (path2 === "/health" && req.method === "GET") {
|
|
1994
2721
|
sendJSON(res, 200, { status: "ok" });
|
|
1995
2722
|
return;
|
|
1996
2723
|
}
|
|
1997
|
-
if (
|
|
2724
|
+
if (path2 === `${basePath}/chat/completions` && req.method === "POST") {
|
|
1998
2725
|
const body = JSON.parse(await parseBody(req));
|
|
1999
2726
|
if (body.stream) {
|
|
2000
2727
|
const stream = await client.chat.completions.create(body);
|
|
@@ -2005,14 +2732,14 @@ function createAnyModelServer(options = {}) {
|
|
|
2005
2732
|
}
|
|
2006
2733
|
return;
|
|
2007
2734
|
}
|
|
2008
|
-
if (
|
|
2735
|
+
if (path2 === `${basePath}/models` && req.method === "GET") {
|
|
2009
2736
|
const provider = url.searchParams.get("provider") || void 0;
|
|
2010
2737
|
const models = await client.models.list({ provider });
|
|
2011
2738
|
sendJSON(res, 200, { object: "list", data: models });
|
|
2012
2739
|
return;
|
|
2013
2740
|
}
|
|
2014
|
-
if (
|
|
2015
|
-
const id =
|
|
2741
|
+
if (path2.startsWith(`${basePath}/generation/`) && req.method === "GET") {
|
|
2742
|
+
const id = path2.substring(`${basePath}/generation/`.length);
|
|
2016
2743
|
const stats = client.generation.get(id);
|
|
2017
2744
|
if (!stats) {
|
|
2018
2745
|
sendError(res, 404, `Generation ${id} not found`);
|
|
@@ -2021,26 +2748,26 @@ function createAnyModelServer(options = {}) {
|
|
|
2021
2748
|
sendJSON(res, 200, stats);
|
|
2022
2749
|
return;
|
|
2023
2750
|
}
|
|
2024
|
-
if (
|
|
2751
|
+
if (path2 === `${basePath}/batches` && req.method === "POST") {
|
|
2025
2752
|
const body = JSON.parse(await parseBody(req));
|
|
2026
2753
|
const batch = await client.batches.create(body);
|
|
2027
2754
|
sendJSON(res, 201, batch);
|
|
2028
2755
|
return;
|
|
2029
2756
|
}
|
|
2030
|
-
if (
|
|
2031
|
-
const batches = client.batches.list();
|
|
2757
|
+
if (path2 === `${basePath}/batches` && req.method === "GET") {
|
|
2758
|
+
const batches = await client.batches.list();
|
|
2032
2759
|
sendJSON(res, 200, { object: "list", data: batches });
|
|
2033
2760
|
return;
|
|
2034
2761
|
}
|
|
2035
|
-
if (
|
|
2036
|
-
const parts =
|
|
2762
|
+
if (path2.startsWith(`${basePath}/batches/`) && req.method === "GET") {
|
|
2763
|
+
const parts = path2.substring(`${basePath}/batches/`.length).split("/");
|
|
2037
2764
|
const id = parts[0];
|
|
2038
2765
|
if (parts[1] === "results") {
|
|
2039
|
-
const results = client.batches.results(id);
|
|
2766
|
+
const results = await client.batches.results(id);
|
|
2040
2767
|
sendJSON(res, 200, results);
|
|
2041
2768
|
return;
|
|
2042
2769
|
}
|
|
2043
|
-
const batch = client.batches.get(id);
|
|
2770
|
+
const batch = await client.batches.get(id);
|
|
2044
2771
|
if (!batch) {
|
|
2045
2772
|
sendError(res, 404, `Batch ${id} not found`);
|
|
2046
2773
|
return;
|
|
@@ -2048,16 +2775,16 @@ function createAnyModelServer(options = {}) {
|
|
|
2048
2775
|
sendJSON(res, 200, batch);
|
|
2049
2776
|
return;
|
|
2050
2777
|
}
|
|
2051
|
-
if (
|
|
2052
|
-
const parts =
|
|
2778
|
+
if (path2.startsWith(`${basePath}/batches/`) && req.method === "POST") {
|
|
2779
|
+
const parts = path2.substring(`${basePath}/batches/`.length).split("/");
|
|
2053
2780
|
const id = parts[0];
|
|
2054
2781
|
if (parts[1] === "cancel") {
|
|
2055
|
-
const batch = client.batches.cancel(id);
|
|
2782
|
+
const batch = await client.batches.cancel(id);
|
|
2056
2783
|
sendJSON(res, 200, batch);
|
|
2057
2784
|
return;
|
|
2058
2785
|
}
|
|
2059
2786
|
}
|
|
2060
|
-
sendError(res, 404, `Not found: ${
|
|
2787
|
+
sendError(res, 404, `Not found: ${path2}`);
|
|
2061
2788
|
} catch (err) {
|
|
2062
2789
|
const code = err?.code || 500;
|
|
2063
2790
|
const message = err?.message || "Internal server error";
|
|
@@ -2093,8 +2820,19 @@ function startServer(options = {}) {
|
|
|
2093
2820
|
BatchManager,
|
|
2094
2821
|
BatchStore,
|
|
2095
2822
|
GenerationStatsStore,
|
|
2823
|
+
appendFileQueued,
|
|
2824
|
+
configureFsIO,
|
|
2825
|
+
createAnthropicBatchAdapter,
|
|
2096
2826
|
createAnyModelServer,
|
|
2827
|
+
createOpenAIBatchAdapter,
|
|
2828
|
+
ensureDir,
|
|
2829
|
+
getFsQueueStatus,
|
|
2830
|
+
joinPath,
|
|
2831
|
+
readFileQueued,
|
|
2097
2832
|
resolveConfig,
|
|
2098
|
-
startServer
|
|
2833
|
+
startServer,
|
|
2834
|
+
waitForFsQueuesIdle,
|
|
2835
|
+
writeFileFlushedQueued,
|
|
2836
|
+
writeFileQueued
|
|
2099
2837
|
});
|
|
2100
2838
|
//# sourceMappingURL=index.cjs.map
|