@probeo/anymodel 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -3
- package/dist/cli.cjs +392 -30
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +392 -30
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +401 -30
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +24 -1
- package/dist/index.d.ts +24 -1
- package/dist/index.js +398 -30
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -480,6 +480,25 @@ var Router = class {
|
|
|
480
480
|
}
|
|
481
481
|
};
|
|
482
482
|
|
|
483
|
+
// src/utils/fetch-with-timeout.ts
|
|
484
|
+
var _defaultTimeout = 12e4;
|
|
485
|
+
var _flexTimeout = 6e5;
|
|
486
|
+
function setDefaultTimeout(ms) {
|
|
487
|
+
_defaultTimeout = ms;
|
|
488
|
+
}
|
|
489
|
+
function getFlexTimeout() {
|
|
490
|
+
return _flexTimeout;
|
|
491
|
+
}
|
|
492
|
+
function fetchWithTimeout(url, init, timeoutMs) {
|
|
493
|
+
const ms = timeoutMs ?? _defaultTimeout;
|
|
494
|
+
const signal = AbortSignal.timeout(ms);
|
|
495
|
+
if (init?.signal) {
|
|
496
|
+
const combined = AbortSignal.any([signal, init.signal]);
|
|
497
|
+
return fetch(url, { ...init, signal: combined });
|
|
498
|
+
}
|
|
499
|
+
return fetch(url, { ...init, signal });
|
|
500
|
+
}
|
|
501
|
+
|
|
483
502
|
// src/providers/openai.ts
|
|
484
503
|
var OPENAI_API_BASE = "https://api.openai.com/v1";
|
|
485
504
|
var SUPPORTED_PARAMS = /* @__PURE__ */ new Set([
|
|
@@ -497,19 +516,20 @@ var SUPPORTED_PARAMS = /* @__PURE__ */ new Set([
|
|
|
497
516
|
"tools",
|
|
498
517
|
"tool_choice",
|
|
499
518
|
"user",
|
|
500
|
-
"logit_bias"
|
|
519
|
+
"logit_bias",
|
|
520
|
+
"service_tier"
|
|
501
521
|
]);
|
|
502
522
|
function createOpenAIAdapter(apiKey, baseURL) {
|
|
503
523
|
const base = baseURL || OPENAI_API_BASE;
|
|
504
|
-
async function makeRequest(path2, body, method = "POST") {
|
|
505
|
-
const res = await
|
|
524
|
+
async function makeRequest(path2, body, method = "POST", timeoutMs) {
|
|
525
|
+
const res = await fetchWithTimeout(`${base}${path2}`, {
|
|
506
526
|
method,
|
|
507
527
|
headers: {
|
|
508
528
|
"Content-Type": "application/json",
|
|
509
529
|
"Authorization": `Bearer ${apiKey}`
|
|
510
530
|
},
|
|
511
531
|
body: body ? JSON.stringify(body) : void 0
|
|
512
|
-
});
|
|
532
|
+
}, timeoutMs);
|
|
513
533
|
if (!res.ok) {
|
|
514
534
|
let errorBody;
|
|
515
535
|
try {
|
|
@@ -557,6 +577,7 @@ function createOpenAIAdapter(apiKey, baseURL) {
|
|
|
557
577
|
if (request.tools !== void 0) body.tools = request.tools;
|
|
558
578
|
if (request.tool_choice !== void 0) body.tool_choice = request.tool_choice;
|
|
559
579
|
if (request.user !== void 0) body.user = request.user;
|
|
580
|
+
if (request.service_tier !== void 0) body.service_tier = request.service_tier;
|
|
560
581
|
return body;
|
|
561
582
|
}
|
|
562
583
|
const adapter = {
|
|
@@ -658,13 +679,15 @@ function createOpenAIAdapter(apiKey, baseURL) {
|
|
|
658
679
|
},
|
|
659
680
|
async sendRequest(request) {
|
|
660
681
|
const body = buildRequestBody(request);
|
|
661
|
-
const
|
|
682
|
+
const timeout = request.service_tier === "flex" ? getFlexTimeout() : void 0;
|
|
683
|
+
const res = await makeRequest("/chat/completions", body, "POST", timeout);
|
|
662
684
|
const json = await res.json();
|
|
663
685
|
return adapter.translateResponse(json);
|
|
664
686
|
},
|
|
665
687
|
async sendStreamingRequest(request) {
|
|
666
688
|
const body = buildRequestBody({ ...request, stream: true });
|
|
667
|
-
const
|
|
689
|
+
const timeout = request.service_tier === "flex" ? getFlexTimeout() : void 0;
|
|
690
|
+
const res = await makeRequest("/chat/completions", body, "POST", timeout);
|
|
668
691
|
if (!res.body) {
|
|
669
692
|
throw new AnyModelError(502, "No response body for streaming request", {
|
|
670
693
|
provider_name: "openai"
|
|
@@ -711,7 +734,7 @@ var FALLBACK_MODELS = [
|
|
|
711
734
|
];
|
|
712
735
|
function createAnthropicAdapter(apiKey) {
|
|
713
736
|
async function makeRequest(path2, body, stream = false) {
|
|
714
|
-
const res = await
|
|
737
|
+
const res = await fetchWithTimeout(`${ANTHROPIC_API_BASE}${path2}`, {
|
|
715
738
|
method: "POST",
|
|
716
739
|
headers: {
|
|
717
740
|
"Content-Type": "application/json",
|
|
@@ -968,7 +991,7 @@ ${body.system}` : jsonInstruction;
|
|
|
968
991
|
},
|
|
969
992
|
async listModels() {
|
|
970
993
|
try {
|
|
971
|
-
const res = await
|
|
994
|
+
const res = await fetchWithTimeout(`${ANTHROPIC_API_BASE}/models`, {
|
|
972
995
|
method: "GET",
|
|
973
996
|
headers: {
|
|
974
997
|
"x-api-key": apiKey,
|
|
@@ -1253,7 +1276,7 @@ function createGoogleAdapter(apiKey) {
|
|
|
1253
1276
|
},
|
|
1254
1277
|
async listModels() {
|
|
1255
1278
|
try {
|
|
1256
|
-
const res = await
|
|
1279
|
+
const res = await fetchWithTimeout(`${GEMINI_API_BASE}/models?key=${apiKey}`);
|
|
1257
1280
|
if (!res.ok) return FALLBACK_MODELS2;
|
|
1258
1281
|
const data = await res.json();
|
|
1259
1282
|
const models = data.models || [];
|
|
@@ -1288,12 +1311,12 @@ function createGoogleAdapter(apiKey) {
|
|
|
1288
1311
|
return SUPPORTED_PARAMS3.has(param);
|
|
1289
1312
|
},
|
|
1290
1313
|
supportsBatch() {
|
|
1291
|
-
return
|
|
1314
|
+
return true;
|
|
1292
1315
|
},
|
|
1293
1316
|
async sendRequest(request) {
|
|
1294
1317
|
const body = translateRequest(request);
|
|
1295
1318
|
const url = getModelEndpoint(request.model, false);
|
|
1296
|
-
const res = await
|
|
1319
|
+
const res = await fetchWithTimeout(url, {
|
|
1297
1320
|
method: "POST",
|
|
1298
1321
|
headers: { "Content-Type": "application/json" },
|
|
1299
1322
|
body: JSON.stringify(body)
|
|
@@ -1316,7 +1339,7 @@ function createGoogleAdapter(apiKey) {
|
|
|
1316
1339
|
async sendStreamingRequest(request) {
|
|
1317
1340
|
const body = translateRequest(request);
|
|
1318
1341
|
const url = getModelEndpoint(request.model, true);
|
|
1319
|
-
const res = await
|
|
1342
|
+
const res = await fetchWithTimeout(url, {
|
|
1320
1343
|
method: "POST",
|
|
1321
1344
|
headers: { "Content-Type": "application/json" },
|
|
1322
1345
|
body: JSON.stringify(body)
|
|
@@ -1366,7 +1389,7 @@ var MODELS = [
|
|
|
1366
1389
|
];
|
|
1367
1390
|
function createPerplexityAdapter(apiKey) {
|
|
1368
1391
|
async function makeRequest(path2, body, method = "POST") {
|
|
1369
|
-
const res = await
|
|
1392
|
+
const res = await fetchWithTimeout(`${PERPLEXITY_API_BASE}${path2}`, {
|
|
1370
1393
|
method,
|
|
1371
1394
|
headers: {
|
|
1372
1395
|
"Content-Type": "application/json",
|
|
@@ -1930,6 +1953,17 @@ var BatchStore = class {
|
|
|
1930
1953
|
const entries = await readDirQueued(this.dir);
|
|
1931
1954
|
return entries.filter((d) => d.isDirectory()).map((d) => d.name).sort();
|
|
1932
1955
|
}
|
|
1956
|
+
/**
|
|
1957
|
+
* Stream requests from JSONL one line at a time (memory-efficient).
|
|
1958
|
+
*/
|
|
1959
|
+
async *streamRequests(id) {
|
|
1960
|
+
const p = joinPath(this.batchDir(id), "requests.jsonl");
|
|
1961
|
+
if (!await fileExistsQueued(p)) return;
|
|
1962
|
+
const raw = await readFileQueued(p, "utf8");
|
|
1963
|
+
for (const line of raw.split("\n")) {
|
|
1964
|
+
if (line.trim()) yield JSON.parse(line);
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1933
1967
|
/**
|
|
1934
1968
|
* Check if a batch exists.
|
|
1935
1969
|
*/
|
|
@@ -1994,7 +2028,7 @@ var BatchManager = class {
|
|
|
1994
2028
|
this.processNativeBatch(id, request, native.adapter).catch(() => {
|
|
1995
2029
|
});
|
|
1996
2030
|
} else {
|
|
1997
|
-
this.processConcurrentBatch(id, request).catch(() => {
|
|
2031
|
+
this.processConcurrentBatch(id, request.model, request.options).catch(() => {
|
|
1998
2032
|
});
|
|
1999
2033
|
}
|
|
2000
2034
|
return batch;
|
|
@@ -2174,28 +2208,28 @@ var BatchManager = class {
|
|
|
2174
2208
|
}
|
|
2175
2209
|
/**
|
|
2176
2210
|
* Process batch requests concurrently (fallback path).
|
|
2211
|
+
* Streams requests from disk to avoid holding them all in memory.
|
|
2177
2212
|
*/
|
|
2178
|
-
async processConcurrentBatch(batchId,
|
|
2213
|
+
async processConcurrentBatch(batchId, model, options) {
|
|
2179
2214
|
const batch = await this.store.getMeta(batchId);
|
|
2180
2215
|
if (!batch) return;
|
|
2181
2216
|
batch.status = "processing";
|
|
2182
2217
|
await this.store.updateMeta(batch);
|
|
2183
|
-
const items = request.requests;
|
|
2184
2218
|
const active = /* @__PURE__ */ new Set();
|
|
2185
2219
|
const processItem = async (item) => {
|
|
2186
2220
|
const current = await this.store.getMeta(batchId);
|
|
2187
2221
|
if (current?.status === "cancelled") return;
|
|
2188
2222
|
const chatRequest = {
|
|
2189
|
-
model
|
|
2223
|
+
model,
|
|
2190
2224
|
messages: item.messages,
|
|
2191
|
-
max_tokens: item.max_tokens ??
|
|
2192
|
-
temperature: item.temperature ??
|
|
2193
|
-
top_p: item.top_p ??
|
|
2194
|
-
top_k: item.top_k ??
|
|
2195
|
-
stop: item.stop ??
|
|
2196
|
-
response_format: item.response_format ??
|
|
2197
|
-
tools: item.tools ??
|
|
2198
|
-
tool_choice: item.tool_choice ??
|
|
2225
|
+
max_tokens: item.max_tokens ?? options?.max_tokens,
|
|
2226
|
+
temperature: item.temperature ?? options?.temperature,
|
|
2227
|
+
top_p: item.top_p ?? options?.top_p,
|
|
2228
|
+
top_k: item.top_k ?? options?.top_k,
|
|
2229
|
+
stop: item.stop ?? options?.stop,
|
|
2230
|
+
response_format: item.response_format ?? options?.response_format,
|
|
2231
|
+
tools: item.tools ?? options?.tools,
|
|
2232
|
+
tool_choice: item.tool_choice ?? options?.tool_choice
|
|
2199
2233
|
};
|
|
2200
2234
|
let result;
|
|
2201
2235
|
try {
|
|
@@ -2226,7 +2260,7 @@ var BatchManager = class {
|
|
|
2226
2260
|
await this.store.updateMeta(meta);
|
|
2227
2261
|
}
|
|
2228
2262
|
};
|
|
2229
|
-
for (const item of
|
|
2263
|
+
for await (const item of this.store.streamRequests(batchId)) {
|
|
2230
2264
|
const current = await this.store.getMeta(batchId);
|
|
2231
2265
|
if (current?.status === "cancelled") break;
|
|
2232
2266
|
if (active.size >= this.concurrencyLimit) {
|
|
@@ -2247,6 +2281,54 @@ var BatchManager = class {
|
|
|
2247
2281
|
}
|
|
2248
2282
|
};
|
|
2249
2283
|
|
|
2284
|
+
// src/utils/token-estimate.ts
|
|
2285
|
+
var CHARS_PER_TOKEN2 = 4;
|
|
2286
|
+
function estimateTokenCount(text) {
|
|
2287
|
+
return Math.ceil(text.length / CHARS_PER_TOKEN2);
|
|
2288
|
+
}
|
|
2289
|
+
var MODEL_LIMITS = [
|
|
2290
|
+
// OpenAI
|
|
2291
|
+
{ pattern: "gpt-4o-mini", limit: { contextLength: 128e3, maxCompletionTokens: 16384 } },
|
|
2292
|
+
{ pattern: "gpt-4o", limit: { contextLength: 128e3, maxCompletionTokens: 16384 } },
|
|
2293
|
+
{ pattern: "gpt-4-turbo", limit: { contextLength: 128e3, maxCompletionTokens: 4096 } },
|
|
2294
|
+
{ pattern: "gpt-3.5-turbo", limit: { contextLength: 16385, maxCompletionTokens: 4096 } },
|
|
2295
|
+
{ pattern: "o1", limit: { contextLength: 2e5, maxCompletionTokens: 1e5 } },
|
|
2296
|
+
{ pattern: "o3", limit: { contextLength: 2e5, maxCompletionTokens: 1e5 } },
|
|
2297
|
+
{ pattern: "o4-mini", limit: { contextLength: 2e5, maxCompletionTokens: 1e5 } },
|
|
2298
|
+
// Anthropic
|
|
2299
|
+
{ pattern: "claude-opus-4", limit: { contextLength: 2e5, maxCompletionTokens: 32768 } },
|
|
2300
|
+
{ pattern: "claude-sonnet-4", limit: { contextLength: 2e5, maxCompletionTokens: 16384 } },
|
|
2301
|
+
{ pattern: "claude-haiku-4", limit: { contextLength: 2e5, maxCompletionTokens: 8192 } },
|
|
2302
|
+
{ pattern: "claude-3.5-sonnet", limit: { contextLength: 2e5, maxCompletionTokens: 8192 } },
|
|
2303
|
+
{ pattern: "claude-3-opus", limit: { contextLength: 2e5, maxCompletionTokens: 4096 } },
|
|
2304
|
+
// Google
|
|
2305
|
+
{ pattern: "gemini-2.5-pro", limit: { contextLength: 1048576, maxCompletionTokens: 65536 } },
|
|
2306
|
+
{ pattern: "gemini-2.5-flash", limit: { contextLength: 1048576, maxCompletionTokens: 65536 } },
|
|
2307
|
+
{ pattern: "gemini-2.0-flash", limit: { contextLength: 1048576, maxCompletionTokens: 65536 } },
|
|
2308
|
+
{ pattern: "gemini-1.5-pro", limit: { contextLength: 2097152, maxCompletionTokens: 8192 } },
|
|
2309
|
+
{ pattern: "gemini-1.5-flash", limit: { contextLength: 1048576, maxCompletionTokens: 8192 } }
|
|
2310
|
+
];
|
|
2311
|
+
var DEFAULT_LIMIT = { contextLength: 128e3, maxCompletionTokens: 4096 };
|
|
2312
|
+
function getModelLimits(model) {
|
|
2313
|
+
const bare = model.includes("/") ? model.slice(model.indexOf("/") + 1) : model;
|
|
2314
|
+
for (const entry of MODEL_LIMITS) {
|
|
2315
|
+
if (bare.startsWith(entry.pattern) || bare.includes(entry.pattern)) {
|
|
2316
|
+
return entry.limit;
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
return DEFAULT_LIMIT;
|
|
2320
|
+
}
|
|
2321
|
+
function resolveMaxTokens(model, messages, userMaxTokens) {
|
|
2322
|
+
if (userMaxTokens !== void 0) return userMaxTokens;
|
|
2323
|
+
const inputChars = JSON.stringify(messages).length;
|
|
2324
|
+
const estimatedInput = Math.ceil(inputChars / CHARS_PER_TOKEN2);
|
|
2325
|
+
const estimatedWithMargin = Math.ceil(estimatedInput * 1.05);
|
|
2326
|
+
const limits = getModelLimits(model);
|
|
2327
|
+
const available = limits.contextLength - estimatedWithMargin;
|
|
2328
|
+
const result = Math.min(limits.maxCompletionTokens, available);
|
|
2329
|
+
return Math.max(1, result);
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2250
2332
|
// src/providers/openai-batch.ts
|
|
2251
2333
|
var OPENAI_API_BASE2 = "https://api.openai.com/v1";
|
|
2252
2334
|
function createOpenAIBatchAdapter(apiKey) {
|
|
@@ -2261,7 +2343,7 @@ function createOpenAIBatchAdapter(apiKey) {
|
|
|
2261
2343
|
headers["Content-Type"] = "application/json";
|
|
2262
2344
|
fetchBody = JSON.stringify(options.body);
|
|
2263
2345
|
}
|
|
2264
|
-
const res = await
|
|
2346
|
+
const res = await fetchWithTimeout(`${OPENAI_API_BASE2}${path2}`, {
|
|
2265
2347
|
method: options.method || "GET",
|
|
2266
2348
|
headers,
|
|
2267
2349
|
body: fetchBody
|
|
@@ -2287,7 +2369,7 @@ function createOpenAIBatchAdapter(apiKey) {
|
|
|
2287
2369
|
model,
|
|
2288
2370
|
messages: req.messages
|
|
2289
2371
|
};
|
|
2290
|
-
|
|
2372
|
+
body.max_tokens = req.max_tokens !== void 0 ? req.max_tokens : resolveMaxTokens(model, req.messages);
|
|
2291
2373
|
if (req.temperature !== void 0) body.temperature = req.temperature;
|
|
2292
2374
|
if (req.top_p !== void 0) body.top_p = req.top_p;
|
|
2293
2375
|
if (req.stop !== void 0) body.stop = req.stop;
|
|
@@ -2446,7 +2528,7 @@ function createAnthropicBatchAdapter(apiKey) {
|
|
|
2446
2528
|
"anthropic-version": ANTHROPIC_VERSION2,
|
|
2447
2529
|
"Content-Type": "application/json"
|
|
2448
2530
|
};
|
|
2449
|
-
const res = await
|
|
2531
|
+
const res = await fetchWithTimeout(`${ANTHROPIC_API_BASE2}${path2}`, {
|
|
2450
2532
|
method: options.method || "GET",
|
|
2451
2533
|
headers,
|
|
2452
2534
|
body: options.body ? JSON.stringify(options.body) : void 0
|
|
@@ -2469,7 +2551,7 @@ function createAnthropicBatchAdapter(apiKey) {
|
|
|
2469
2551
|
function translateToAnthropicParams(model, req) {
|
|
2470
2552
|
const params = {
|
|
2471
2553
|
model,
|
|
2472
|
-
max_tokens: req.max_tokens || DEFAULT_MAX_TOKENS2
|
|
2554
|
+
max_tokens: resolveMaxTokens(model, req.messages, req.max_tokens || DEFAULT_MAX_TOKENS2)
|
|
2473
2555
|
};
|
|
2474
2556
|
const systemMessages = req.messages.filter((m) => m.role === "system");
|
|
2475
2557
|
const nonSystemMessages = req.messages.filter((m) => m.role !== "system");
|
|
@@ -2643,6 +2725,284 @@ ${params.system}` : jsonInstruction;
|
|
|
2643
2725
|
};
|
|
2644
2726
|
}
|
|
2645
2727
|
|
|
2728
|
+
// src/providers/google-batch.ts
|
|
2729
|
+
var GEMINI_API_BASE2 = "https://generativelanguage.googleapis.com/v1beta";
|
|
2730
|
+
function createGoogleBatchAdapter(apiKey) {
|
|
2731
|
+
async function apiRequest(path2, options = {}) {
|
|
2732
|
+
const headers = {
|
|
2733
|
+
"Content-Type": "application/json",
|
|
2734
|
+
"x-goog-api-key": apiKey
|
|
2735
|
+
};
|
|
2736
|
+
const res = await fetchWithTimeout(`${GEMINI_API_BASE2}${path2}`, {
|
|
2737
|
+
method: options.method || "GET",
|
|
2738
|
+
headers,
|
|
2739
|
+
body: options.body ? JSON.stringify(options.body) : void 0
|
|
2740
|
+
});
|
|
2741
|
+
if (!res.ok) {
|
|
2742
|
+
let errorBody;
|
|
2743
|
+
try {
|
|
2744
|
+
errorBody = await res.json();
|
|
2745
|
+
} catch {
|
|
2746
|
+
errorBody = { message: res.statusText };
|
|
2747
|
+
}
|
|
2748
|
+
const msg = errorBody?.error?.message || errorBody?.message || res.statusText;
|
|
2749
|
+
throw new AnyModelError(res.status >= 500 ? 502 : res.status, msg, {
|
|
2750
|
+
provider_name: "google",
|
|
2751
|
+
raw: errorBody
|
|
2752
|
+
});
|
|
2753
|
+
}
|
|
2754
|
+
return res;
|
|
2755
|
+
}
|
|
2756
|
+
function translateRequestToGemini(model, req) {
|
|
2757
|
+
const body = {};
|
|
2758
|
+
const systemMessages = req.messages.filter((m) => m.role === "system");
|
|
2759
|
+
const nonSystemMessages = req.messages.filter((m) => m.role !== "system");
|
|
2760
|
+
if (systemMessages.length > 0) {
|
|
2761
|
+
body.systemInstruction = {
|
|
2762
|
+
parts: [{ text: systemMessages.map((m) => typeof m.content === "string" ? m.content : "").join("\n") }]
|
|
2763
|
+
};
|
|
2764
|
+
}
|
|
2765
|
+
body.contents = nonSystemMessages.map((m) => ({
|
|
2766
|
+
role: m.role === "assistant" ? "model" : "user",
|
|
2767
|
+
parts: typeof m.content === "string" ? [{ text: m.content }] : Array.isArray(m.content) ? m.content.map((p) => p.type === "text" ? { text: p.text } : { text: "" }) : [{ text: "" }]
|
|
2768
|
+
}));
|
|
2769
|
+
const generationConfig = {};
|
|
2770
|
+
if (req.temperature !== void 0) generationConfig.temperature = req.temperature;
|
|
2771
|
+
generationConfig.maxOutputTokens = req.max_tokens !== void 0 ? req.max_tokens : resolveMaxTokens(model, req.messages);
|
|
2772
|
+
if (req.top_p !== void 0) generationConfig.topP = req.top_p;
|
|
2773
|
+
if (req.top_k !== void 0) generationConfig.topK = req.top_k;
|
|
2774
|
+
if (req.stop !== void 0) {
|
|
2775
|
+
generationConfig.stopSequences = Array.isArray(req.stop) ? req.stop : [req.stop];
|
|
2776
|
+
}
|
|
2777
|
+
if (req.response_format) {
|
|
2778
|
+
if (req.response_format.type === "json_object") {
|
|
2779
|
+
generationConfig.responseMimeType = "application/json";
|
|
2780
|
+
} else if (req.response_format.type === "json_schema") {
|
|
2781
|
+
generationConfig.responseMimeType = "application/json";
|
|
2782
|
+
generationConfig.responseSchema = req.response_format.json_schema?.schema;
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2785
|
+
if (Object.keys(generationConfig).length > 0) {
|
|
2786
|
+
body.generationConfig = generationConfig;
|
|
2787
|
+
}
|
|
2788
|
+
if (req.tools && req.tools.length > 0) {
|
|
2789
|
+
body.tools = [{
|
|
2790
|
+
functionDeclarations: req.tools.map((t) => ({
|
|
2791
|
+
name: t.function.name,
|
|
2792
|
+
description: t.function.description || "",
|
|
2793
|
+
parameters: t.function.parameters || {}
|
|
2794
|
+
}))
|
|
2795
|
+
}];
|
|
2796
|
+
if (req.tool_choice) {
|
|
2797
|
+
if (req.tool_choice === "auto") {
|
|
2798
|
+
body.toolConfig = { functionCallingConfig: { mode: "AUTO" } };
|
|
2799
|
+
} else if (req.tool_choice === "required") {
|
|
2800
|
+
body.toolConfig = { functionCallingConfig: { mode: "ANY" } };
|
|
2801
|
+
} else if (req.tool_choice === "none") {
|
|
2802
|
+
body.toolConfig = { functionCallingConfig: { mode: "NONE" } };
|
|
2803
|
+
} else if (typeof req.tool_choice === "object") {
|
|
2804
|
+
body.toolConfig = {
|
|
2805
|
+
functionCallingConfig: {
|
|
2806
|
+
mode: "ANY",
|
|
2807
|
+
allowedFunctionNames: [req.tool_choice.function.name]
|
|
2808
|
+
}
|
|
2809
|
+
};
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2812
|
+
}
|
|
2813
|
+
return body;
|
|
2814
|
+
}
|
|
2815
|
+
function mapFinishReason(reason) {
|
|
2816
|
+
switch (reason) {
|
|
2817
|
+
case "STOP":
|
|
2818
|
+
return "stop";
|
|
2819
|
+
case "MAX_TOKENS":
|
|
2820
|
+
return "length";
|
|
2821
|
+
case "SAFETY":
|
|
2822
|
+
return "content_filter";
|
|
2823
|
+
case "RECITATION":
|
|
2824
|
+
return "content_filter";
|
|
2825
|
+
default:
|
|
2826
|
+
return "stop";
|
|
2827
|
+
}
|
|
2828
|
+
}
|
|
2829
|
+
function translateGeminiResponse(response, model) {
|
|
2830
|
+
const candidate = response.candidates?.[0];
|
|
2831
|
+
let content = "";
|
|
2832
|
+
const toolCalls = [];
|
|
2833
|
+
for (const part of candidate?.content?.parts || []) {
|
|
2834
|
+
if (part.text) {
|
|
2835
|
+
content += part.text;
|
|
2836
|
+
} else if (part.functionCall) {
|
|
2837
|
+
toolCalls.push({
|
|
2838
|
+
id: generateId("call"),
|
|
2839
|
+
type: "function",
|
|
2840
|
+
function: {
|
|
2841
|
+
name: part.functionCall.name,
|
|
2842
|
+
arguments: JSON.stringify(part.functionCall.args || {})
|
|
2843
|
+
}
|
|
2844
|
+
});
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
const message = { role: "assistant", content };
|
|
2848
|
+
if (toolCalls.length > 0) {
|
|
2849
|
+
message.tool_calls = toolCalls;
|
|
2850
|
+
}
|
|
2851
|
+
const finishReason = toolCalls.length > 0 ? "tool_calls" : mapFinishReason(candidate?.finishReason || "STOP");
|
|
2852
|
+
return {
|
|
2853
|
+
id: generateId(),
|
|
2854
|
+
object: "chat.completion",
|
|
2855
|
+
created: Math.floor(Date.now() / 1e3),
|
|
2856
|
+
model: `google/${model}`,
|
|
2857
|
+
choices: [{ index: 0, message, finish_reason: finishReason }],
|
|
2858
|
+
usage: {
|
|
2859
|
+
prompt_tokens: response.usageMetadata?.promptTokenCount || 0,
|
|
2860
|
+
completion_tokens: response.usageMetadata?.candidatesTokenCount || 0,
|
|
2861
|
+
total_tokens: response.usageMetadata?.totalTokenCount || 0
|
|
2862
|
+
}
|
|
2863
|
+
};
|
|
2864
|
+
}
|
|
2865
|
+
function mapBatchState(state) {
|
|
2866
|
+
switch (state) {
|
|
2867
|
+
case "JOB_STATE_PENDING":
|
|
2868
|
+
return "pending";
|
|
2869
|
+
case "JOB_STATE_RUNNING":
|
|
2870
|
+
return "processing";
|
|
2871
|
+
case "JOB_STATE_SUCCEEDED":
|
|
2872
|
+
return "completed";
|
|
2873
|
+
case "JOB_STATE_FAILED":
|
|
2874
|
+
return "failed";
|
|
2875
|
+
case "JOB_STATE_CANCELLED":
|
|
2876
|
+
return "cancelled";
|
|
2877
|
+
case "JOB_STATE_EXPIRED":
|
|
2878
|
+
return "failed";
|
|
2879
|
+
default:
|
|
2880
|
+
return "pending";
|
|
2881
|
+
}
|
|
2882
|
+
}
|
|
2883
|
+
return {
|
|
2884
|
+
async createBatch(model, requests, _options) {
|
|
2885
|
+
const batchRequests = requests.map((req) => ({
|
|
2886
|
+
request: translateRequestToGemini(model, req),
|
|
2887
|
+
metadata: { key: req.custom_id }
|
|
2888
|
+
}));
|
|
2889
|
+
const res = await apiRequest(`/models/${model}:batchGenerateContent`, {
|
|
2890
|
+
method: "POST",
|
|
2891
|
+
body: {
|
|
2892
|
+
batch: {
|
|
2893
|
+
display_name: `anymodel-batch-${Date.now()}`,
|
|
2894
|
+
input_config: {
|
|
2895
|
+
requests: {
|
|
2896
|
+
requests: batchRequests
|
|
2897
|
+
}
|
|
2898
|
+
}
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
});
|
|
2902
|
+
const data = await res.json();
|
|
2903
|
+
const batchName = data.name || data.batch?.name;
|
|
2904
|
+
if (!batchName) {
|
|
2905
|
+
throw new AnyModelError(502, "No batch name in Google response", {
|
|
2906
|
+
provider_name: "google",
|
|
2907
|
+
raw: data
|
|
2908
|
+
});
|
|
2909
|
+
}
|
|
2910
|
+
return {
|
|
2911
|
+
providerBatchId: batchName,
|
|
2912
|
+
metadata: {
|
|
2913
|
+
model,
|
|
2914
|
+
total_requests: requests.length
|
|
2915
|
+
}
|
|
2916
|
+
};
|
|
2917
|
+
},
|
|
2918
|
+
async pollBatch(providerBatchId) {
|
|
2919
|
+
const res = await apiRequest(`/${providerBatchId}`);
|
|
2920
|
+
const data = await res.json();
|
|
2921
|
+
const state = data.state || "JOB_STATE_PENDING";
|
|
2922
|
+
const status = mapBatchState(state);
|
|
2923
|
+
const totalCount = data.totalCount || data.metadata?.total_requests || 0;
|
|
2924
|
+
const successCount = data.succeededCount || 0;
|
|
2925
|
+
const failedCount = data.failedCount || 0;
|
|
2926
|
+
return {
|
|
2927
|
+
status,
|
|
2928
|
+
total: totalCount || successCount + failedCount,
|
|
2929
|
+
completed: successCount,
|
|
2930
|
+
failed: failedCount
|
|
2931
|
+
};
|
|
2932
|
+
},
|
|
2933
|
+
async getBatchResults(providerBatchId) {
|
|
2934
|
+
const batchRes = await apiRequest(`/${providerBatchId}`);
|
|
2935
|
+
const batchData = await batchRes.json();
|
|
2936
|
+
const results = [];
|
|
2937
|
+
const model = batchData.metadata?.model || "unknown";
|
|
2938
|
+
if (batchData.response?.inlinedResponses) {
|
|
2939
|
+
for (const item of batchData.response.inlinedResponses) {
|
|
2940
|
+
const customId = item.metadata?.key || `request-${results.length}`;
|
|
2941
|
+
if (item.response) {
|
|
2942
|
+
results.push({
|
|
2943
|
+
custom_id: customId,
|
|
2944
|
+
status: "success",
|
|
2945
|
+
response: translateGeminiResponse(item.response, model),
|
|
2946
|
+
error: null
|
|
2947
|
+
});
|
|
2948
|
+
} else if (item.error) {
|
|
2949
|
+
results.push({
|
|
2950
|
+
custom_id: customId,
|
|
2951
|
+
status: "error",
|
|
2952
|
+
response: null,
|
|
2953
|
+
error: {
|
|
2954
|
+
code: item.error.code || 500,
|
|
2955
|
+
message: item.error.message || "Batch item failed"
|
|
2956
|
+
}
|
|
2957
|
+
});
|
|
2958
|
+
}
|
|
2959
|
+
}
|
|
2960
|
+
return results;
|
|
2961
|
+
}
|
|
2962
|
+
const responsesFile = batchData.response?.responsesFileName || batchData.outputConfig?.file_name;
|
|
2963
|
+
if (responsesFile) {
|
|
2964
|
+
const downloadUrl = `${GEMINI_API_BASE2}/${responsesFile}:download?alt=media`;
|
|
2965
|
+
const fileRes = await fetchWithTimeout(downloadUrl, {
|
|
2966
|
+
headers: { "x-goog-api-key": apiKey }
|
|
2967
|
+
});
|
|
2968
|
+
if (!fileRes.ok) {
|
|
2969
|
+
throw new AnyModelError(502, "Failed to download batch results file", {
|
|
2970
|
+
provider_name: "google"
|
|
2971
|
+
});
|
|
2972
|
+
}
|
|
2973
|
+
const text = await fileRes.text();
|
|
2974
|
+
for (const line of text.trim().split("\n")) {
|
|
2975
|
+
if (!line) continue;
|
|
2976
|
+
const item = JSON.parse(line);
|
|
2977
|
+
const customId = item.key || item.metadata?.key || `request-${results.length}`;
|
|
2978
|
+
if (item.response) {
|
|
2979
|
+
results.push({
|
|
2980
|
+
custom_id: customId,
|
|
2981
|
+
status: "success",
|
|
2982
|
+
response: translateGeminiResponse(item.response, model),
|
|
2983
|
+
error: null
|
|
2984
|
+
});
|
|
2985
|
+
} else if (item.error) {
|
|
2986
|
+
results.push({
|
|
2987
|
+
custom_id: customId,
|
|
2988
|
+
status: "error",
|
|
2989
|
+
response: null,
|
|
2990
|
+
error: {
|
|
2991
|
+
code: item.error.code || 500,
|
|
2992
|
+
message: item.error.message || "Batch item failed"
|
|
2993
|
+
}
|
|
2994
|
+
});
|
|
2995
|
+
}
|
|
2996
|
+
}
|
|
2997
|
+
}
|
|
2998
|
+
return results;
|
|
2999
|
+
},
|
|
3000
|
+
async cancelBatch(providerBatchId) {
|
|
3001
|
+
await apiRequest(`/${providerBatchId}:cancel`, { method: "POST" });
|
|
3002
|
+
}
|
|
3003
|
+
};
|
|
3004
|
+
}
|
|
3005
|
+
|
|
2646
3006
|
// src/client.ts
|
|
2647
3007
|
var AnyModel = class {
|
|
2648
3008
|
registry;
|
|
@@ -2658,6 +3018,7 @@ var AnyModel = class {
|
|
|
2658
3018
|
constructor(config = {}) {
|
|
2659
3019
|
this.config = resolveConfig(config);
|
|
2660
3020
|
this.registry = new ProviderRegistry();
|
|
3021
|
+
setDefaultTimeout((this.config.defaults?.timeout ?? 120) * 1e3);
|
|
2661
3022
|
if (this.config.io) {
|
|
2662
3023
|
configureFsIO(this.config.io);
|
|
2663
3024
|
}
|
|
@@ -2778,6 +3139,10 @@ var AnyModel = class {
|
|
|
2778
3139
|
if (anthropicKey) {
|
|
2779
3140
|
this.batchManager.registerBatchAdapter("anthropic", createAnthropicBatchAdapter(anthropicKey));
|
|
2780
3141
|
}
|
|
3142
|
+
const googleKey = config.google?.apiKey || process.env.GOOGLE_API_KEY;
|
|
3143
|
+
if (googleKey) {
|
|
3144
|
+
this.batchManager.registerBatchAdapter("google", createGoogleBatchAdapter(googleKey));
|
|
3145
|
+
}
|
|
2781
3146
|
}
|
|
2782
3147
|
applyDefaults(request) {
|
|
2783
3148
|
const defaults = this.config.defaults;
|
|
@@ -2956,12 +3321,15 @@ export {
|
|
|
2956
3321
|
configureFsIO,
|
|
2957
3322
|
createAnthropicBatchAdapter,
|
|
2958
3323
|
createAnyModelServer,
|
|
3324
|
+
createGoogleBatchAdapter,
|
|
2959
3325
|
createOpenAIBatchAdapter,
|
|
2960
3326
|
ensureDir,
|
|
3327
|
+
estimateTokenCount,
|
|
2961
3328
|
getFsQueueStatus,
|
|
2962
3329
|
joinPath,
|
|
2963
3330
|
readFileQueued,
|
|
2964
3331
|
resolveConfig,
|
|
3332
|
+
resolveMaxTokens,
|
|
2965
3333
|
startServer,
|
|
2966
3334
|
waitForFsQueuesIdle,
|
|
2967
3335
|
writeFileFlushedQueued,
|