@chainfuse/ai-tools 0.13.1 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import type { GoogleGenerativeAIProvider } from '@ai-sdk/google';
|
|
2
2
|
import { AiBase } from '../base.mjs';
|
|
3
|
-
import type { AiRequestConfig
|
|
4
|
-
import type { AzureOpenAIProvider } from './types.mjs';
|
|
3
|
+
import type { AiRequestConfig } from '../types.mjs';
|
|
5
4
|
export declare class AiCustomProviders extends AiBase {
|
|
6
5
|
oaiOpenai(args: AiRequestConfig): Promise<import("@ai-sdk/openai").OpenAIProvider>;
|
|
7
|
-
azOpenai(args: AiRequestConfig
|
|
6
|
+
azOpenai(args: AiRequestConfig): Promise<import("./types.mts").AzureOpenAIProvider>;
|
|
8
7
|
anthropic(args: AiRequestConfig): Promise<import("@ai-sdk/anthropic").AnthropicProvider>;
|
|
9
8
|
private static workersAiIsRest;
|
|
10
|
-
cfWorkersAi(args: AiRequestConfig): Promise<import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<"@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", "@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct",
|
|
9
|
+
cfWorkersAi(args: AiRequestConfig): Promise<import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<"@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", "@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", string, string>>;
|
|
11
10
|
custom(args: AiRequestConfig): Promise<import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<string, string, string, string>>;
|
|
12
11
|
googleAi(args: AiRequestConfig): Promise<GoogleGenerativeAIProvider>;
|
|
13
12
|
}
|
|
@@ -1,108 +1,15 @@
|
|
|
1
1
|
import { Helpers } from '@chainfuse/helpers';
|
|
2
2
|
import { AiModels, enabledCloudflareLlmProviders } from '@chainfuse/types';
|
|
3
|
-
import {
|
|
3
|
+
import { customProvider, TypeValidationError, wrapLanguageModel } from 'ai';
|
|
4
4
|
import { ZodError } from 'zod';
|
|
5
5
|
import { AiBase } from '../base.mjs';
|
|
6
|
-
import { ServerSelector } from "../serverSelector.mjs";
|
|
7
6
|
import { AiRawProviders } from './rawProviders.mjs';
|
|
8
7
|
export class AiCustomProviders extends AiBase {
|
|
9
8
|
oaiOpenai(args) {
|
|
10
9
|
return new AiRawProviders(this.config).oaiOpenai(args);
|
|
11
10
|
}
|
|
12
|
-
async azOpenai(args
|
|
13
|
-
|
|
14
|
-
const [server, ...servers] = filteredServers;
|
|
15
|
-
const raw = new AiRawProviders(this.config);
|
|
16
|
-
return customProvider({
|
|
17
|
-
// @ts-expect-error override for types
|
|
18
|
-
languageModels: await server.languageModelAvailability.reduce(async (accPromise, model) => {
|
|
19
|
-
const acc = await accPromise;
|
|
20
|
-
// @ts-expect-error override for types
|
|
21
|
-
acc[model.name] = wrapLanguageModel({
|
|
22
|
-
model: (await raw.azOpenai(args, server, 'inputTokenCost' in model || 'outputTokenCost' in model
|
|
23
|
-
? {
|
|
24
|
-
inputTokenCost: 'inputTokenCost' in model && !isNaN(model.inputTokenCost) ? model.inputTokenCost : undefined,
|
|
25
|
-
outputTokenCost: 'outputTokenCost' in model && !isNaN(model.outputTokenCost) ? model.outputTokenCost : undefined,
|
|
26
|
-
}
|
|
27
|
-
: undefined))(model.name),
|
|
28
|
-
middleware: {
|
|
29
|
-
wrapGenerate: async ({ doGenerate, model, params }) => {
|
|
30
|
-
try {
|
|
31
|
-
// Must be double awaited to prevent a promise from being returned
|
|
32
|
-
return await doGenerate();
|
|
33
|
-
}
|
|
34
|
-
catch (error) {
|
|
35
|
-
if (APICallError.isInstance(error)) {
|
|
36
|
-
const idempotencyId = new Headers(error.responseHeaders).get('X-Idempotency-Id');
|
|
37
|
-
const lastServer = new URL(error.url).pathname.split('/')[5];
|
|
38
|
-
const compatibleServers = await new ServerSelector(this.config).closestServers(await import('@chainfuse/types/ai-tools/catalog/azure').then(({ azureCatalog }) => azureCatalog), model.modelId);
|
|
39
|
-
const lastServerIndex = compatibleServers.findIndex((s) => s.id.toLowerCase() === lastServer.toLowerCase());
|
|
40
|
-
if (args.logging ?? this.gatewayLog)
|
|
41
|
-
console.error('ai', 'custom provider', this.chalk.rgb(...Helpers.uniqueIdColor(idempotencyId))(`[${idempotencyId}]`), this.chalk.red('FAIL'), compatibleServers[lastServerIndex].id, 'REMAINING', JSON.stringify(compatibleServers.slice(lastServerIndex + 1).map((s) => s.id)));
|
|
42
|
-
// Should retry with the next server
|
|
43
|
-
const leftOverServers = compatibleServers.slice(lastServerIndex + 1);
|
|
44
|
-
const errors = [error];
|
|
45
|
-
for (const nextServer of leftOverServers) {
|
|
46
|
-
try {
|
|
47
|
-
if (args.logging ?? this.gatewayLog)
|
|
48
|
-
console.error('ai', 'custom provider', this.chalk.rgb(...Helpers.uniqueIdColor(idempotencyId))(`[${idempotencyId}]`), this.chalk.blue('FALLBACK'), nextServer.id, 'REMAINING', JSON.stringify(leftOverServers.slice(leftOverServers.indexOf(nextServer) + 1).map((s) => s.id)));
|
|
49
|
-
// Must be double awaited to prevent a promise from being returned
|
|
50
|
-
return await (await raw.azOpenai({ ...args, idempotencyId }, nextServer, (() => {
|
|
51
|
-
const foundModel = server.languageModelAvailability.find((languageModel) => languageModel.name === model.modelId);
|
|
52
|
-
if (foundModel && ('inputTokenCost' in foundModel || 'outputTokenCost' in foundModel)) {
|
|
53
|
-
return {
|
|
54
|
-
inputTokenCost: 'inputTokenCost' in model && !isNaN(model.inputTokenCost) ? model.inputTokenCost : undefined,
|
|
55
|
-
outputTokenCost: 'outputTokenCost' in model && !isNaN(model.outputTokenCost) ? model.outputTokenCost : undefined,
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
|
-
else {
|
|
59
|
-
return undefined;
|
|
60
|
-
}
|
|
61
|
-
})()))(model.modelId).doGenerate(params);
|
|
62
|
-
}
|
|
63
|
-
catch (nextServerError) {
|
|
64
|
-
if (APICallError.isInstance(nextServerError)) {
|
|
65
|
-
if (args.logging ?? this.gatewayLog)
|
|
66
|
-
console.error('ai', 'custom provider', this.chalk.rgb(...Helpers.uniqueIdColor(idempotencyId))(`[${idempotencyId}]`), this.chalk.red('FAIL'), nextServer.id, 'REMAINING', JSON.stringify(leftOverServers.slice(leftOverServers.indexOf(nextServer) + 1).map((s) => s.id)));
|
|
67
|
-
errors.push(nextServerError);
|
|
68
|
-
}
|
|
69
|
-
else {
|
|
70
|
-
errors.push(nextServerError);
|
|
71
|
-
throw nextServerError;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
// eslint-disable-next-line @typescript-eslint/only-throw-error
|
|
76
|
-
throw errors;
|
|
77
|
-
}
|
|
78
|
-
else {
|
|
79
|
-
throw error;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
},
|
|
83
|
-
},
|
|
84
|
-
});
|
|
85
|
-
return acc;
|
|
86
|
-
}, Promise.resolve({})),
|
|
87
|
-
// imageModels: await server!.imageModelAvailability.reduce(
|
|
88
|
-
// async (accPromise, model) => {
|
|
89
|
-
// const acc = await accPromise;
|
|
90
|
-
// // @ts-expect-error override for types
|
|
91
|
-
// acc[model as AzureImageModels] = (await raw.azOpenai(args, server!)).imageModel(model);
|
|
92
|
-
// return acc;
|
|
93
|
-
// },
|
|
94
|
-
// Promise.resolve({} as Record<AzureImageModels, Awaited<ReturnType<AiRawProviders['azOpenai']>>>),
|
|
95
|
-
// ),
|
|
96
|
-
// @ts-expect-error override for types
|
|
97
|
-
textEmbeddingModels: await server.textEmbeddingModelAvailability.reduce(async (accPromise, model) => {
|
|
98
|
-
const acc = await accPromise;
|
|
99
|
-
// @ts-expect-error override for types
|
|
100
|
-
acc[model.name] = (await raw.azOpenai(args, server)).textEmbeddingModel(model.name);
|
|
101
|
-
return acc;
|
|
102
|
-
}, Promise.resolve({})),
|
|
103
|
-
// An optional fallback provider to use when a requested model is not found in the custom provider.
|
|
104
|
-
...(servers.length > 0 && { fallbackProvider: await this.azOpenai(args, servers) }),
|
|
105
|
-
}); // Override type so autocomplete works
|
|
11
|
+
async azOpenai(args) {
|
|
12
|
+
return new AiRawProviders(this.config).azOpenai(args);
|
|
106
13
|
}
|
|
107
14
|
anthropic(args) {
|
|
108
15
|
return new AiRawProviders(this.config).anthropic(args);
|
|
@@ -1,16 +1,14 @@
|
|
|
1
1
|
import type { OpenAICompatibleProvider } from '@ai-sdk/openai-compatible';
|
|
2
2
|
import type { cloudflareModelPossibilities } from '@chainfuse/types';
|
|
3
3
|
import { AiBase } from '../base.mjs';
|
|
4
|
-
import type { AiRequestConfig
|
|
4
|
+
import type { AiRequestConfig } from '../types.mjs';
|
|
5
|
+
import type { AzureOpenAIProvider } from './types.mts';
|
|
5
6
|
export declare class AiRawProviders extends AiBase {
|
|
6
7
|
private readonly cacheTtl;
|
|
7
8
|
private static serverTimingHeader;
|
|
8
9
|
private updateGatewayLog;
|
|
9
10
|
oaiOpenai(args: AiRequestConfig): Promise<import("@ai-sdk/openai").OpenAIProvider>;
|
|
10
|
-
azOpenai(args: AiRequestConfig
|
|
11
|
-
inputTokenCost?: number;
|
|
12
|
-
outputTokenCost?: number;
|
|
13
|
-
}): Promise<import("@ai-sdk/azure").AzureOpenAIProvider>;
|
|
11
|
+
azOpenai(args: AiRequestConfig): Promise<AzureOpenAIProvider>;
|
|
14
12
|
anthropic(args: AiRequestConfig): Promise<import("@ai-sdk/anthropic").AnthropicProvider>;
|
|
15
13
|
custom(args: AiRequestConfig): Promise<OpenAICompatibleProvider<string, string, string, string>>;
|
|
16
14
|
googleAi(args: AiRequestConfig): Promise<import("@ai-sdk/google").GoogleGenerativeAIProvider>;
|
|
@@ -78,10 +78,10 @@ export class AiRawProviders extends AiBase {
|
|
|
78
78
|
headers.set('cf-aig-metadata', JSON.stringify(metadataHeader));
|
|
79
79
|
}
|
|
80
80
|
if (args.logging ?? this.gatewayLog)
|
|
81
|
-
console.info(
|
|
81
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.magenta(rawInit?.method), this.chalk.magenta(new URL(new Request(input).url).pathname));
|
|
82
82
|
return fetch(input, { ...rawInit, headers }).then(async (response) => {
|
|
83
83
|
if (args.logging ?? this.gatewayLog)
|
|
84
|
-
console.info(
|
|
84
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname));
|
|
85
85
|
// Inject it to have it available for retries
|
|
86
86
|
const mutableHeaders = new Headers(response.headers);
|
|
87
87
|
const serverTiming = await this.updateGatewayLog(response, metadataHeader, startRoundTrip, response.headers.has('openai-processing-ms') ? parseInt(response.headers.get('openai-processing-ms')) : undefined);
|
|
@@ -92,73 +92,146 @@ export class AiRawProviders extends AiBase {
|
|
|
92
92
|
}
|
|
93
93
|
else {
|
|
94
94
|
const [body1, body2] = response.body.tee();
|
|
95
|
-
console.error(
|
|
95
|
+
console.error(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.red(JSON.stringify(await new Response(body1, response).json())));
|
|
96
96
|
return new Response(body2, { ...response, headers: mutableHeaders });
|
|
97
97
|
}
|
|
98
98
|
});
|
|
99
99
|
},
|
|
100
100
|
}));
|
|
101
101
|
}
|
|
102
|
-
azOpenai(args
|
|
102
|
+
azOpenai(args) {
|
|
103
103
|
return import('@ai-sdk/azure').then(async ({ createAzure }) => createAzure({
|
|
104
|
-
apiKey:
|
|
104
|
+
apiKey: 'apikey-placeholder',
|
|
105
105
|
/**
|
|
106
106
|
* @link https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
|
|
107
107
|
* From the table, pick the `Latest GA release` for `Data plane - inference`
|
|
108
108
|
*/
|
|
109
109
|
apiVersion: '2024-10-21',
|
|
110
|
-
baseURL: new URL(['v1', this.config.gateway.accountId, this.gatewayName, 'azure-openai', server
|
|
110
|
+
baseURL: new URL(['v1', this.config.gateway.accountId, this.gatewayName, 'azure-openai', 'server-placeholder'].join('/'), 'https://gateway.ai.cloudflare.com').toString(),
|
|
111
111
|
headers: {
|
|
112
112
|
'cf-aig-authorization': `Bearer ${this.config.gateway.apiToken}`,
|
|
113
|
-
...(cost && { 'cf-aig-custom-cost': JSON.stringify({ per_token_in: cost.inputTokenCost ?? undefined, per_token_out: cost.outputTokenCost ?? undefined }) }),
|
|
113
|
+
// ...(cost && { 'cf-aig-custom-cost': JSON.stringify({ per_token_in: cost.inputTokenCost ?? undefined, per_token_out: cost.outputTokenCost ?? undefined }) }),
|
|
114
114
|
'cf-aig-metadata': JSON.stringify({
|
|
115
115
|
dataspaceId: (await BufferHelpers.uuidConvert(args.dataspaceId)).utf8,
|
|
116
116
|
...(args.groupBillingId && { groupBillingId: (await BufferHelpers.uuidConvert(args.groupBillingId)).utf8 }),
|
|
117
|
-
serverInfo: JSON.stringify({
|
|
118
|
-
name: `azure-${server.id}`,
|
|
119
|
-
distance: await import('haversine-distance').then(async ({ default: haversine }) => haversine(await import("../serverSelector.mjs").then(({ ServerSelector }) => new ServerSelector(this.config).determineLocation().then(({ coordinate }) => ({
|
|
120
|
-
lat: Helpers.precisionFloat(coordinate.lat),
|
|
121
|
-
lon: Helpers.precisionFloat(coordinate.lon),
|
|
122
|
-
}))), {
|
|
123
|
-
lat: Helpers.precisionFloat(server.coordinate.lat),
|
|
124
|
-
lon: Helpers.precisionFloat(server.coordinate.lon),
|
|
125
|
-
})),
|
|
126
|
-
}),
|
|
127
117
|
// Generate incomplete id because we don't have the body to hash yet. Fill it in in the `fetch()`
|
|
128
118
|
idempotencyId: args.idempotencyId ?? (await BufferHelpers.generateUuid).utf8.slice(0, 23),
|
|
129
119
|
executor: JSON.stringify(args.executor),
|
|
120
|
+
// @ts-expect-error server info gets added in afterwards
|
|
130
121
|
}),
|
|
131
122
|
...(args.cache && { 'cf-aig-cache-ttl': (typeof args.cache === 'boolean' ? (args.cache ? this.cacheTtl : 0) : args.cache).toString() }),
|
|
132
123
|
...(args.skipCache && { 'cf-aig-skip-cache': 'true' }),
|
|
133
124
|
},
|
|
134
|
-
fetch:
|
|
125
|
+
fetch: (input, rawInit) => Promise.all([import('../serverSelector.mjs'), import('@chainfuse/types/ai-tools/catalog/azure')])
|
|
126
|
+
.then(([{ ServerSelector }, { azureCatalog }]) => new ServerSelector(this.config).closestServers(azureCatalog))
|
|
127
|
+
.then(async (filteredServers) => {
|
|
135
128
|
const startRoundTrip = performance.now();
|
|
136
129
|
const headers = new Headers(rawInit?.headers);
|
|
137
130
|
const metadataHeader = JSON.parse(headers.get('cf-aig-metadata'));
|
|
131
|
+
// Calculate the idempotencyId if it doesn't exist yet
|
|
138
132
|
if (metadataHeader.idempotencyId.split('-').length === 4) {
|
|
139
133
|
metadataHeader.idempotencyId = `${metadataHeader.idempotencyId}-${(await CryptoHelpers.getHash('SHA-256', await new Request(input, rawInit).arrayBuffer())).slice(0, 12)}`;
|
|
140
134
|
headers.set('cf-aig-metadata', JSON.stringify(metadataHeader));
|
|
141
135
|
}
|
|
136
|
+
// Get endpoint dynamically from library
|
|
137
|
+
const fallbackedUrl = await import('@chainfuse/helpers').then(({ NetHelpers }) => (NetHelpers.isRequestLike(input) ? new URL(input.url) : new URL(input)));
|
|
138
|
+
const fallbackedEndpointParts = (() => {
|
|
139
|
+
const pathParts = fallbackedUrl.pathname
|
|
140
|
+
.split('/')
|
|
141
|
+
// removes empty from leading/trailing slashes
|
|
142
|
+
.filter(Boolean);
|
|
143
|
+
const index = pathParts.indexOf('azure-openai');
|
|
144
|
+
return index === -1 ? [] : pathParts.slice(index + 2);
|
|
145
|
+
})();
|
|
146
|
+
// Prevent double stringification
|
|
147
|
+
let fallbackedQuery;
|
|
148
|
+
try {
|
|
149
|
+
fallbackedQuery = JSON.parse(rawInit?.body);
|
|
150
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
fallbackedQuery = rawInit?.body;
|
|
154
|
+
}
|
|
155
|
+
// Build universal gateway request
|
|
156
|
+
const fallbackedBody = await Promise.all([import('haversine-distance'), import("../serverSelector.mjs")]).then(([{ default: haversine }, { ServerSelector }]) => Promise.all(filteredServers.map(async (server) => {
|
|
157
|
+
const fallbackedHeaders = {
|
|
158
|
+
...Object.fromEntries(Array.from(headers.entries()).filter(([key]) => !key.toLowerCase().startsWith('cf-aig-'))),
|
|
159
|
+
'api-key': this.config.providers.azureOpenAi.apiTokens[`AZURE_API_KEY_${server.id.toUpperCase().replaceAll('-', '_')}`],
|
|
160
|
+
'cf-aig-metadata': JSON.stringify({
|
|
161
|
+
...metadataHeader,
|
|
162
|
+
serverInfo: JSON.stringify({
|
|
163
|
+
name: `azure-${server.id}`,
|
|
164
|
+
distance: haversine(await new ServerSelector(this.config).determineLocation().then(({ coordinate }) => ({
|
|
165
|
+
lat: Helpers.precisionFloat(coordinate.lat),
|
|
166
|
+
lon: Helpers.precisionFloat(coordinate.lon),
|
|
167
|
+
})), {
|
|
168
|
+
lat: Helpers.precisionFloat(server.coordinate.lat),
|
|
169
|
+
lon: Helpers.precisionFloat(server.coordinate.lon),
|
|
170
|
+
}),
|
|
171
|
+
}),
|
|
172
|
+
}),
|
|
173
|
+
};
|
|
174
|
+
const modelName = fallbackedEndpointParts[0];
|
|
175
|
+
const languageModel = server.languageModelAvailability.find((model) => model.name === modelName);
|
|
176
|
+
if (languageModel && ('inputTokenCost' in languageModel || 'outputTokenCost' in languageModel)) {
|
|
177
|
+
fallbackedHeaders['cf-aig-custom-cost'] = {
|
|
178
|
+
per_token_in: 'inputTokenCost' in languageModel && !isNaN(languageModel.inputTokenCost) ? languageModel.inputTokenCost : undefined,
|
|
179
|
+
per_token_out: 'outputTokenCost' in languageModel && !isNaN(languageModel.outputTokenCost) ? languageModel.outputTokenCost : undefined,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
const embeddingModel = server.textEmbeddingModelAvailability.find((model) => model.name === modelName);
|
|
183
|
+
if (embeddingModel && 'tokenCost' in embeddingModel) {
|
|
184
|
+
fallbackedHeaders['cf-aig-custom-cost'] = {
|
|
185
|
+
per_token_in: 'tokenCost' in embeddingModel && !isNaN(embeddingModel.tokenCost) ? embeddingModel.tokenCost : undefined,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
return {
|
|
189
|
+
provider: 'azure-openai',
|
|
190
|
+
endpoint: `${[server.id.toLowerCase(), ...fallbackedEndpointParts].join('/')}${fallbackedUrl.search}${fallbackedUrl.hash}`,
|
|
191
|
+
headers: fallbackedHeaders,
|
|
192
|
+
query: fallbackedQuery,
|
|
193
|
+
};
|
|
194
|
+
})));
|
|
142
195
|
if (args.logging ?? this.gatewayLog)
|
|
143
|
-
console.info(
|
|
144
|
-
return fetch(
|
|
145
|
-
if (args.logging ?? this.gatewayLog)
|
|
146
|
-
console.info('ai', 'raw provider', this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname));
|
|
196
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.magenta(rawInit?.method), this.chalk.magenta(new URL(new Request(input).url).pathname));
|
|
197
|
+
return fetch(new URL(['v1', this.config.gateway.accountId, this.gatewayName].join('/'), 'https://gateway.ai.cloudflare.com'), { ...rawInit, headers, body: JSON.stringify(fallbackedBody) }).then(async (response) => {
|
|
147
198
|
// Inject it to have it available for retries
|
|
148
199
|
const mutableHeaders = new Headers(response.headers);
|
|
149
|
-
|
|
200
|
+
// Carry down
|
|
150
201
|
mutableHeaders.set('X-Idempotency-Id', metadataHeader.idempotencyId);
|
|
151
|
-
|
|
202
|
+
// Step references which server it hit
|
|
203
|
+
const fallbackedServerRaw = response.headers.get('cf-aig-step');
|
|
204
|
+
if (fallbackedServerRaw) {
|
|
205
|
+
const fallbackedRequest = fallbackedBody[parseInt(fallbackedServerRaw)];
|
|
206
|
+
if (fallbackedRequest) {
|
|
207
|
+
// Get the server's specific metadata
|
|
208
|
+
const fallbackedMetadataHeader = JSON.parse(new Headers(fallbackedRequest.headers).get('cf-aig-metadata'));
|
|
209
|
+
if (args.logging ?? this.gatewayLog)
|
|
210
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname), response.ok ? this.chalk.green(`[${fallbackedRequest.endpoint.split('/')[0]}]`) : this.chalk.red(`[${fallbackedRequest.endpoint.split('/')[0]}]`));
|
|
211
|
+
const serverTiming = await this.updateGatewayLog(response, fallbackedMetadataHeader, startRoundTrip, response.headers.has('x-envoy-upstream-service-time') ? parseInt(response.headers.get('x-envoy-upstream-service-time')) : undefined);
|
|
212
|
+
mutableHeaders.set('Server-Timing', serverTiming);
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
// Log without picked server
|
|
216
|
+
if (args.logging ?? this.gatewayLog)
|
|
217
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname));
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
else {
|
|
221
|
+
// Log without picked server
|
|
222
|
+
if (args.logging ?? this.gatewayLog)
|
|
223
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname));
|
|
224
|
+
}
|
|
152
225
|
if (response.ok) {
|
|
153
226
|
return new Response(response.body, { ...response, headers: mutableHeaders });
|
|
154
227
|
}
|
|
155
228
|
else {
|
|
156
229
|
const [body1, body2] = response.body.tee();
|
|
157
|
-
console.error(
|
|
230
|
+
console.error(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.red(JSON.stringify(await new Response(body1, response).json())));
|
|
158
231
|
return new Response(body2, { ...response, headers: mutableHeaders });
|
|
159
232
|
}
|
|
160
233
|
});
|
|
161
|
-
},
|
|
234
|
+
}),
|
|
162
235
|
}));
|
|
163
236
|
}
|
|
164
237
|
anthropic(args) {
|
|
@@ -189,10 +262,10 @@ export class AiRawProviders extends AiBase {
|
|
|
189
262
|
headers.set('cf-aig-metadata', JSON.stringify(metadataHeader));
|
|
190
263
|
}
|
|
191
264
|
if (args.logging ?? this.gatewayLog)
|
|
192
|
-
console.info(
|
|
265
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.magenta(rawInit?.method), this.chalk.magenta(new URL(new Request(input).url).pathname));
|
|
193
266
|
return fetch(input, { ...rawInit, headers }).then(async (response) => {
|
|
194
267
|
if (args.logging ?? this.gatewayLog)
|
|
195
|
-
console.info(
|
|
268
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname));
|
|
196
269
|
// Inject it to have it available for retries
|
|
197
270
|
const mutableHeaders = new Headers(response.headers);
|
|
198
271
|
const serverTiming = await this.updateGatewayLog(response, metadataHeader, startRoundTrip);
|
|
@@ -203,7 +276,7 @@ export class AiRawProviders extends AiBase {
|
|
|
203
276
|
}
|
|
204
277
|
else {
|
|
205
278
|
const [body1, body2] = response.body.tee();
|
|
206
|
-
console.error(
|
|
279
|
+
console.error(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.red(JSON.stringify(await new Response(body1, response).json())));
|
|
207
280
|
return new Response(body2, { ...response, headers: mutableHeaders });
|
|
208
281
|
}
|
|
209
282
|
});
|
|
@@ -276,10 +349,10 @@ export class AiRawProviders extends AiBase {
|
|
|
276
349
|
headers.set('X-Idempotency-Id', idempotencyId);
|
|
277
350
|
}
|
|
278
351
|
if (args.logging ?? this.gatewayLog)
|
|
279
|
-
console.info(
|
|
352
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(idempotencyId))(`[${idempotencyId}]`), this.chalk.magenta(rawInit?.method), this.chalk.magenta(new URL(new Request(input).url).pathname));
|
|
280
353
|
return fetch(input, { ...rawInit, headers }).then(async (response) => {
|
|
281
354
|
if (args.logging ?? this.gatewayLog)
|
|
282
|
-
console.info(
|
|
355
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(idempotencyId))(`[${idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname));
|
|
283
356
|
// Inject it to have it available for retries
|
|
284
357
|
const mutableHeaders = new Headers(response.headers);
|
|
285
358
|
mutableHeaders.set('X-Idempotency-Id', idempotencyId);
|
|
@@ -288,7 +361,7 @@ export class AiRawProviders extends AiBase {
|
|
|
288
361
|
}
|
|
289
362
|
else {
|
|
290
363
|
const [body1, body2] = response.body.tee();
|
|
291
|
-
console.error(
|
|
364
|
+
console.error(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(idempotencyId))(`[${idempotencyId}]`), this.chalk.red(JSON.stringify(await new Response(body1, response).json())));
|
|
292
365
|
return new Response(body2, { ...response, headers: mutableHeaders });
|
|
293
366
|
}
|
|
294
367
|
});
|
|
@@ -350,10 +423,10 @@ export class AiRawProviders extends AiBase {
|
|
|
350
423
|
headers.set('cf-aig-metadata', JSON.stringify(metadataHeader));
|
|
351
424
|
}
|
|
352
425
|
if (args.logging ?? this.gatewayLog)
|
|
353
|
-
console.info(
|
|
426
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.magenta(rawInit?.method), this.chalk.magenta(new URL(new Request(input).url).pathname));
|
|
354
427
|
return fetch(input, { ...rawInit, headers }).then(async (response) => {
|
|
355
428
|
if (args.logging ?? this.gatewayLog)
|
|
356
|
-
console.info(
|
|
429
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname));
|
|
357
430
|
// Inject it to have it available for retries
|
|
358
431
|
const mutableHeaders = new Headers(response.headers);
|
|
359
432
|
const serverTiming = await this.updateGatewayLog(response, metadataHeader, startRoundTrip);
|
|
@@ -364,7 +437,7 @@ export class AiRawProviders extends AiBase {
|
|
|
364
437
|
}
|
|
365
438
|
else {
|
|
366
439
|
const [body1, body2] = response.body.tee();
|
|
367
|
-
console.error(
|
|
440
|
+
console.error(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.red(JSON.stringify(await new Response(body1, response).json())));
|
|
368
441
|
return new Response(body2, { ...response, headers: mutableHeaders });
|
|
369
442
|
}
|
|
370
443
|
});
|
|
@@ -400,10 +473,10 @@ export class AiRawProviders extends AiBase {
|
|
|
400
473
|
headers.set('cf-aig-metadata', JSON.stringify(metadataHeader));
|
|
401
474
|
}
|
|
402
475
|
if (args.logging ?? this.gatewayLog)
|
|
403
|
-
console.info(
|
|
476
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.magenta(rawInit?.method), this.chalk.magenta(new URL(new Request(input).url).pathname));
|
|
404
477
|
return fetch(input, { ...rawInit, headers }).then(async (response) => {
|
|
405
478
|
if (args.logging ?? this.gatewayLog)
|
|
406
|
-
console.info(
|
|
479
|
+
console.info(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), response.ok ? this.chalk.green(response.status) : this.chalk.red(response.status), response.ok ? this.chalk.green(new URL(response.url).pathname) : this.chalk.red(new URL(response.url).pathname));
|
|
407
480
|
// Inject it to have it available for retries
|
|
408
481
|
const mutableHeaders = new Headers(response.headers);
|
|
409
482
|
if (headers.get('cf-ray')) {
|
|
@@ -419,7 +492,7 @@ export class AiRawProviders extends AiBase {
|
|
|
419
492
|
}
|
|
420
493
|
else {
|
|
421
494
|
const [body1, body2] = response.body.tee();
|
|
422
|
-
console.error(
|
|
495
|
+
console.error(new Date().toISOString(), this.chalk.rgb(...Helpers.uniqueIdColor(metadataHeader.idempotencyId))(`[${metadataHeader.idempotencyId}]`), this.chalk.red(JSON.stringify(await new Response(body1, response).json())));
|
|
423
496
|
return new Response(body2, { ...response, headers: mutableHeaders });
|
|
424
497
|
}
|
|
425
498
|
});
|
package/dist/registry.d.mts
CHANGED
|
@@ -7,7 +7,7 @@ export declare class AiRegistry extends AiBase {
|
|
|
7
7
|
anthropic: import("@ai-sdk/anthropic").AnthropicProvider;
|
|
8
8
|
custom: import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<string, string, string, string>;
|
|
9
9
|
'google.generative-ai': import("@ai-sdk/google").GoogleGenerativeAIProvider;
|
|
10
|
-
workersai: import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<"@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", "@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct",
|
|
10
|
+
workersai: import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<"@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", "@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", string, string>;
|
|
11
11
|
}>>;
|
|
12
12
|
registry(args: AiRequestConfig): Promise<import("ai").ProviderRegistryProvider<Readonly<{
|
|
13
13
|
openai: import("@ai-sdk/openai").OpenAIProvider;
|
|
@@ -15,6 +15,6 @@ export declare class AiRegistry extends AiBase {
|
|
|
15
15
|
anthropic: import("@ai-sdk/anthropic").AnthropicProvider;
|
|
16
16
|
custom: import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<string, string, string, string>;
|
|
17
17
|
'google.generative-ai': import("@ai-sdk/google").GoogleGenerativeAIProvider;
|
|
18
|
-
workersai: import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<"@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", "@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct",
|
|
18
|
+
workersai: import("@ai-sdk/openai-compatible").OpenAICompatibleProvider<"@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", "@cf/qwen/qwen1.5-0.5b-chat" | "@cf/google/gemma-2b-it-lora" | "@hf/nexusflow/starling-lm-7b-beta" | "@cf/meta/llama-3-8b-instruct" | "@cf/meta/llama-3.2-3b-instruct" | "@hf/thebloke/llamaguard-7b-awq" | "@hf/thebloke/neural-chat-7b-v3-1-awq" | "@cf/meta/llama-guard-3-8b" | "@cf/meta/llama-2-7b-chat-fp16" | "@cf/mistral/mistral-7b-instruct-v0.1" | "@cf/mistral/mistral-7b-instruct-v0.2-lora" | "@cf/tinyllama/tinyllama-1.1b-chat-v1.0" | "@hf/mistral/mistral-7b-instruct-v0.2" | "@cf/fblgit/una-cybertron-7b-v2-bf16" | "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" | "@cf/thebloke/discolm-german-7b-v1-awq" | "@cf/meta/llama-2-7b-chat-int8" | "@cf/meta/llama-3.1-8b-instruct-fp8" | "@hf/thebloke/mistral-7b-instruct-v0.1-awq" | "@cf/qwen/qwen1.5-7b-chat-awq" | "@cf/meta/llama-3.2-1b-instruct" | "@hf/thebloke/llama-2-13b-chat-awq" | "@hf/thebloke/deepseek-coder-6.7b-base-awq" | "@cf/meta-llama/llama-2-7b-chat-hf-lora" | "@cf/meta/llama-3.3-70b-instruct-fp8-fast" | "@hf/thebloke/openhermes-2.5-mistral-7b-awq" | "@hf/thebloke/deepseek-coder-6.7b-instruct-awq" | "@cf/deepseek-ai/deepseek-math-7b-instruct" | "@cf/tiiuae/falcon-7b-instruct" | "@hf/nousresearch/hermes-2-pro-mistral-7b" | "@cf/meta/llama-3.1-8b-instruct" | "@cf/meta/llama-3.1-8b-instruct-awq" | "@hf/thebloke/zephyr-7b-beta-awq" | "@cf/google/gemma-7b-it-lora" | "@cf/qwen/qwen1.5-1.8b-chat" | "@cf/meta/llama-3-8b-instruct-awq" | "@cf/meta/llama-3.2-11b-vision-instruct" | "@cf/defog/sqlcoder-7b-2" | "@cf/microsoft/phi-2" | "@hf/meta-llama/meta-llama-3-8b-instruct" | "@hf/google/gemma-7b-it" | "@cf/qwen/qwen1.5-14b-chat-awq" | "@cf/openchat/openchat-3.5-0106" | "@cf/meta/llama-4-scout-17b-16e-instruct", string, string>;
|
|
19
19
|
}>, ":">>;
|
|
20
20
|
}
|
package/dist/serverSelector.mjs
CHANGED
|
@@ -108,10 +108,8 @@ export class ServerSelector extends AiBase {
|
|
|
108
108
|
async closestServers(servers, requiredCapability, userCoordinate, privacyRegion) {
|
|
109
109
|
if (!userCoordinate || !privacyRegion) {
|
|
110
110
|
const { coordinate, country, continent } = await this.determineLocation();
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
if (!privacyRegion)
|
|
114
|
-
privacyRegion = ServerSelector.determinePrivacyRegion(country, continent);
|
|
111
|
+
userCoordinate ??= coordinate;
|
|
112
|
+
privacyRegion ??= ServerSelector.determinePrivacyRegion(country, continent);
|
|
115
113
|
}
|
|
116
114
|
// Skip over the rest of logic if the server can't handle the incoming request
|
|
117
115
|
// @ts-expect-error it's always strings, just sometimes string literals
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chainfuse/ai-tools",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.2",
|
|
4
4
|
"description": "",
|
|
5
5
|
"author": "ChainFuse",
|
|
6
6
|
"homepage": "https://github.com/ChainFuse/packages/tree/main/packages/ai-tools#readme",
|
|
@@ -49,20 +49,20 @@
|
|
|
49
49
|
"prettier": "@demosjarco/prettier-config",
|
|
50
50
|
"dependencies": {
|
|
51
51
|
"@ai-sdk/anthropic": "^1.2.9",
|
|
52
|
-
"@ai-sdk/azure": "^1.3.
|
|
52
|
+
"@ai-sdk/azure": "^1.3.11",
|
|
53
53
|
"@ai-sdk/google": "^1.2.10",
|
|
54
54
|
"@ai-sdk/openai": "^1.0.5",
|
|
55
|
-
"@ai-sdk/openai-compatible": "^0.2.
|
|
56
|
-
"@chainfuse/helpers": "^2.3.
|
|
57
|
-
"@chainfuse/types": "^2.
|
|
58
|
-
"ai": "^4.3.
|
|
55
|
+
"@ai-sdk/openai-compatible": "^0.2.9",
|
|
56
|
+
"@chainfuse/helpers": "^2.3.3",
|
|
57
|
+
"@chainfuse/types": "^2.9.1",
|
|
58
|
+
"ai": "^4.3.5",
|
|
59
59
|
"chalk": "^5.4.1",
|
|
60
60
|
"haversine-distance": "^1.2.3",
|
|
61
61
|
"workers-ai-provider": "^0.3.0"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
64
|
-
"@cloudflare/workers-types": "^4.
|
|
64
|
+
"@cloudflare/workers-types": "^4.20250410.0",
|
|
65
65
|
"openai": "^4.93.0"
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "e7ec0b382add48d991bacd096563691bc74f9a46"
|
|
68
68
|
}
|