@burtson-labs/bandit-engine 2.0.34 → 2.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/{aiProviderStore-3YS2BZU3.mjs → aiProviderStore-UJRDUYOF.mjs} +2 -2
- package/dist/{chat-2LYIZNWZ.mjs → chat-SZK3EBDO.mjs} +5 -5
- package/dist/chat-provider.js +227 -11
- package/dist/chat-provider.js.map +1 -1
- package/dist/chat-provider.mjs +4 -4
- package/dist/{chunk-6PQRG6W4.mjs → chunk-2ZZA2IFL.mjs} +3 -3
- package/dist/{chunk-GBANNFRD.mjs → chunk-ED5NNDKO.mjs} +3 -3
- package/dist/{chunk-XD5VJCFN.mjs → chunk-FJO5ZWYU.mjs} +3 -3
- package/dist/{chunk-XXMCI2WK.mjs → chunk-G4OXOTNJ.mjs} +41 -8
- package/dist/{chunk-XXMCI2WK.mjs.map → chunk-G4OXOTNJ.mjs.map} +1 -1
- package/dist/{chunk-LG2JCTOE.mjs → chunk-PLNFTIGX.mjs} +4 -4
- package/dist/{chunk-7RLN6ZGT.mjs → chunk-S635Q6OQ.mjs} +3 -3
- package/dist/{chunk-IGD4KGB5.mjs → chunk-ZAVV2AT5.mjs} +4 -4
- package/dist/{chunk-IHJPVIGB.mjs → chunk-ZNNOTDRD.mjs} +208 -1
- package/dist/chunk-ZNNOTDRD.mjs.map +1 -0
- package/dist/cli/cli.js +1964 -126
- package/dist/cli/cli.js.map +1 -1
- package/dist/{gateway-BiHRHJMM.d.ts → gateway-Ckf_KusF.d.mts} +4 -4
- package/dist/{gateway-BiHRHJMM.d.mts → gateway-Ckf_KusF.d.ts} +4 -4
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +318 -69
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +8 -8
- package/dist/management/management.js +316 -67
- package/dist/management/management.js.map +1 -1
- package/dist/management/management.mjs +6 -6
- package/dist/modals/chat-modal/chat-modal.js +236 -20
- package/dist/modals/chat-modal/chat-modal.js.map +1 -1
- package/dist/modals/chat-modal/chat-modal.mjs +4 -4
- package/dist/public-types.d.mts +1 -1
- package/dist/public-types.d.ts +1 -1
- package/docs/api_reference/classes/DebugLogger.html +11 -11
- package/docs/api_reference/classes/FeatureFlagService.html +13 -13
- package/docs/api_reference/classes/NotificationService.html +10 -10
- package/docs/api_reference/classes/StreamingTTSClient.html +9 -9
- package/docs/api_reference/classes/VectorDatabaseService.html +24 -24
- package/docs/api_reference/classes/VectorMigrationService.html +8 -8
- package/docs/api_reference/classes/VoiceService.html +2 -2
- package/docs/api_reference/enums/TTSState.html +2 -2
- package/docs/api_reference/functions/Chat.html +1 -1
- package/docs/api_reference/functions/ChatModal.html +1 -1
- package/docs/api_reference/functions/ChatProvider.html +1 -1
- package/docs/api_reference/functions/FeatureFlagProvider.html +1 -1
- package/docs/api_reference/functions/FeedbackButton.html +1 -1
- package/docs/api_reference/functions/FeedbackModal.html +1 -1
- package/docs/api_reference/functions/Management.html +1 -1
- package/docs/api_reference/functions/NotificationProvider.html +1 -1
- package/docs/api_reference/functions/SubscriptionExpiredGuard.html +1 -1
- package/docs/api_reference/functions/SubscriptionExpiredModal.html +1 -1
- package/docs/api_reference/functions/defineCustomElement.html +1 -1
- package/docs/api_reference/functions/getCriticalConfig.html +1 -1
- package/docs/api_reference/functions/getFeatureMatrix.html +1 -1
- package/docs/api_reference/functions/getStreamingTTSClient.html +1 -1
- package/docs/api_reference/functions/getSystemConstants.html +1 -1
- package/docs/api_reference/functions/getTTSState.html +1 -1
- package/docs/api_reference/functions/handleHttpError.html +1 -1
- package/docs/api_reference/functions/handleSubscriptionUpgrade.html +1 -1
- package/docs/api_reference/functions/handleValidationError.html +1 -1
- package/docs/api_reference/functions/initializeCoreSystem.html +1 -1
- package/docs/api_reference/functions/pauseTTS.html +1 -1
- package/docs/api_reference/functions/previewTierUpgrade.html +1 -1
- package/docs/api_reference/functions/resumeTTS.html +1 -1
- package/docs/api_reference/functions/showInfoNotification.html +1 -1
- package/docs/api_reference/functions/showSuccessNotification.html +1 -1
- package/docs/api_reference/functions/speakWithStreaming.html +1 -1
- package/docs/api_reference/functions/stopTTS.html +1 -1
- package/docs/api_reference/functions/syncSubscriptionWithAPI.html +1 -1
- package/docs/api_reference/functions/updateSubscriptionTier.html +1 -1
- package/docs/api_reference/functions/useFeatureFlag.html +1 -1
- package/docs/api_reference/functions/useFeatureVisibility.html +1 -1
- package/docs/api_reference/functions/useFeatures.html +1 -1
- package/docs/api_reference/functions/useGatewayHealth.html +1 -1
- package/docs/api_reference/functions/useGatewayMemory.html +1 -1
- package/docs/api_reference/functions/useGatewayModels.html +1 -1
- package/docs/api_reference/functions/useGlobalTTS.html +1 -1
- package/docs/api_reference/functions/useNotification.html +1 -1
- package/docs/api_reference/functions/useNotificationService.html +1 -1
- package/docs/api_reference/functions/useTTS.html +1 -1
- package/docs/api_reference/functions/useVectorStore.html +1 -1
- package/docs/api_reference/functions/useVoiceStore.html +2 -2
- package/docs/api_reference/functions/useVoices.html +1 -1
- package/docs/api_reference/functions/validateEnvironment.html +1 -1
- package/docs/api_reference/functions/validateSystemIntegrity.html +1 -1
- package/docs/api_reference/interfaces/AIChatRequest.html +2 -2
- package/docs/api_reference/interfaces/AIChatResponse.html +2 -2
- package/docs/api_reference/interfaces/AIGenerateRequest.html +2 -2
- package/docs/api_reference/interfaces/AIGenerateResponse.html +2 -2
- package/docs/api_reference/interfaces/AIMessage.html +2 -2
- package/docs/api_reference/interfaces/AIModel.html +2 -2
- package/docs/api_reference/interfaces/AIProviderConfig.html +2 -2
- package/docs/api_reference/interfaces/ChatConfig.html +3 -3
- package/docs/api_reference/interfaces/ChatModalProps.html +3 -3
- package/docs/api_reference/interfaces/CreateMemoryOptions.html +2 -2
- package/docs/api_reference/interfaces/FeatureEvaluation.html +7 -7
- package/docs/api_reference/interfaces/FeatureFlagConfig.html +9 -9
- package/docs/api_reference/interfaces/FeatureFlagContextValue.html +8 -8
- package/docs/api_reference/interfaces/FeatureFlagProviderProps.html +2 -2
- package/docs/api_reference/interfaces/FeedbackButtonProps.html +10 -10
- package/docs/api_reference/interfaces/FeedbackCategories.html +2 -2
- package/docs/api_reference/interfaces/FeedbackModalProps.html +2 -2
- package/docs/api_reference/interfaces/FeedbackPriorities.html +2 -2
- package/docs/api_reference/interfaces/FeedbackRequest.html +2 -2
- package/docs/api_reference/interfaces/FeedbackResponse.html +2 -2
- package/docs/api_reference/interfaces/FileUploadResult.html +2 -2
- package/docs/api_reference/interfaces/GatewayChatRequest.html +2 -2
- package/docs/api_reference/interfaces/GatewayChatResponse.html +2 -2
- package/docs/api_reference/interfaces/GatewayContract.html +2 -2
- package/docs/api_reference/interfaces/GatewayGenerateRequest.html +2 -2
- package/docs/api_reference/interfaces/GatewayGenerateResponse.html +2 -2
- package/docs/api_reference/interfaces/GatewayHealthResponse.html +2 -2
- package/docs/api_reference/interfaces/GatewayMemoryRecord.html +2 -2
- package/docs/api_reference/interfaces/GatewayMemoryResponse.html +2 -2
- package/docs/api_reference/interfaces/GatewayMessage.html +2 -2
- package/docs/api_reference/interfaces/GatewayMessageContent.html +2 -2
- package/docs/api_reference/interfaces/GatewayModel.html +2 -2
- package/docs/api_reference/interfaces/GatewayModelsResponse.html +2 -2
- package/docs/api_reference/interfaces/MemorySearchFilters.html +2 -2
- package/docs/api_reference/interfaces/MigrationProgress.html +2 -2
- package/docs/api_reference/interfaces/MigrationStatus.html +2 -2
- package/docs/api_reference/interfaces/NotificationConfig.html +2 -2
- package/docs/api_reference/interfaces/NotificationContextType.html +2 -2
- package/docs/api_reference/interfaces/NotificationProviderProps.html +2 -2
- package/docs/api_reference/interfaces/PackageSettings.html +2 -2
- package/docs/api_reference/interfaces/SearchOptions.html +2 -2
- package/docs/api_reference/interfaces/SearchResult.html +2 -2
- package/docs/api_reference/interfaces/SubscriptionExpiredGuardProps.html +2 -2
- package/docs/api_reference/interfaces/SubscriptionExpiredModalProps.html +2 -2
- package/docs/api_reference/interfaces/TTSOptions.html +2 -2
- package/docs/api_reference/interfaces/TTSProgress.html +2 -2
- package/docs/api_reference/interfaces/TrialUsage.html +2 -2
- package/docs/api_reference/interfaces/UploadRequest.html +3 -3
- package/docs/api_reference/interfaces/UseTTSReturn.html +2 -2
- package/docs/api_reference/interfaces/VectorDocument.html +2 -2
- package/docs/api_reference/interfaces/VectorMemory.html +2 -2
- package/docs/api_reference/interfaces/VectorMemoryMetadata.html +2 -2
- package/docs/api_reference/interfaces/VectorStoreStatus.html +2 -2
- package/docs/api_reference/interfaces/VoiceModelsResponse.html +2 -2
- package/docs/api_reference/interfaces/VoiceState.html +2 -2
- package/docs/api_reference/types/FeatureKey.html +1 -1
- package/docs/api_reference/types/FeatureMatrix.html +1 -1
- package/docs/api_reference/types/GatewayQueryOptions.html +1 -1
- package/docs/api_reference/types/LogContext.html +1 -1
- package/docs/api_reference/types/SubscriptionTier.html +1 -1
- package/docs/api_reference/variables/DEFAULT_TIER_FEATURES.html +1 -1
- package/docs/api_reference/variables/FeatureFlagContext.html +1 -1
- package/docs/api_reference/variables/OSS_DEFAULT_FEATURES.html +1 -1
- package/docs/api_reference/variables/SYSTEM_FLAGS.html +1 -1
- package/docs/api_reference/variables/authenticationService.html +1 -1
- package/docs/api_reference/variables/debugLogger-1.html +1 -1
- package/docs/api_reference/variables/featureFlagService-1.html +1 -1
- package/docs/api_reference/variables/notificationService-1.html +1 -1
- package/docs/api_reference/variables/vectorDatabaseService-1.html +1 -1
- package/docs/api_reference/variables/vectorMigrationService-1.html +1 -1
- package/docs/api_reference/variables/voiceService-1.html +1 -1
- package/package.json +1 -1
- package/dist/chunk-IHJPVIGB.mjs.map +0 -1
- /package/dist/{aiProviderStore-3YS2BZU3.mjs.map → aiProviderStore-UJRDUYOF.mjs.map} +0 -0
- /package/dist/{chat-2LYIZNWZ.mjs.map → chat-SZK3EBDO.mjs.map} +0 -0
- /package/dist/{chunk-6PQRG6W4.mjs.map → chunk-2ZZA2IFL.mjs.map} +0 -0
- /package/dist/{chunk-GBANNFRD.mjs.map → chunk-ED5NNDKO.mjs.map} +0 -0
- /package/dist/{chunk-XD5VJCFN.mjs.map → chunk-FJO5ZWYU.mjs.map} +0 -0
- /package/dist/{chunk-LG2JCTOE.mjs.map → chunk-PLNFTIGX.mjs.map} +0 -0
- /package/dist/{chunk-7RLN6ZGT.mjs.map → chunk-S635Q6OQ.mjs.map} +0 -0
- /package/dist/{chunk-IGD4KGB5.mjs.map → chunk-ZAVV2AT5.mjs.map} +0 -0
package/dist/cli/cli.js
CHANGED
|
@@ -30,7 +30,7 @@ var import_commander = require("commander");
|
|
|
30
30
|
// package.json
|
|
31
31
|
var package_default = {
|
|
32
32
|
name: "@burtson-labs/bandit-engine",
|
|
33
|
-
version: "2.0.
|
|
33
|
+
version: "2.0.36",
|
|
34
34
|
license: "BUSL-1.1",
|
|
35
35
|
main: "dist/index.js",
|
|
36
36
|
module: "dist/index.mjs",
|
|
@@ -214,23 +214,52 @@ var buildPackageJson = (ctx) => formatJson({
|
|
|
214
214
|
"vite": "^7.1.9"
|
|
215
215
|
}
|
|
216
216
|
});
|
|
217
|
-
var buildEnvExample = (ctx) =>
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
VITE_DEV_PORT=${ctx.frontendPort}
|
|
221
|
-
VITE_GATEWAY_URL=${ctx.defaultGatewayUrl}
|
|
222
|
-
VITE_DEFAULT_MODEL=${ctx.defaultModelId}
|
|
223
|
-
VITE_FALLBACK_MODEL=${ctx.fallbackModelId ?? ""}
|
|
224
|
-
VITE_GATEWAY_PROVIDER=${ctx.defaultProvider}
|
|
225
|
-
VITE_BRANDING_TEXT=${ctx.brandingText}
|
|
226
|
-
|
|
227
|
-
# Gateway configuration
|
|
228
|
-
#
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
217
|
+
var buildEnvExample = (ctx) => {
|
|
218
|
+
const lines = [
|
|
219
|
+
"# Frontend configuration",
|
|
220
|
+
`VITE_DEV_PORT=${ctx.frontendPort}`,
|
|
221
|
+
`VITE_GATEWAY_URL=${ctx.defaultGatewayUrl}`,
|
|
222
|
+
`VITE_DEFAULT_MODEL=${ctx.defaultModelId}`,
|
|
223
|
+
`VITE_FALLBACK_MODEL=${ctx.fallbackModelId ?? ""}`,
|
|
224
|
+
`VITE_GATEWAY_PROVIDER=${ctx.defaultProvider}`,
|
|
225
|
+
`VITE_BRANDING_TEXT=${ctx.brandingText}`,
|
|
226
|
+
"",
|
|
227
|
+
"# Gateway configuration",
|
|
228
|
+
"# These values power server/gateway.js \u2014 update them before running in production."
|
|
229
|
+
];
|
|
230
|
+
switch (ctx.defaultProvider) {
|
|
231
|
+
case "openai":
|
|
232
|
+
lines.push("OPENAI_API_KEY=");
|
|
233
|
+
break;
|
|
234
|
+
case "azure":
|
|
235
|
+
lines.push("AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com");
|
|
236
|
+
lines.push("AZURE_OPENAI_API_KEY=");
|
|
237
|
+
lines.push("AZURE_OPENAI_API_VERSION=2024-08-01-preview");
|
|
238
|
+
lines.push("AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-4o");
|
|
239
|
+
lines.push("AZURE_OPENAI_COMPLETIONS_DEPLOYMENT=gpt-35-turbo-instruct");
|
|
240
|
+
lines.push("AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT=text-embedding-3-large");
|
|
241
|
+
break;
|
|
242
|
+
case "anthropic":
|
|
243
|
+
lines.push("ANTHROPIC_API_KEY=");
|
|
244
|
+
lines.push("ANTHROPIC_BASE_URL=https://api.anthropic.com");
|
|
245
|
+
lines.push("ANTHROPIC_API_VERSION=2023-06-01");
|
|
246
|
+
lines.push("ANTHROPIC_MAX_TOKENS=1024");
|
|
247
|
+
break;
|
|
248
|
+
case "xai":
|
|
249
|
+
lines.push("XAI_API_KEY=");
|
|
250
|
+
lines.push("XAI_BASE_URL=https://api.x.ai/v1");
|
|
251
|
+
break;
|
|
252
|
+
case "ollama":
|
|
253
|
+
default:
|
|
254
|
+
lines.push("OLLAMA_URL=http://localhost:11434");
|
|
255
|
+
break;
|
|
256
|
+
}
|
|
257
|
+
lines.push(`PORT=${ctx.gatewayPort}`);
|
|
258
|
+
lines.push(
|
|
259
|
+
"# If you switch providers later, copy the relevant block above and update the credentials."
|
|
260
|
+
);
|
|
261
|
+
return ensureTrailingNewline(normalizeLineEndings(lines.join("\n")));
|
|
262
|
+
};
|
|
234
263
|
var buildTsConfig = () => formatJson({
|
|
235
264
|
compilerOptions: {
|
|
236
265
|
target: "ESNext",
|
|
@@ -429,7 +458,7 @@ const gatewayBaseUrl = (import.meta.env.VITE_GATEWAY_URL ?? "${ctx.defaultGatewa
|
|
|
429
458
|
const defaultModelId = import.meta.env.VITE_DEFAULT_MODEL ?? "${ctx.defaultModelId}";
|
|
430
459
|
const fallbackModelId = import.meta.env.VITE_FALLBACK_MODEL ?? ${ctx.fallbackModelId ? `${QUOTE}${ctx.fallbackModelId}${QUOTE}` : "undefined"};
|
|
431
460
|
const brandingText = import.meta.env.VITE_BRANDING_TEXT ?? "${ctx.brandingText}";
|
|
432
|
-
const provider = (import.meta.env.VITE_GATEWAY_PROVIDER ?? "${ctx.defaultProvider}") as "openai" | "ollama";
|
|
461
|
+
const provider = (import.meta.env.VITE_GATEWAY_PROVIDER ?? "${ctx.defaultProvider}") as "openai" | "ollama" | "azure" | "anthropic" | "xai";
|
|
433
462
|
|
|
434
463
|
const gatewayApiUrl = gatewayBaseUrl.endsWith("/api") ? gatewayBaseUrl : gatewayBaseUrl + "/api";
|
|
435
464
|
const banditHeadLogoUrl = "https://cdn.burtson.ai/images/bandit-head.png";
|
|
@@ -611,7 +640,7 @@ function App() {
|
|
|
611
640
|
{brandingText}
|
|
612
641
|
</Typography>
|
|
613
642
|
<Typography variant="body1" color="text.secondary">
|
|
614
|
-
Build, brand, and launch your assistant with a drop-in chat surface plus a secure gateway for OpenAI or Ollama.
|
|
643
|
+
Build, brand, and launch your assistant with a drop-in chat surface plus a secure gateway for OpenAI, Azure OpenAI, Anthropic, XAI, or Ollama.
|
|
615
644
|
</Typography>
|
|
616
645
|
<Stack direction={{ xs: "column", sm: "row" }} spacing={2}>
|
|
617
646
|
<Button component={RouterLink} to="/chat" variant="contained" color="primary">
|
|
@@ -658,7 +687,7 @@ function App() {
|
|
|
658
687
|
Ship secure gateways
|
|
659
688
|
</Typography>
|
|
660
689
|
<Typography variant="body2" color="text.secondary">
|
|
661
|
-
Keep API keys server-side while proxying requests to OpenAI or Ollama through the included Express gateway.
|
|
690
|
+
Keep API keys server-side while proxying requests to OpenAI, Azure OpenAI, Anthropic, XAI, or Ollama through the included Express gateway.
|
|
662
691
|
</Typography>
|
|
663
692
|
</CardContent>
|
|
664
693
|
</Card>
|
|
@@ -820,6 +849,664 @@ var buildBrandingConfig = (ctx) => formatJson({
|
|
|
820
849
|
},
|
|
821
850
|
knowledgeDocs: []
|
|
822
851
|
});
|
|
852
|
+
var NEXT_CHAT_ROUTE_TEMPLATE = `import { NextRequest, NextResponse } from "next/server";
|
|
853
|
+
|
|
854
|
+
export const dynamic = "force-dynamic";
|
|
855
|
+
|
|
856
|
+
const DEFAULT_PROVIDER = "__DEFAULT_PROVIDER__";
|
|
857
|
+
const DEFAULT_MODEL = "__DEFAULT_MODEL__";
|
|
858
|
+
const FALLBACK_MODEL = __FALLBACK_MODEL__;
|
|
859
|
+
|
|
860
|
+
const OLLAMA_URL = (process.env.OLLAMA_URL ?? "http://localhost:11434").replace(/\\/$/, "");
|
|
861
|
+
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
|
|
862
|
+
const AZURE_OPENAI_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT ? process.env.AZURE_OPENAI_ENDPOINT.replace(/\\/$/, "") : undefined;
|
|
863
|
+
const AZURE_OPENAI_API_KEY = process.env.AZURE_OPENAI_API_KEY;
|
|
864
|
+
const AZURE_OPENAI_API_VERSION = process.env.AZURE_OPENAI_API_VERSION ?? "2024-08-01-preview";
|
|
865
|
+
const AZURE_OPENAI_CHAT_DEPLOYMENT = process.env.AZURE_OPENAI_CHAT_DEPLOYMENT;
|
|
866
|
+
const AZURE_OPENAI_COMPLETIONS_DEPLOYMENT = process.env.AZURE_OPENAI_COMPLETIONS_DEPLOYMENT ?? AZURE_OPENAI_CHAT_DEPLOYMENT;
|
|
867
|
+
const AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT = process.env.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT;
|
|
868
|
+
const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
|
|
869
|
+
const ANTHROPIC_BASE_URL = (process.env.ANTHROPIC_BASE_URL ?? "https://api.anthropic.com").replace(/\\/$/, "");
|
|
870
|
+
const ANTHROPIC_API_VERSION = process.env.ANTHROPIC_API_VERSION ?? "2023-06-01";
|
|
871
|
+
const ANTHROPIC_MAX_TOKENS = Number.isFinite(Number(process.env.ANTHROPIC_MAX_TOKENS))
|
|
872
|
+
? Number(process.env.ANTHROPIC_MAX_TOKENS)
|
|
873
|
+
: 1024;
|
|
874
|
+
const XAI_API_KEY = process.env.XAI_API_KEY;
|
|
875
|
+
const XAI_BASE_URL = (process.env.XAI_BASE_URL ?? "https://api.x.ai/v1").replace(/\\/$/, "");
|
|
876
|
+
|
|
877
|
+
interface GatewayChatBody {
|
|
878
|
+
provider?: string;
|
|
879
|
+
model?: string;
|
|
880
|
+
messages?: Array<{ role: string; content: unknown }>;
|
|
881
|
+
prompt?: string;
|
|
882
|
+
stream?: boolean;
|
|
883
|
+
temperature?: number;
|
|
884
|
+
max_tokens?: number;
|
|
885
|
+
top_p?: number;
|
|
886
|
+
stop?: string | string[];
|
|
887
|
+
stop_sequences?: string | string[];
|
|
888
|
+
tools?: unknown;
|
|
889
|
+
tool_choice?: unknown;
|
|
890
|
+
metadata?: unknown;
|
|
891
|
+
thinking?: unknown;
|
|
892
|
+
images?: string[];
|
|
893
|
+
[key: string]: unknown;
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
const normalizeProvider = (input: string): "openai" | "azure" | "anthropic" | "ollama" | "xai" => {
|
|
897
|
+
const value = input.toLowerCase();
|
|
898
|
+
if (value === "azure-openai" || value === "azureopenai" || value === "azure") return "azure";
|
|
899
|
+
if (value === "anthropic" || value === "claude") return "anthropic";
|
|
900
|
+
if (value === "ollama") return "ollama";
|
|
901
|
+
if (value === "xai" || value === "grok") return "xai";
|
|
902
|
+
return "openai";
|
|
903
|
+
};
|
|
904
|
+
|
|
905
|
+
const stripPrefix = (model: unknown, prefix: string, fallback: string): string => {
|
|
906
|
+
if (typeof model === "string") {
|
|
907
|
+
return model.replace(new RegExp(\`^\${prefix}:\`), "");
|
|
908
|
+
}
|
|
909
|
+
return fallback;
|
|
910
|
+
};
|
|
911
|
+
|
|
912
|
+
const requireOpenAIKey = () => {
|
|
913
|
+
if (!OPENAI_API_KEY) {
|
|
914
|
+
throw new Error("Missing OPENAI_API_KEY. Add it to your .env file to route requests to OpenAI.");
|
|
915
|
+
}
|
|
916
|
+
return OPENAI_API_KEY;
|
|
917
|
+
};
|
|
918
|
+
|
|
919
|
+
const requireXAIKey = () => {
|
|
920
|
+
if (!XAI_API_KEY) {
|
|
921
|
+
throw new Error("Missing XAI_API_KEY. Add it to your .env file to route requests to xAI.");
|
|
922
|
+
}
|
|
923
|
+
return XAI_API_KEY;
|
|
924
|
+
};
|
|
925
|
+
|
|
926
|
+
const requireAnthropicKey = () => {
|
|
927
|
+
if (!ANTHROPIC_API_KEY) {
|
|
928
|
+
throw new Error("Missing ANTHROPIC_API_KEY. Add it to your .env file to route requests to Anthropic.");
|
|
929
|
+
}
|
|
930
|
+
return ANTHROPIC_API_KEY;
|
|
931
|
+
};
|
|
932
|
+
|
|
933
|
+
const isAzureConfigured = () => Boolean(AZURE_OPENAI_ENDPOINT && AZURE_OPENAI_API_KEY);
|
|
934
|
+
|
|
935
|
+
const requireAzureBaseConfig = () => {
|
|
936
|
+
if (!AZURE_OPENAI_ENDPOINT) {
|
|
937
|
+
throw new Error("Missing AZURE_OPENAI_ENDPOINT. Add it to your .env file to route requests to Azure OpenAI.");
|
|
938
|
+
}
|
|
939
|
+
if (!AZURE_OPENAI_API_KEY) {
|
|
940
|
+
throw new Error("Missing AZURE_OPENAI_API_KEY. Add it to your .env file to route requests to Azure OpenAI.");
|
|
941
|
+
}
|
|
942
|
+
return {
|
|
943
|
+
endpoint: AZURE_OPENAI_ENDPOINT,
|
|
944
|
+
apiKey: AZURE_OPENAI_API_KEY,
|
|
945
|
+
};
|
|
946
|
+
};
|
|
947
|
+
|
|
948
|
+
const buildAzureDeploymentUrl = (deployment: string | undefined, suffix: string) => {
|
|
949
|
+
if (!deployment) {
|
|
950
|
+
throw new Error(\`Missing Azure OpenAI \${suffix.split("/")[0]} deployment name.\`);
|
|
951
|
+
}
|
|
952
|
+
const { endpoint } = requireAzureBaseConfig();
|
|
953
|
+
const normalized = suffix.replace(/^\\/+/, "");
|
|
954
|
+
return \`\${endpoint}/openai/deployments/\${deployment}/\${normalized}?api-version=\${AZURE_OPENAI_API_VERSION}\`;
|
|
955
|
+
};
|
|
956
|
+
|
|
957
|
+
const resolveAzureDeployment = (model: unknown, fallback: string | undefined, kind: "chat" | "completions" | "embeddings") => {
|
|
958
|
+
const explicit = typeof model === "string" ? model.replace(/^azure:/, "") : undefined;
|
|
959
|
+
if (explicit) return explicit;
|
|
960
|
+
if (kind === "embeddings") return AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT ?? fallback;
|
|
961
|
+
if (kind === "completions") return AZURE_OPENAI_COMPLETIONS_DEPLOYMENT ?? fallback;
|
|
962
|
+
return AZURE_OPENAI_CHAT_DEPLOYMENT ?? fallback;
|
|
963
|
+
};
|
|
964
|
+
|
|
965
|
+
const flattenGatewayContent = (content: unknown): string => {
|
|
966
|
+
if (typeof content === "string") return content;
|
|
967
|
+
if (Array.isArray(content)) {
|
|
968
|
+
return content
|
|
969
|
+
.map((part) => {
|
|
970
|
+
if (typeof part === "string") return part;
|
|
971
|
+
if (part && typeof part === "object" && "type" in part) {
|
|
972
|
+
const typed = part as { type?: string; text?: string; image_url?: { url?: string } };
|
|
973
|
+
if (typed.type === "text" && typeof typed.text === "string") return typed.text;
|
|
974
|
+
if (typed.type === "image_url" && typed.image_url?.url) return \`[Image: \${typed.image_url.url}]\`;
|
|
975
|
+
}
|
|
976
|
+
return JSON.stringify(part ?? {});
|
|
977
|
+
})
|
|
978
|
+
.join("\\n");
|
|
979
|
+
}
|
|
980
|
+
if (content && typeof content === "object") return JSON.stringify(content);
|
|
981
|
+
return "";
|
|
982
|
+
};
|
|
983
|
+
|
|
984
|
+
const toAnthropicMessages = (messages: Array<{ role: string; content: unknown }> = []) => {
|
|
985
|
+
const anthropicMessages: Array<{ role: "user" | "assistant"; content: Array<{ type: "text"; text: string }> }> = [];
|
|
986
|
+
let systemPrompt = "";
|
|
987
|
+
|
|
988
|
+
for (const message of messages) {
|
|
989
|
+
if (!message) continue;
|
|
990
|
+
const text = flattenGatewayContent(message.content);
|
|
991
|
+
if (message.role === "system") {
|
|
992
|
+
systemPrompt = systemPrompt ? \`\${systemPrompt}\\n\\n\${text}\` : text;
|
|
993
|
+
continue;
|
|
994
|
+
}
|
|
995
|
+
const role = message.role === "assistant" ? "assistant" : "user";
|
|
996
|
+
anthropicMessages.push({
|
|
997
|
+
role,
|
|
998
|
+
content: [{ type: "text", text }],
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
return { messages: anthropicMessages, system: systemPrompt || undefined };
|
|
1003
|
+
};
|
|
1004
|
+
|
|
1005
|
+
const convertAnthropicResponseToGateway = (responseBody: any, modelName: string) => {
|
|
1006
|
+
if (!responseBody) {
|
|
1007
|
+
return {
|
|
1008
|
+
id: \`anthropic-\${Date.now()}\`,
|
|
1009
|
+
object: "chat.completion",
|
|
1010
|
+
created: Math.floor(Date.now() / 1000),
|
|
1011
|
+
model: modelName.startsWith("anthropic:") ? modelName : \`anthropic:\${modelName}\`,
|
|
1012
|
+
choices: [],
|
|
1013
|
+
};
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
const textContent = Array.isArray(responseBody.content)
|
|
1017
|
+
? responseBody.content
|
|
1018
|
+
.filter((item: any) => item && item.type === "text" && typeof item.text === "string")
|
|
1019
|
+
.map((item: any) => item.text)
|
|
1020
|
+
.join("\\n")
|
|
1021
|
+
: typeof responseBody.content === "string"
|
|
1022
|
+
? responseBody.content
|
|
1023
|
+
: "";
|
|
1024
|
+
|
|
1025
|
+
const promptTokens = responseBody.usage?.input_tokens ?? 0;
|
|
1026
|
+
const completionTokens = responseBody.usage?.output_tokens ?? 0;
|
|
1027
|
+
|
|
1028
|
+
return {
|
|
1029
|
+
id: responseBody.id ?? \`anthropic-\${Date.now()}\`,
|
|
1030
|
+
object: "chat.completion",
|
|
1031
|
+
created: Math.floor(Date.now() / 1000),
|
|
1032
|
+
model: modelName.startsWith("anthropic:") ? modelName : \`anthropic:\${modelName}\`,
|
|
1033
|
+
choices: [
|
|
1034
|
+
{
|
|
1035
|
+
index: 0,
|
|
1036
|
+
message: {
|
|
1037
|
+
role: responseBody.role ?? "assistant",
|
|
1038
|
+
content: textContent,
|
|
1039
|
+
},
|
|
1040
|
+
finish_reason: responseBody.stop_reason ?? responseBody.stop_sequence ?? null,
|
|
1041
|
+
},
|
|
1042
|
+
],
|
|
1043
|
+
usage: responseBody.usage
|
|
1044
|
+
? {
|
|
1045
|
+
prompt_tokens: promptTokens,
|
|
1046
|
+
completion_tokens: completionTokens,
|
|
1047
|
+
total_tokens: promptTokens + completionTokens,
|
|
1048
|
+
}
|
|
1049
|
+
: undefined,
|
|
1050
|
+
};
|
|
1051
|
+
};
|
|
1052
|
+
|
|
1053
|
+
const passthroughResponse = (upstream: Response) => {
|
|
1054
|
+
const headers = new Headers(upstream.headers);
|
|
1055
|
+
return new Response(upstream.body, {
|
|
1056
|
+
status: upstream.status,
|
|
1057
|
+
statusText: upstream.statusText,
|
|
1058
|
+
headers,
|
|
1059
|
+
});
|
|
1060
|
+
};
|
|
1061
|
+
|
|
1062
|
+
const jsonResponse = async (upstream: Response) => {
|
|
1063
|
+
const data = await upstream.json().catch(async () => ({ raw: await upstream.text() }));
|
|
1064
|
+
return NextResponse.json(data, { status: upstream.status });
|
|
1065
|
+
};
|
|
1066
|
+
|
|
1067
|
+
const errorResponse = (status: number, error: unknown) =>
|
|
1068
|
+
NextResponse.json(
|
|
1069
|
+
{
|
|
1070
|
+
error: error instanceof Error ? error.message : String(error ?? "Unknown error"),
|
|
1071
|
+
},
|
|
1072
|
+
{ status }
|
|
1073
|
+
);
|
|
1074
|
+
|
|
1075
|
+
export async function POST(request: NextRequest) {
|
|
1076
|
+
const body = (await request.json()) as GatewayChatBody;
|
|
1077
|
+
const provider = normalizeProvider(body.provider ?? DEFAULT_PROVIDER);
|
|
1078
|
+
const stream = body.stream !== false;
|
|
1079
|
+
|
|
1080
|
+
try {
|
|
1081
|
+
switch (provider) {
|
|
1082
|
+
case "openai": {
|
|
1083
|
+
const openaiKey = requireOpenAIKey();
|
|
1084
|
+
const { provider: _provider, ...cleanBody } = body;
|
|
1085
|
+
const requestBody = {
|
|
1086
|
+
...cleanBody,
|
|
1087
|
+
stream,
|
|
1088
|
+
model: stripPrefix(body.model ?? DEFAULT_MODEL, "openai", "gpt-4o"),
|
|
1089
|
+
};
|
|
1090
|
+
const response = await fetch("https://api.openai.com/v1/chat/completions", {
|
|
1091
|
+
method: "POST",
|
|
1092
|
+
headers: {
|
|
1093
|
+
"Content-Type": "application/json",
|
|
1094
|
+
Authorization: \`Bearer \${openaiKey}\`,
|
|
1095
|
+
},
|
|
1096
|
+
body: JSON.stringify(requestBody),
|
|
1097
|
+
});
|
|
1098
|
+
if (!response.ok) {
|
|
1099
|
+
const details = await response.text();
|
|
1100
|
+
return NextResponse.json({ error: \`OpenAI chat failed: \${response.status}\`, details }, { status: response.status });
|
|
1101
|
+
}
|
|
1102
|
+
return stream ? passthroughResponse(response) : jsonResponse(response);
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
case "xai": {
|
|
1106
|
+
const xaiKey = requireXAIKey();
|
|
1107
|
+
const { provider: _provider, ...cleanBody } = body;
|
|
1108
|
+
const requestBody = {
|
|
1109
|
+
...cleanBody,
|
|
1110
|
+
stream,
|
|
1111
|
+
model: stripPrefix(body.model ?? DEFAULT_MODEL, "xai", "grok-2-latest"),
|
|
1112
|
+
};
|
|
1113
|
+
const response = await fetch(XAI_BASE_URL + "/chat/completions", {
|
|
1114
|
+
method: "POST",
|
|
1115
|
+
headers: {
|
|
1116
|
+
"Content-Type": "application/json",
|
|
1117
|
+
Authorization: "Bearer " + xaiKey,
|
|
1118
|
+
},
|
|
1119
|
+
body: JSON.stringify(requestBody),
|
|
1120
|
+
});
|
|
1121
|
+
if (!response.ok) {
|
|
1122
|
+
const details = await response.text();
|
|
1123
|
+
return NextResponse.json({ error: "xAI chat failed: " + response.status, details }, { status: response.status });
|
|
1124
|
+
}
|
|
1125
|
+
return stream ? passthroughResponse(response) : jsonResponse(response);
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
case "anthropic": {
|
|
1129
|
+
const anthropicKey = requireAnthropicKey();
|
|
1130
|
+
const requestedModel = stripPrefix(body.model ?? DEFAULT_MODEL, "anthropic", "claude-3-5-haiku-latest");
|
|
1131
|
+
const stopSequences = Array.isArray(body.stop)
|
|
1132
|
+
? body.stop
|
|
1133
|
+
: Array.isArray(body.stop_sequences)
|
|
1134
|
+
? body.stop_sequences
|
|
1135
|
+
: body.stop
|
|
1136
|
+
? [body.stop]
|
|
1137
|
+
: undefined;
|
|
1138
|
+
const { messages, system } = toAnthropicMessages(Array.isArray(body.messages) ? body.messages : []);
|
|
1139
|
+
const fallbackText = typeof body.prompt === "string" && body.prompt.trim().length > 0
|
|
1140
|
+
? body.prompt
|
|
1141
|
+
: "Hello from Bandit quickstart gateway";
|
|
1142
|
+
|
|
1143
|
+
const requestBody: Record<string, unknown> = {
|
|
1144
|
+
model: requestedModel,
|
|
1145
|
+
messages: messages.length > 0
|
|
1146
|
+
? messages
|
|
1147
|
+
: [
|
|
1148
|
+
{
|
|
1149
|
+
role: "user",
|
|
1150
|
+
content: [{ type: "text", text: fallbackText }],
|
|
1151
|
+
},
|
|
1152
|
+
],
|
|
1153
|
+
stream,
|
|
1154
|
+
max_tokens: typeof body.max_tokens === "number" && body.max_tokens > 0 ? body.max_tokens : ANTHROPIC_MAX_TOKENS,
|
|
1155
|
+
};
|
|
1156
|
+
|
|
1157
|
+
if (system) requestBody.system = system;
|
|
1158
|
+
if (typeof body.temperature === "number") requestBody.temperature = body.temperature;
|
|
1159
|
+
if (typeof body.top_p === "number") requestBody.top_p = body.top_p;
|
|
1160
|
+
if (typeof body.top_k === "number") requestBody.top_k = body.top_k;
|
|
1161
|
+
if (stopSequences) requestBody.stop_sequences = stopSequences;
|
|
1162
|
+
if (body.metadata) requestBody.metadata = body.metadata;
|
|
1163
|
+
if (body.tools) requestBody.tools = body.tools;
|
|
1164
|
+
if (body.tool_choice) requestBody.tool_choice = body.tool_choice;
|
|
1165
|
+
if (body.thinking) requestBody.thinking = body.thinking;
|
|
1166
|
+
|
|
1167
|
+
const response = await fetch(\`\${ANTHROPIC_BASE_URL}/v1/messages\`, {
|
|
1168
|
+
method: "POST",
|
|
1169
|
+
headers: {
|
|
1170
|
+
"Content-Type": "application/json",
|
|
1171
|
+
"x-api-key": anthropicKey,
|
|
1172
|
+
"anthropic-version": ANTHROPIC_API_VERSION,
|
|
1173
|
+
},
|
|
1174
|
+
body: JSON.stringify(requestBody),
|
|
1175
|
+
});
|
|
1176
|
+
|
|
1177
|
+
if (!response.ok) {
|
|
1178
|
+
const details = await response.text();
|
|
1179
|
+
return NextResponse.json({ error: \`Anthropic chat failed: \${response.status}\`, details }, { status: response.status });
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
if (stream) {
|
|
1183
|
+
return passthroughResponse(response);
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
const data = await response.json();
|
|
1187
|
+
const normalized = convertAnthropicResponseToGateway(data, requestedModel);
|
|
1188
|
+
return NextResponse.json(normalized);
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
case "azure": {
|
|
1192
|
+
const { apiKey } = requireAzureBaseConfig();
|
|
1193
|
+
const deployment = resolveAzureDeployment(body.model, AZURE_OPENAI_CHAT_DEPLOYMENT, "chat");
|
|
1194
|
+
const { provider: _provider, model: _model, ...cleanBody } = body;
|
|
1195
|
+
const requestBody = {
|
|
1196
|
+
...cleanBody,
|
|
1197
|
+
stream,
|
|
1198
|
+
};
|
|
1199
|
+
|
|
1200
|
+
const response = await fetch(buildAzureDeploymentUrl(deployment, "chat/completions"), {
|
|
1201
|
+
method: "POST",
|
|
1202
|
+
headers: {
|
|
1203
|
+
"Content-Type": "application/json",
|
|
1204
|
+
"api-key": apiKey,
|
|
1205
|
+
},
|
|
1206
|
+
body: JSON.stringify(requestBody),
|
|
1207
|
+
});
|
|
1208
|
+
|
|
1209
|
+
if (!response.ok) {
|
|
1210
|
+
const details = await response.text();
|
|
1211
|
+
return NextResponse.json({ error: \`Azure OpenAI chat failed: \${response.status}\`, details }, { status: response.status });
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
return stream ? passthroughResponse(response) : jsonResponse(response);
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
case "ollama": {
|
|
1218
|
+
const { provider: _provider, ...cleanBody } = body;
|
|
1219
|
+
const requestBody = {
|
|
1220
|
+
...cleanBody,
|
|
1221
|
+
stream,
|
|
1222
|
+
model: stripPrefix(body.model ?? DEFAULT_MODEL, "ollama", "llama3.1"),
|
|
1223
|
+
};
|
|
1224
|
+
|
|
1225
|
+
const response = await fetch(\`\${OLLAMA_URL}/api/chat\`, {
|
|
1226
|
+
method: "POST",
|
|
1227
|
+
headers: {
|
|
1228
|
+
"Content-Type": "application/json",
|
|
1229
|
+
},
|
|
1230
|
+
body: JSON.stringify(requestBody),
|
|
1231
|
+
});
|
|
1232
|
+
|
|
1233
|
+
if (!response.ok) {
|
|
1234
|
+
const details = await response.text();
|
|
1235
|
+
return NextResponse.json({ error: \`Ollama chat failed: \${response.status}\`, details }, { status: response.status });
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
return stream ? passthroughResponse(response) : jsonResponse(response);
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
default:
|
|
1242
|
+
return errorResponse(400, \`Unsupported provider: \${provider}\`);
|
|
1243
|
+
}
|
|
1244
|
+
} catch (error) {
|
|
1245
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1246
|
+
const status = message.startsWith("Missing") ? 400 : 500;
|
|
1247
|
+
return errorResponse(status, error);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
`;
|
|
1252
|
+
var NEXT_HEALTH_ROUTE_TEMPLATE = `import { NextResponse } from "next/server";
|
|
1253
|
+
|
|
1254
|
+
export const dynamic = "force-dynamic";
|
|
1255
|
+
|
|
1256
|
+
const QUICKSTART_VERSION = "0.1.0";
|
|
1257
|
+
const OLLAMA_URL = (process.env.OLLAMA_URL ?? "http://localhost:11434").replace(/\\/$/, "");
|
|
1258
|
+
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
|
|
1259
|
+
const AZURE_OPENAI_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT ? process.env.AZURE_OPENAI_ENDPOINT.replace(/\\/$/, "") : undefined;
|
|
1260
|
+
const AZURE_OPENAI_API_KEY = process.env.AZURE_OPENAI_API_KEY;
|
|
1261
|
+
const AZURE_OPENAI_API_VERSION = process.env.AZURE_OPENAI_API_VERSION ?? "2024-08-01-preview";
|
|
1262
|
+
const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
|
|
1263
|
+
const ANTHROPIC_BASE_URL = (process.env.ANTHROPIC_BASE_URL ?? "https://api.anthropic.com").replace(/\\/$/, "");
|
|
1264
|
+
const ANTHROPIC_API_VERSION = process.env.ANTHROPIC_API_VERSION ?? "2023-06-01";
|
|
1265
|
+
const XAI_API_KEY = process.env.XAI_API_KEY;
|
|
1266
|
+
const XAI_BASE_URL = (process.env.XAI_BASE_URL ?? "https://api.x.ai/v1").replace(/\\/$/, "");
|
|
1267
|
+
|
|
1268
|
+
const isAzureConfigured = () => Boolean(AZURE_OPENAI_ENDPOINT && AZURE_OPENAI_API_KEY);
|
|
1269
|
+
|
|
1270
|
+
const buildAzurePath = (path: string) => {
|
|
1271
|
+
const normalized = path.replace(/^\\/+/, "");
|
|
1272
|
+
if (!AZURE_OPENAI_ENDPOINT) {
|
|
1273
|
+
throw new Error("Missing AZURE_OPENAI_ENDPOINT. Add it to your .env file to route requests to Azure OpenAI.");
|
|
1274
|
+
}
|
|
1275
|
+
return \`\${AZURE_OPENAI_ENDPOINT}/openai/\${normalized}?api-version=\${AZURE_OPENAI_API_VERSION}\`;
|
|
1276
|
+
};
|
|
1277
|
+
|
|
1278
|
+
export async function GET() {
|
|
1279
|
+
const providers: Array<Record<string, unknown>> = [];
|
|
1280
|
+
|
|
1281
|
+
// OpenAI
|
|
1282
|
+
try {
|
|
1283
|
+
if (OPENAI_API_KEY) {
|
|
1284
|
+
const response = await fetch("https://api.openai.com/v1/models", {
|
|
1285
|
+
headers: { Authorization: \`Bearer \${OPENAI_API_KEY}\` },
|
|
1286
|
+
});
|
|
1287
|
+
providers.push({
|
|
1288
|
+
name: "openai",
|
|
1289
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1290
|
+
provider: "openai",
|
|
1291
|
+
});
|
|
1292
|
+
} else {
|
|
1293
|
+
providers.push({
|
|
1294
|
+
name: "openai",
|
|
1295
|
+
status: "unconfigured",
|
|
1296
|
+
provider: "openai",
|
|
1297
|
+
error: "API key not configured",
|
|
1298
|
+
});
|
|
1299
|
+
}
|
|
1300
|
+
} catch (error) {
|
|
1301
|
+
providers.push({
|
|
1302
|
+
name: "openai",
|
|
1303
|
+
status: "unhealthy",
|
|
1304
|
+
provider: "openai",
|
|
1305
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
// Azure
|
|
1310
|
+
if (AZURE_OPENAI_ENDPOINT || AZURE_OPENAI_API_KEY) {
|
|
1311
|
+
if (!isAzureConfigured()) {
|
|
1312
|
+
providers.push({
|
|
1313
|
+
name: "azure",
|
|
1314
|
+
status: "unconfigured",
|
|
1315
|
+
provider: "azure",
|
|
1316
|
+
error: "Endpoint or API key not configured",
|
|
1317
|
+
endpoint: AZURE_OPENAI_ENDPOINT,
|
|
1318
|
+
});
|
|
1319
|
+
} else {
|
|
1320
|
+
try {
|
|
1321
|
+
const response = await fetch(buildAzurePath("deployments"), {
|
|
1322
|
+
headers: { "api-key": AZURE_OPENAI_API_KEY ?? "" },
|
|
1323
|
+
});
|
|
1324
|
+
providers.push({
|
|
1325
|
+
name: "azure",
|
|
1326
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1327
|
+
provider: "azure",
|
|
1328
|
+
endpoint: AZURE_OPENAI_ENDPOINT,
|
|
1329
|
+
});
|
|
1330
|
+
} catch (error) {
|
|
1331
|
+
providers.push({
|
|
1332
|
+
name: "azure",
|
|
1333
|
+
status: "unhealthy",
|
|
1334
|
+
provider: "azure",
|
|
1335
|
+
endpoint: AZURE_OPENAI_ENDPOINT,
|
|
1336
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
} else {
|
|
1341
|
+
providers.push({
|
|
1342
|
+
name: "azure",
|
|
1343
|
+
status: "unconfigured",
|
|
1344
|
+
provider: "azure",
|
|
1345
|
+
error: "Endpoint or API key not configured",
|
|
1346
|
+
});
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
// Anthropic
|
|
1350
|
+
if (ANTHROPIC_API_KEY) {
|
|
1351
|
+
try {
|
|
1352
|
+
const response = await fetch(\`\${ANTHROPIC_BASE_URL}/v1/models\`, {
|
|
1353
|
+
headers: {
|
|
1354
|
+
"x-api-key": ANTHROPIC_API_KEY,
|
|
1355
|
+
"anthropic-version": ANTHROPIC_API_VERSION,
|
|
1356
|
+
},
|
|
1357
|
+
});
|
|
1358
|
+
providers.push({
|
|
1359
|
+
name: "anthropic",
|
|
1360
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1361
|
+
provider: "anthropic",
|
|
1362
|
+
endpoint: ANTHROPIC_BASE_URL,
|
|
1363
|
+
});
|
|
1364
|
+
} catch (error) {
|
|
1365
|
+
providers.push({
|
|
1366
|
+
name: "anthropic",
|
|
1367
|
+
status: "unhealthy",
|
|
1368
|
+
provider: "anthropic",
|
|
1369
|
+
endpoint: ANTHROPIC_BASE_URL,
|
|
1370
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1371
|
+
});
|
|
1372
|
+
}
|
|
1373
|
+
} else {
|
|
1374
|
+
providers.push({
|
|
1375
|
+
name: "anthropic",
|
|
1376
|
+
status: "unconfigured",
|
|
1377
|
+
provider: "anthropic",
|
|
1378
|
+
error: "API key not configured",
|
|
1379
|
+
});
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
// xAI
|
|
1383
|
+
if (XAI_API_KEY) {
|
|
1384
|
+
try {
|
|
1385
|
+
const response = await fetch(XAI_BASE_URL + "/models", {
|
|
1386
|
+
headers: { Authorization: "Bearer " + XAI_API_KEY },
|
|
1387
|
+
});
|
|
1388
|
+
providers.push({
|
|
1389
|
+
name: "xai",
|
|
1390
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1391
|
+
provider: "xai",
|
|
1392
|
+
endpoint: XAI_BASE_URL,
|
|
1393
|
+
});
|
|
1394
|
+
} catch (error) {
|
|
1395
|
+
providers.push({
|
|
1396
|
+
name: "xai",
|
|
1397
|
+
status: "unhealthy",
|
|
1398
|
+
provider: "xai",
|
|
1399
|
+
endpoint: XAI_BASE_URL,
|
|
1400
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1401
|
+
});
|
|
1402
|
+
}
|
|
1403
|
+
} else {
|
|
1404
|
+
providers.push({
|
|
1405
|
+
name: "xai",
|
|
1406
|
+
status: "unconfigured",
|
|
1407
|
+
provider: "xai",
|
|
1408
|
+
error: "API key not configured",
|
|
1409
|
+
endpoint: XAI_BASE_URL,
|
|
1410
|
+
});
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
// Ollama
|
|
1414
|
+
try {
|
|
1415
|
+
const response = await fetch(\`\${OLLAMA_URL}/api/tags\`);
|
|
1416
|
+
providers.push({
|
|
1417
|
+
name: "ollama",
|
|
1418
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1419
|
+
provider: "ollama",
|
|
1420
|
+
url: OLLAMA_URL,
|
|
1421
|
+
});
|
|
1422
|
+
} catch (error) {
|
|
1423
|
+
providers.push({
|
|
1424
|
+
name: "ollama",
|
|
1425
|
+
status: "offline",
|
|
1426
|
+
provider: "ollama",
|
|
1427
|
+
url: OLLAMA_URL,
|
|
1428
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1429
|
+
});
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
const overallHealthy = providers.some((provider) => provider.status === "healthy");
|
|
1433
|
+
|
|
1434
|
+
return NextResponse.json({
|
|
1435
|
+
status: overallHealthy ? "healthy" : "unhealthy",
|
|
1436
|
+
version: QUICKSTART_VERSION,
|
|
1437
|
+
uptime: Math.round(process.uptime()),
|
|
1438
|
+
providers,
|
|
1439
|
+
});
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
`;
|
|
1443
|
+
var NEXT_MODELS_ROUTE_TEMPLATE = `import { NextResponse } from "next/server";
|
|
1444
|
+
|
|
1445
|
+
export const dynamic = "force-dynamic";
|
|
1446
|
+
|
|
1447
|
+
const BASE_GATEWAY_MODELS = __GATEWAY_MODELS__;
|
|
1448
|
+
|
|
1449
|
+
export function toGatewayModels() {
|
|
1450
|
+
return BASE_GATEWAY_MODELS.map((model) => ({
|
|
1451
|
+
...model,
|
|
1452
|
+
created: Date.now(),
|
|
1453
|
+
modified_at: new Date().toISOString(),
|
|
1454
|
+
size: 0,
|
|
1455
|
+
digest: "",
|
|
1456
|
+
details: {
|
|
1457
|
+
format: "chat",
|
|
1458
|
+
family: model.provider,
|
|
1459
|
+
families: [model.provider],
|
|
1460
|
+
parameter_size: "",
|
|
1461
|
+
quantization_level: "",
|
|
1462
|
+
},
|
|
1463
|
+
}));
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
export async function GET() {
|
|
1467
|
+
return NextResponse.json({ models: toGatewayModels() });
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
`;
|
|
1471
|
+
var NEXT_GATEWAY_README_TEMPLATE = `# Next.js Gateway API
|
|
1472
|
+
|
|
1473
|
+
This directory contains a minimal Next.js App Router implementation of the Bandit gateway API. It mirrors the Express gateway in
|
|
1474
|
+
\`server/gateway.js\` but is ready to drop into a Next.js project.
|
|
1475
|
+
|
|
1476
|
+
## Routes
|
|
1477
|
+
|
|
1478
|
+
- \`app/api/health/route.ts\` \u2013 provider health and availability checks
|
|
1479
|
+
- \`app/api/chat/completions/route.ts\` \u2013 provider-aware chat completions endpoint (OpenAI, Azure OpenAI, Anthropic, xAI, Ollama)
|
|
1480
|
+
- \`app/api/models/route.ts\` \u2013 exposes the scaffolded gateway model metadata used by the frontend
|
|
1481
|
+
|
|
1482
|
+
## Usage
|
|
1483
|
+
|
|
1484
|
+
1. Copy the contents of \`server/next-app/\` into the \`app/\` directory of a Next.js project.
|
|
1485
|
+
2. Ensure the environment variables listed in \`.env.example\` are available to the Next.js runtime. At minimum you will want the
|
|
1486
|
+
provider API keys you plan to use (OpenAI, Azure OpenAI, Anthropic, xAI, or Ollama).
|
|
1487
|
+
3. Start Next.js with \`npm run dev\` (or your project\u2019s equivalent). The routes are server-only (\`export const dynamic = "force-dynamic"\`)
|
|
1488
|
+
and can coexist with any frontend pages.
|
|
1489
|
+
|
|
1490
|
+
The generated routes favour clarity over cleverness so you can extend them with custom auth, logging, and provider routing logic.
|
|
1491
|
+
`;
|
|
1492
|
+
var buildNextChatRoute = (ctx) => {
|
|
1493
|
+
const fallbackModel = ctx.fallbackModelId ? `"${ctx.fallbackModelId}"` : "undefined";
|
|
1494
|
+
return ensureTrailingNewline(
|
|
1495
|
+
normalizeLineEndings(
|
|
1496
|
+
NEXT_CHAT_ROUTE_TEMPLATE.replace(/__DEFAULT_PROVIDER__/g, ctx.defaultProvider).replace(/__DEFAULT_MODEL__/g, ctx.defaultModelId).replace(/__FALLBACK_MODEL__/g, fallbackModel)
|
|
1497
|
+
)
|
|
1498
|
+
);
|
|
1499
|
+
};
|
|
1500
|
+
var buildNextHealthRoute = () => ensureTrailingNewline(normalizeLineEndings(NEXT_HEALTH_ROUTE_TEMPLATE));
|
|
1501
|
+
var buildNextModelsRoute = (ctx) => {
|
|
1502
|
+
const modelsDefinition = JSON.stringify(ctx.gatewayModels, null, 2);
|
|
1503
|
+
return ensureTrailingNewline(
|
|
1504
|
+
normalizeLineEndings(
|
|
1505
|
+
NEXT_MODELS_ROUTE_TEMPLATE.replace("__GATEWAY_MODELS__", modelsDefinition)
|
|
1506
|
+
)
|
|
1507
|
+
);
|
|
1508
|
+
};
|
|
1509
|
+
var buildNextGatewayReadme = () => ensureTrailingNewline(normalizeLineEndings(NEXT_GATEWAY_README_TEMPLATE));
|
|
823
1510
|
var buildGatewayServer = (ctx) => {
|
|
824
1511
|
const modelsDefinition = JSON.stringify(ctx.gatewayModels, null, 2);
|
|
825
1512
|
const gatewaySource = `import express from "express";
|
|
@@ -837,6 +1524,20 @@ const QUICKSTART_VERSION = "0.1.0";
|
|
|
837
1524
|
const DEFAULT_PROVIDER = "${ctx.defaultProvider}";
|
|
838
1525
|
const BASE_GATEWAY_MODELS = ${modelsDefinition};
|
|
839
1526
|
const OLLAMA_BASE_URL = (process.env.OLLAMA_URL ?? "http://localhost:11434").replace(/\\/$/, "");
|
|
1527
|
+
const AZURE_OPENAI_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT ? process.env.AZURE_OPENAI_ENDPOINT.replace(/\\/$/, "") : undefined;
|
|
1528
|
+
const AZURE_OPENAI_API_KEY = process.env.AZURE_OPENAI_API_KEY;
|
|
1529
|
+
const AZURE_OPENAI_API_VERSION = process.env.AZURE_OPENAI_API_VERSION ?? "2024-08-01-preview";
|
|
1530
|
+
const AZURE_OPENAI_CHAT_DEPLOYMENT = process.env.AZURE_OPENAI_CHAT_DEPLOYMENT;
|
|
1531
|
+
const AZURE_OPENAI_COMPLETIONS_DEPLOYMENT = process.env.AZURE_OPENAI_COMPLETIONS_DEPLOYMENT ?? AZURE_OPENAI_CHAT_DEPLOYMENT;
|
|
1532
|
+
const AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT = process.env.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT;
|
|
1533
|
+
const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
|
|
1534
|
+
const ANTHROPIC_BASE_URL = (process.env.ANTHROPIC_BASE_URL ?? "https://api.anthropic.com").replace(/\\/$/, "");
|
|
1535
|
+
const ANTHROPIC_API_VERSION = process.env.ANTHROPIC_API_VERSION ?? "2023-06-01";
|
|
1536
|
+
const ANTHROPIC_MAX_TOKENS = Number.isFinite(Number(process.env.ANTHROPIC_MAX_TOKENS))
|
|
1537
|
+
? Number(process.env.ANTHROPIC_MAX_TOKENS)
|
|
1538
|
+
: 1024;
|
|
1539
|
+
const XAI_API_KEY = process.env.XAI_API_KEY;
|
|
1540
|
+
const XAI_BASE_URL = (process.env.XAI_BASE_URL ?? "https://api.x.ai/v1").replace(/\\/$/, "");
|
|
840
1541
|
|
|
841
1542
|
const toGatewayModels = () =>
|
|
842
1543
|
BASE_GATEWAY_MODELS.map((model) => ({
|
|
@@ -854,113 +1555,1177 @@ const toGatewayModels = () =>
|
|
|
854
1555
|
},
|
|
855
1556
|
}));
|
|
856
1557
|
|
|
857
|
-
const
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
1558
|
+
const stripAzureModelPrefix = (value) =>
|
|
1559
|
+
typeof value === "string" ? value.replace(/^azure:/, "") : undefined;
|
|
1560
|
+
|
|
1561
|
+
const isAzureConfigured = () => Boolean(AZURE_OPENAI_ENDPOINT && AZURE_OPENAI_API_KEY);
|
|
1562
|
+
|
|
1563
|
+
const requireAzureBaseConfig = () => {
|
|
1564
|
+
if (!AZURE_OPENAI_ENDPOINT) {
|
|
1565
|
+
throw new Error("Missing AZURE_OPENAI_ENDPOINT. Add it to your .env file to route requests to Azure OpenAI.");
|
|
1566
|
+
}
|
|
1567
|
+
if (!AZURE_OPENAI_API_KEY) {
|
|
1568
|
+
throw new Error("Missing AZURE_OPENAI_API_KEY. Add it to your .env file to route requests to Azure OpenAI.");
|
|
1569
|
+
}
|
|
1570
|
+
return {
|
|
1571
|
+
endpoint: AZURE_OPENAI_ENDPOINT,
|
|
1572
|
+
apiKey: AZURE_OPENAI_API_KEY,
|
|
1573
|
+
apiVersion: AZURE_OPENAI_API_VERSION,
|
|
1574
|
+
};
|
|
1575
|
+
};
|
|
1576
|
+
|
|
1577
|
+
const resolveAzureDeployment = (explicitValue, fallbackValue, kind) => {
|
|
1578
|
+
const fromRequest = stripAzureModelPrefix(explicitValue);
|
|
1579
|
+
if (fromRequest) {
|
|
1580
|
+
return fromRequest;
|
|
1581
|
+
}
|
|
1582
|
+
if (fallbackValue) {
|
|
1583
|
+
return fallbackValue;
|
|
1584
|
+
}
|
|
1585
|
+
throw new Error(\`Missing Azure OpenAI \${kind} deployment name. Set AZURE_OPENAI_\${kind.toUpperCase()}_DEPLOYMENT in your .env file.\`);
|
|
1586
|
+
};
|
|
1587
|
+
|
|
1588
|
+
const buildAzureDeploymentUrl = (deployment, suffix) => {
|
|
1589
|
+
const { endpoint } = requireAzureBaseConfig();
|
|
1590
|
+
const normalizedSuffix = suffix.replace(/^\\//, "");
|
|
1591
|
+
return \`\${endpoint}/openai/deployments/\${deployment}/\${normalizedSuffix}?api-version=\${AZURE_OPENAI_API_VERSION}\`;
|
|
1592
|
+
};
|
|
1593
|
+
|
|
1594
|
+
const buildAzurePath = (suffix) => {
|
|
1595
|
+
const { endpoint } = requireAzureBaseConfig();
|
|
1596
|
+
const normalizedSuffix = suffix.replace(/^\\//, "");
|
|
1597
|
+
const hasQuery = normalizedSuffix.includes("?");
|
|
1598
|
+
const separator = hasQuery ? "&" : "?";
|
|
1599
|
+
return \`\${endpoint}/openai/\${normalizedSuffix}\${separator}api-version=\${AZURE_OPENAI_API_VERSION}\`;
|
|
1600
|
+
};
|
|
1601
|
+
|
|
1602
|
+
const stripAnthropicModelPrefix = (value) =>
|
|
1603
|
+
typeof value === "string" ? value.replace(/^anthropic:/, "") : undefined;
|
|
1604
|
+
|
|
1605
|
+
const isAnthropicConfigured = () => Boolean(ANTHROPIC_API_KEY);
|
|
1606
|
+
|
|
1607
|
+
const requireAnthropicKey = () => {
|
|
1608
|
+
if (!ANTHROPIC_API_KEY) {
|
|
1609
|
+
throw new Error("Missing ANTHROPIC_API_KEY. Add it to your .env file to route requests to Anthropic.");
|
|
1610
|
+
}
|
|
1611
|
+
return ANTHROPIC_API_KEY;
|
|
1612
|
+
};
|
|
1613
|
+
|
|
1614
|
+
const buildAnthropicUrl = (path) => {
|
|
1615
|
+
const normalized = path.replace(/^\\//, "");
|
|
1616
|
+
return \`\${ANTHROPIC_BASE_URL}/v1/\${normalized}\`;
|
|
1617
|
+
};
|
|
1618
|
+
|
|
1619
|
+
const buildAnthropicHeaders = () => ({
|
|
1620
|
+
"Content-Type": "application/json",
|
|
1621
|
+
"x-api-key": requireAnthropicKey(),
|
|
1622
|
+
"anthropic-version": ANTHROPIC_API_VERSION,
|
|
1623
|
+
});
|
|
1624
|
+
|
|
1625
|
+
const flattenGatewayContent = (content) => {
|
|
1626
|
+
if (typeof content === "string") {
|
|
1627
|
+
return content;
|
|
1628
|
+
}
|
|
1629
|
+
if (Array.isArray(content)) {
|
|
1630
|
+
return content
|
|
1631
|
+
.map((part) => {
|
|
1632
|
+
if (typeof part === "string") {
|
|
1633
|
+
return part;
|
|
1634
|
+
}
|
|
1635
|
+
if (part?.type === "text" && typeof part.text === "string") {
|
|
1636
|
+
return part.text;
|
|
1637
|
+
}
|
|
1638
|
+
if (part?.type === "image_url" && part.image_url?.url) {
|
|
1639
|
+
return \`[Image: \${part.image_url.url}]\`;
|
|
1640
|
+
}
|
|
1641
|
+
return JSON.stringify(part ?? {});
|
|
1642
|
+
})
|
|
1643
|
+
.join("\\n");
|
|
1644
|
+
}
|
|
1645
|
+
if (content && typeof content === "object") {
|
|
1646
|
+
return JSON.stringify(content);
|
|
1647
|
+
}
|
|
1648
|
+
return "";
|
|
1649
|
+
};
|
|
1650
|
+
|
|
1651
|
+
const toAnthropicMessages = (messages = []) => {
|
|
1652
|
+
const anthropicMessages = [];
|
|
1653
|
+
let systemPrompt = "";
|
|
1654
|
+
|
|
1655
|
+
for (const message of messages) {
|
|
1656
|
+
if (!message) continue;
|
|
1657
|
+
const text = flattenGatewayContent(message.content);
|
|
1658
|
+
|
|
1659
|
+
if (message.role === "system") {
|
|
1660
|
+
systemPrompt = systemPrompt ? \`\${systemPrompt}\\n\\n\${text}\` : text;
|
|
1661
|
+
continue;
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
const role = message.role === "assistant" ? "assistant" : "user";
|
|
1665
|
+
anthropicMessages.push({
|
|
1666
|
+
role,
|
|
1667
|
+
content: [{ type: "text", text }],
|
|
1668
|
+
});
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
return { messages: anthropicMessages, system: systemPrompt || undefined };
|
|
1672
|
+
};
|
|
1673
|
+
|
|
1674
|
+
const convertAnthropicResponseToGateway = (responseBody, modelName) => {
|
|
1675
|
+
if (!responseBody) {
|
|
1676
|
+
return {
|
|
1677
|
+
id: \`anthropic-\${Date.now()}\`,
|
|
1678
|
+
object: "chat.completion",
|
|
1679
|
+
created: Math.floor(Date.now() / 1000),
|
|
1680
|
+
model: modelName.startsWith("anthropic:") ? modelName : \`anthropic:\${modelName}\`,
|
|
1681
|
+
choices: [],
|
|
1682
|
+
};
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
const textContent = Array.isArray(responseBody.content)
|
|
1686
|
+
? responseBody.content
|
|
1687
|
+
.filter((item) => item && item.type === "text" && typeof item.text === "string")
|
|
1688
|
+
.map((item) => item.text)
|
|
1689
|
+
.join("\\n")
|
|
1690
|
+
: typeof responseBody.content === "string"
|
|
1691
|
+
? responseBody.content
|
|
1692
|
+
: "";
|
|
1693
|
+
|
|
1694
|
+
const promptTokens = responseBody.usage?.input_tokens ?? 0;
|
|
1695
|
+
const completionTokens = responseBody.usage?.output_tokens ?? 0;
|
|
1696
|
+
|
|
1697
|
+
return {
|
|
1698
|
+
id: responseBody.id ?? \`anthropic-\${Date.now()}\`,
|
|
1699
|
+
object: "chat.completion",
|
|
1700
|
+
created: Math.floor(Date.now() / 1000),
|
|
1701
|
+
model: modelName.startsWith("anthropic:") ? modelName : \`anthropic:\${modelName}\`,
|
|
1702
|
+
choices: [
|
|
1703
|
+
{
|
|
1704
|
+
index: 0,
|
|
1705
|
+
message: {
|
|
1706
|
+
role: responseBody.role ?? "assistant",
|
|
1707
|
+
content: textContent,
|
|
1708
|
+
},
|
|
1709
|
+
finish_reason: responseBody.stop_reason ?? responseBody.stop_sequence ?? null,
|
|
1710
|
+
},
|
|
1711
|
+
],
|
|
1712
|
+
usage: responseBody.usage
|
|
1713
|
+
? {
|
|
1714
|
+
prompt_tokens: promptTokens,
|
|
1715
|
+
completion_tokens: completionTokens,
|
|
1716
|
+
total_tokens: promptTokens + completionTokens,
|
|
1717
|
+
}
|
|
1718
|
+
: undefined,
|
|
1719
|
+
};
|
|
1720
|
+
};
|
|
1721
|
+
|
|
1722
|
+
const convertAnthropicResponseToGenerate = (responseBody, modelName) => {
|
|
1723
|
+
const gatewayResponse = convertAnthropicResponseToGateway(responseBody, modelName);
|
|
1724
|
+
const content = gatewayResponse.choices?.[0]?.message?.content ?? "";
|
|
1725
|
+
return {
|
|
1726
|
+
model: gatewayResponse.model,
|
|
1727
|
+
created_at: new Date().toISOString(),
|
|
1728
|
+
response: content,
|
|
1729
|
+
done: true,
|
|
1730
|
+
total_duration: 0,
|
|
1731
|
+
load_duration: 0,
|
|
1732
|
+
prompt_eval_count: gatewayResponse.usage?.prompt_tokens ?? 0,
|
|
1733
|
+
prompt_eval_duration: 0,
|
|
1734
|
+
eval_count: gatewayResponse.usage?.completion_tokens ?? 0,
|
|
1735
|
+
eval_duration: 0,
|
|
1736
|
+
};
|
|
1737
|
+
};
|
|
1738
|
+
|
|
1739
|
+
const requireOpenAIKey = () => {
|
|
1740
|
+
const key = process.env.OPENAI_API_KEY;
|
|
1741
|
+
if (!key) {
|
|
1742
|
+
throw new Error("Missing OPENAI_API_KEY. Add it to your .env file to route requests to OpenAI.");
|
|
1743
|
+
}
|
|
1744
|
+
return key;
|
|
1745
|
+
};
|
|
1746
|
+
|
|
1747
|
+
const requireXAIKey = () => {
|
|
1748
|
+
const key = XAI_API_KEY;
|
|
1749
|
+
if (!key) {
|
|
1750
|
+
throw new Error("Missing XAI_API_KEY. Add it to your .env file to route requests to xAI.");
|
|
1751
|
+
}
|
|
1752
|
+
return key;
|
|
1753
|
+
};
|
|
1754
|
+
|
|
1755
|
+
// Utility function to handle streaming responses
|
|
1756
|
+
const handleStreamingResponse = async (upstreamResponse, res) => {
|
|
1757
|
+
res.setHeader('Content-Type', 'text/event-stream');
|
|
1758
|
+
res.setHeader('Cache-Control', 'no-cache');
|
|
1759
|
+
res.setHeader('Connection', 'keep-alive');
|
|
1760
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
1761
|
+
|
|
1762
|
+
try {
|
|
1763
|
+
// Get the readable stream from the response
|
|
1764
|
+
const reader = upstreamResponse.body.getReader();
|
|
1765
|
+
|
|
1766
|
+
while (true) {
|
|
1767
|
+
const { done, value } = await reader.read();
|
|
1768
|
+
if (done) break;
|
|
1769
|
+
|
|
1770
|
+
// Write the chunk to the response
|
|
1771
|
+
res.write(value);
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
res.end();
|
|
1775
|
+
} catch (error) {
|
|
1776
|
+
console.error('Streaming error:', error);
|
|
1777
|
+
// Fallback to non-streaming
|
|
1778
|
+
const text = await upstreamResponse.text();
|
|
1779
|
+
res.send(text);
|
|
1780
|
+
}
|
|
1781
|
+
};
|
|
1782
|
+
|
|
1783
|
+
const relayAnthropicStream = async (upstreamResponse, res) => {
|
|
1784
|
+
res.setHeader('Content-Type', 'text/event-stream');
|
|
1785
|
+
res.setHeader('Cache-Control', 'no-cache');
|
|
1786
|
+
res.setHeader('Connection', 'keep-alive');
|
|
1787
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
1788
|
+
|
|
1789
|
+
const reader = upstreamResponse.body?.getReader();
|
|
1790
|
+
if (!reader) {
|
|
1791
|
+
const fallback = await upstreamResponse.text();
|
|
1792
|
+
res.write("data: " + JSON.stringify({ choices: [{ delta: { content: fallback } }] }) + "\\n\\n");
|
|
1793
|
+
res.write("data: [DONE]\\n\\n");
|
|
1794
|
+
return res.end();
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
const decoder = new TextDecoder();
|
|
1798
|
+
let buffer = '';
|
|
1799
|
+
|
|
1800
|
+
const sendChunk = (payload) => {
|
|
1801
|
+
res.write("data: " + JSON.stringify(payload) + "\\n\\n");
|
|
1802
|
+
};
|
|
1803
|
+
|
|
1804
|
+
try {
|
|
1805
|
+
while (true) {
|
|
1806
|
+
const { value, done } = await reader.read();
|
|
1807
|
+
if (done) break;
|
|
1808
|
+
buffer += decoder.decode(value, { stream: true });
|
|
1809
|
+
|
|
1810
|
+
let delimiterIndex;
|
|
1811
|
+
while ((delimiterIndex = buffer.indexOf('\\n\\n')) >= 0) {
|
|
1812
|
+
const rawEvent = buffer.slice(0, delimiterIndex).trim();
|
|
1813
|
+
buffer = buffer.slice(delimiterIndex + 2);
|
|
1814
|
+
if (!rawEvent) continue;
|
|
1815
|
+
|
|
1816
|
+
const lines = rawEvent.split('\\n');
|
|
1817
|
+
const eventLine = lines.find((line) => line.startsWith('event:')) ?? '';
|
|
1818
|
+
const dataLine = lines.find((line) => line.startsWith('data:')) ?? '';
|
|
1819
|
+
const event = eventLine.replace('event:', '').trim();
|
|
1820
|
+
const trimmedData = dataLine.replace('data:', '').trim();
|
|
1821
|
+
|
|
1822
|
+
if (!trimmedData) {
|
|
1823
|
+
continue;
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
let parsed;
|
|
1827
|
+
try {
|
|
1828
|
+
parsed = JSON.parse(trimmedData);
|
|
1829
|
+
} catch (error) {
|
|
1830
|
+
console.error('Anthropic stream parse error', error, { rawEvent });
|
|
1831
|
+
continue;
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
if (event === 'content_block_delta') {
|
|
1835
|
+
const textChunk = parsed?.delta?.text ?? '';
|
|
1836
|
+
if (textChunk) {
|
|
1837
|
+
sendChunk({
|
|
1838
|
+
choices: [
|
|
1839
|
+
{
|
|
1840
|
+
delta: {
|
|
1841
|
+
content: textChunk,
|
|
1842
|
+
},
|
|
1843
|
+
},
|
|
1844
|
+
],
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
} else if (event === 'message_stop') {
|
|
1848
|
+
sendChunk({
|
|
1849
|
+
choices: [
|
|
1850
|
+
{
|
|
1851
|
+
delta: {},
|
|
1852
|
+
finish_reason: 'stop',
|
|
1853
|
+
},
|
|
1854
|
+
],
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
} catch (error) {
|
|
1860
|
+
console.error('Anthropic streaming relay error', error);
|
|
1861
|
+
sendChunk({
|
|
1862
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1863
|
+
});
|
|
1864
|
+
} finally {
|
|
1865
|
+
res.write("data: [DONE]\\n\\n");
|
|
1866
|
+
res.end();
|
|
1867
|
+
}
|
|
1868
|
+
};
|
|
1869
|
+
|
|
1870
|
+
// ============================================================================
|
|
1871
|
+
// GENERAL HEALTH & MODELS
|
|
1872
|
+
// ============================================================================
|
|
1873
|
+
|
|
1874
|
+
app.get("/api/health", async (_req, res) => {
|
|
1875
|
+
const providers = [];
|
|
1876
|
+
|
|
1877
|
+
// Check OpenAI
|
|
1878
|
+
try {
|
|
1879
|
+
const openaiKey = process.env.OPENAI_API_KEY;
|
|
1880
|
+
if (openaiKey) {
|
|
1881
|
+
const response = await fetch("https://api.openai.com/v1/models", {
|
|
1882
|
+
headers: { "Authorization": \`Bearer \${openaiKey}\` }
|
|
1883
|
+
});
|
|
1884
|
+
providers.push({
|
|
1885
|
+
name: "openai",
|
|
1886
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1887
|
+
provider: "openai"
|
|
1888
|
+
});
|
|
1889
|
+
} else {
|
|
1890
|
+
providers.push({
|
|
1891
|
+
name: "openai",
|
|
1892
|
+
status: "unconfigured",
|
|
1893
|
+
provider: "openai",
|
|
1894
|
+
error: "API key not configured"
|
|
1895
|
+
});
|
|
1896
|
+
}
|
|
1897
|
+
} catch (error) {
|
|
1898
|
+
providers.push({
|
|
1899
|
+
name: "openai",
|
|
1900
|
+
status: "unhealthy",
|
|
1901
|
+
provider: "openai",
|
|
1902
|
+
error: error.message
|
|
1903
|
+
});
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
// Check Azure OpenAI
|
|
1907
|
+
if (AZURE_OPENAI_ENDPOINT || AZURE_OPENAI_API_KEY) {
|
|
1908
|
+
if (!isAzureConfigured()) {
|
|
1909
|
+
providers.push({
|
|
1910
|
+
name: "azure",
|
|
1911
|
+
status: "unconfigured",
|
|
1912
|
+
provider: "azure",
|
|
1913
|
+
error: "Endpoint or API key not configured",
|
|
1914
|
+
endpoint: AZURE_OPENAI_ENDPOINT
|
|
1915
|
+
});
|
|
1916
|
+
} else {
|
|
1917
|
+
try {
|
|
1918
|
+
const { endpoint } = requireAzureBaseConfig();
|
|
1919
|
+
const deploymentsUrl = buildAzurePath("deployments");
|
|
1920
|
+
const response = await fetch(deploymentsUrl, {
|
|
1921
|
+
headers: { "api-key": AZURE_OPENAI_API_KEY }
|
|
1922
|
+
});
|
|
1923
|
+
providers.push({
|
|
1924
|
+
name: "azure",
|
|
1925
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1926
|
+
provider: "azure",
|
|
1927
|
+
endpoint
|
|
1928
|
+
});
|
|
1929
|
+
} catch (error) {
|
|
1930
|
+
providers.push({
|
|
1931
|
+
name: "azure",
|
|
1932
|
+
status: "unhealthy",
|
|
1933
|
+
provider: "azure",
|
|
1934
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1935
|
+
endpoint: AZURE_OPENAI_ENDPOINT
|
|
1936
|
+
});
|
|
1937
|
+
}
|
|
1938
|
+
}
|
|
1939
|
+
} else {
|
|
1940
|
+
providers.push({
|
|
1941
|
+
name: "azure",
|
|
1942
|
+
status: "unconfigured",
|
|
1943
|
+
provider: "azure",
|
|
1944
|
+
error: "Endpoint or API key not configured"
|
|
1945
|
+
});
|
|
1946
|
+
}
|
|
1947
|
+
|
|
1948
|
+
// Check Anthropic
|
|
1949
|
+
if (ANTHROPIC_API_KEY) {
|
|
1950
|
+
try {
|
|
1951
|
+
const response = await fetch(buildAnthropicUrl("models"), {
|
|
1952
|
+
headers: buildAnthropicHeaders(),
|
|
1953
|
+
method: "GET"
|
|
1954
|
+
});
|
|
1955
|
+
providers.push({
|
|
1956
|
+
name: "anthropic",
|
|
1957
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1958
|
+
provider: "anthropic",
|
|
1959
|
+
endpoint: ANTHROPIC_BASE_URL
|
|
1960
|
+
});
|
|
1961
|
+
} catch (error) {
|
|
1962
|
+
providers.push({
|
|
1963
|
+
name: "anthropic",
|
|
1964
|
+
status: "unhealthy",
|
|
1965
|
+
provider: "anthropic",
|
|
1966
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1967
|
+
endpoint: ANTHROPIC_BASE_URL
|
|
1968
|
+
});
|
|
1969
|
+
}
|
|
1970
|
+
} else {
|
|
1971
|
+
providers.push({
|
|
1972
|
+
name: "anthropic",
|
|
1973
|
+
status: "unconfigured",
|
|
1974
|
+
provider: "anthropic",
|
|
1975
|
+
error: "API key not configured"
|
|
1976
|
+
});
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
// Check xAI
|
|
1980
|
+
if (XAI_API_KEY) {
|
|
1981
|
+
try {
|
|
1982
|
+
const response = await fetch(XAI_BASE_URL + "/models", {
|
|
1983
|
+
headers: { "Authorization": "Bearer " + XAI_API_KEY }
|
|
1984
|
+
});
|
|
1985
|
+
providers.push({
|
|
1986
|
+
name: "xai",
|
|
1987
|
+
status: response.ok ? "healthy" : "unhealthy",
|
|
1988
|
+
provider: "xai",
|
|
1989
|
+
endpoint: XAI_BASE_URL
|
|
1990
|
+
});
|
|
1991
|
+
} catch (error) {
|
|
1992
|
+
providers.push({
|
|
1993
|
+
name: "xai",
|
|
1994
|
+
status: "unhealthy",
|
|
1995
|
+
provider: "xai",
|
|
1996
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1997
|
+
endpoint: XAI_BASE_URL
|
|
1998
|
+
});
|
|
1999
|
+
}
|
|
2000
|
+
} else {
|
|
2001
|
+
providers.push({
|
|
2002
|
+
name: "xai",
|
|
2003
|
+
status: "unconfigured",
|
|
2004
|
+
provider: "xai",
|
|
2005
|
+
error: "API key not configured",
|
|
2006
|
+
endpoint: XAI_BASE_URL
|
|
2007
|
+
});
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
// Check Ollama
|
|
2011
|
+
try {
|
|
2012
|
+
console.log(\`Checking Ollama health at: \${OLLAMA_BASE_URL}/api/tags\`);
|
|
2013
|
+
const response = await fetch(\`\${OLLAMA_BASE_URL}/api/tags\`);
|
|
2014
|
+
const status = response.ok ? "healthy" : "unhealthy";
|
|
2015
|
+
console.log(\`Ollama health check result: \${status}\`);
|
|
2016
|
+
providers.push({
|
|
2017
|
+
name: "ollama",
|
|
2018
|
+
status: status,
|
|
2019
|
+
provider: "ollama",
|
|
2020
|
+
url: OLLAMA_BASE_URL
|
|
2021
|
+
});
|
|
2022
|
+
} catch (error) {
|
|
2023
|
+
console.log(\`Ollama health check error: \${error.message}\`);
|
|
2024
|
+
providers.push({
|
|
2025
|
+
name: "ollama",
|
|
2026
|
+
status: "offline",
|
|
2027
|
+
provider: "ollama",
|
|
2028
|
+
error: error.message,
|
|
2029
|
+
url: OLLAMA_BASE_URL
|
|
2030
|
+
});
|
|
2031
|
+
}
|
|
2032
|
+
|
|
2033
|
+
const overallHealthy = providers.some(p => p.status === "healthy");
|
|
2034
|
+
|
|
2035
|
+
res.json({
|
|
2036
|
+
status: overallHealthy ? "healthy" : "unhealthy",
|
|
2037
|
+
version: QUICKSTART_VERSION,
|
|
2038
|
+
uptime: Math.round(process.uptime()),
|
|
2039
|
+
providers
|
|
2040
|
+
});
|
|
2041
|
+
});
|
|
2042
|
+
|
|
2043
|
+
app.get("/api/models", (_req, res) => {
|
|
2044
|
+
res.json({ models: toGatewayModels() });
|
|
2045
|
+
});
|
|
2046
|
+
|
|
2047
|
+
// ============================================================================
|
|
2048
|
+
// ANTHROPIC ROUTES
|
|
2049
|
+
// ============================================================================
|
|
2050
|
+
|
|
2051
|
+
app.get("/api/anthropic/health", async (_req, res) => {
|
|
2052
|
+
try {
|
|
2053
|
+
requireAnthropicKey();
|
|
2054
|
+
const response = await fetch(buildAnthropicUrl("models"), {
|
|
2055
|
+
method: "GET",
|
|
2056
|
+
headers: buildAnthropicHeaders()
|
|
2057
|
+
});
|
|
2058
|
+
const isHealthy = response.ok;
|
|
2059
|
+
res.json({
|
|
2060
|
+
status: isHealthy ? "healthy" : "unhealthy",
|
|
2061
|
+
anthropic_status: isHealthy,
|
|
2062
|
+
provider: "anthropic",
|
|
2063
|
+
endpoint: ANTHROPIC_BASE_URL
|
|
2064
|
+
});
|
|
2065
|
+
} catch (error) {
|
|
2066
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2067
|
+
res.status(503).json({
|
|
2068
|
+
status: "unhealthy",
|
|
2069
|
+
anthropic_status: false,
|
|
2070
|
+
provider: "anthropic",
|
|
2071
|
+
error: message,
|
|
2072
|
+
endpoint: ANTHROPIC_BASE_URL
|
|
2073
|
+
});
|
|
2074
|
+
}
|
|
2075
|
+
});
|
|
2076
|
+
|
|
2077
|
+
app.post("/api/anthropic/chat/completions", async (req, res) => {
|
|
2078
|
+
try {
|
|
2079
|
+
requireAnthropicKey();
|
|
2080
|
+
const rawBody = req.body ?? {};
|
|
2081
|
+
const isStreaming = rawBody.stream === true;
|
|
2082
|
+
const requestedModel =
|
|
2083
|
+
stripAnthropicModelPrefix(rawBody.model) ??
|
|
2084
|
+
stripAnthropicModelPrefix("${ctx.defaultModelId}") ??
|
|
2085
|
+
"claude-3-5-haiku-latest";
|
|
2086
|
+
|
|
2087
|
+
const stopSequences = Array.isArray(rawBody.stop)
|
|
2088
|
+
? rawBody.stop
|
|
2089
|
+
: Array.isArray(rawBody.stop_sequences)
|
|
2090
|
+
? rawBody.stop_sequences
|
|
2091
|
+
: rawBody.stop
|
|
2092
|
+
? [rawBody.stop]
|
|
2093
|
+
: undefined;
|
|
2094
|
+
|
|
2095
|
+
const { messages: anthropicMessages, system } = toAnthropicMessages(
|
|
2096
|
+
Array.isArray(rawBody.messages) ? rawBody.messages : []
|
|
2097
|
+
);
|
|
2098
|
+
|
|
2099
|
+
const fallbackText =
|
|
2100
|
+
typeof rawBody.prompt === "string" && rawBody.prompt.trim().length > 0
|
|
2101
|
+
? rawBody.prompt
|
|
2102
|
+
: "Hello from Bandit quickstart gateway";
|
|
2103
|
+
|
|
2104
|
+
const requestBody = {
|
|
2105
|
+
model: requestedModel,
|
|
2106
|
+
messages:
|
|
2107
|
+
anthropicMessages.length > 0
|
|
2108
|
+
? anthropicMessages
|
|
2109
|
+
: [
|
|
2110
|
+
{
|
|
2111
|
+
role: "user",
|
|
2112
|
+
content: [{ type: "text", text: fallbackText }],
|
|
2113
|
+
},
|
|
2114
|
+
],
|
|
2115
|
+
stream: isStreaming,
|
|
2116
|
+
max_tokens:
|
|
2117
|
+
typeof rawBody.max_tokens === "number" && rawBody.max_tokens > 0
|
|
2118
|
+
? rawBody.max_tokens
|
|
2119
|
+
: ANTHROPIC_MAX_TOKENS,
|
|
2120
|
+
};
|
|
2121
|
+
|
|
2122
|
+
if (system) {
|
|
2123
|
+
requestBody.system = system;
|
|
2124
|
+
}
|
|
2125
|
+
if (typeof rawBody.temperature === "number") {
|
|
2126
|
+
requestBody.temperature = rawBody.temperature;
|
|
2127
|
+
}
|
|
2128
|
+
if (typeof rawBody.top_p === "number") {
|
|
2129
|
+
requestBody.top_p = rawBody.top_p;
|
|
2130
|
+
}
|
|
2131
|
+
if (typeof rawBody.top_k === "number") {
|
|
2132
|
+
requestBody.top_k = rawBody.top_k;
|
|
2133
|
+
}
|
|
2134
|
+
if (stopSequences) {
|
|
2135
|
+
requestBody.stop_sequences = stopSequences;
|
|
2136
|
+
}
|
|
2137
|
+
if (rawBody.metadata) {
|
|
2138
|
+
requestBody.metadata = rawBody.metadata;
|
|
2139
|
+
}
|
|
2140
|
+
if (rawBody.tools) {
|
|
2141
|
+
requestBody.tools = rawBody.tools;
|
|
2142
|
+
}
|
|
2143
|
+
if (rawBody.tool_choice) {
|
|
2144
|
+
requestBody.tool_choice = rawBody.tool_choice;
|
|
2145
|
+
}
|
|
2146
|
+
if (rawBody.thinking) {
|
|
2147
|
+
requestBody.thinking = rawBody.thinking;
|
|
2148
|
+
}
|
|
2149
|
+
if (rawBody.extra_headers) {
|
|
2150
|
+
requestBody.extra_headers = rawBody.extra_headers;
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
const response = await fetch(buildAnthropicUrl("messages"), {
|
|
2154
|
+
method: "POST",
|
|
2155
|
+
headers: buildAnthropicHeaders(),
|
|
2156
|
+
body: JSON.stringify(requestBody),
|
|
2157
|
+
});
|
|
2158
|
+
|
|
2159
|
+
if (!response.ok) {
|
|
2160
|
+
const errorText = await response.text();
|
|
2161
|
+
return res.status(response.status).json({
|
|
2162
|
+
error: \`Anthropic chat failed: \${response.status}\`,
|
|
2163
|
+
details: errorText,
|
|
2164
|
+
});
|
|
2165
|
+
}
|
|
2166
|
+
|
|
2167
|
+
if (isStreaming) {
|
|
2168
|
+
await relayAnthropicStream(response, res);
|
|
2169
|
+
} else {
|
|
2170
|
+
const data = await response.json();
|
|
2171
|
+
const normalized = convertAnthropicResponseToGateway(data, requestedModel);
|
|
2172
|
+
res.json(normalized);
|
|
2173
|
+
}
|
|
2174
|
+
} catch (error) {
|
|
2175
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2176
|
+
const status = message.startsWith("Missing ANTHROPIC_API_KEY") ? 400 : 500;
|
|
2177
|
+
res.status(status).json({ error: message });
|
|
2178
|
+
}
|
|
2179
|
+
});
|
|
2180
|
+
|
|
2181
|
+
app.post("/api/anthropic/chat", async (req, res) => {
|
|
2182
|
+
req.url = "/api/anthropic/chat/completions";
|
|
2183
|
+
return app._router.handle(req, res);
|
|
2184
|
+
});
|
|
2185
|
+
|
|
2186
|
+
app.post("/api/anthropic/completions", async (req, res) => {
|
|
2187
|
+
try {
|
|
2188
|
+
requireAnthropicKey();
|
|
2189
|
+
const rawBody = req.body ?? {};
|
|
2190
|
+
const isStreaming = rawBody.stream === true;
|
|
2191
|
+
const requestedModel =
|
|
2192
|
+
stripAnthropicModelPrefix(rawBody.model) ??
|
|
2193
|
+
stripAnthropicModelPrefix("${ctx.defaultModelId}") ??
|
|
2194
|
+
"claude-3-5-sonnet-latest";
|
|
2195
|
+
|
|
2196
|
+
const stopSequences = Array.isArray(rawBody.stop)
|
|
2197
|
+
? rawBody.stop
|
|
2198
|
+
: Array.isArray(rawBody.stop_sequences)
|
|
2199
|
+
? rawBody.stop_sequences
|
|
2200
|
+
: rawBody.stop
|
|
2201
|
+
? [rawBody.stop]
|
|
2202
|
+
: undefined;
|
|
2203
|
+
|
|
2204
|
+
const prompt =
|
|
2205
|
+
typeof rawBody.prompt === "string" && rawBody.prompt.trim().length > 0
|
|
2206
|
+
? rawBody.prompt
|
|
2207
|
+
: "Hello from Bandit quickstart gateway";
|
|
2208
|
+
|
|
2209
|
+
const { messages, system } = toAnthropicMessages([
|
|
2210
|
+
{ role: "user", content: prompt },
|
|
2211
|
+
]);
|
|
2212
|
+
|
|
2213
|
+
const requestBody = {
|
|
2214
|
+
model: requestedModel,
|
|
2215
|
+
messages,
|
|
2216
|
+
stream: isStreaming,
|
|
2217
|
+
max_tokens:
|
|
2218
|
+
typeof rawBody.max_tokens === "number" && rawBody.max_tokens > 0
|
|
2219
|
+
? rawBody.max_tokens
|
|
2220
|
+
: ANTHROPIC_MAX_TOKENS,
|
|
2221
|
+
};
|
|
2222
|
+
|
|
2223
|
+
if (system) {
|
|
2224
|
+
requestBody.system = system;
|
|
2225
|
+
}
|
|
2226
|
+
if (typeof rawBody.temperature === "number") {
|
|
2227
|
+
requestBody.temperature = rawBody.temperature;
|
|
2228
|
+
}
|
|
2229
|
+
if (typeof rawBody.top_p === "number") {
|
|
2230
|
+
requestBody.top_p = rawBody.top_p;
|
|
2231
|
+
}
|
|
2232
|
+
if (typeof rawBody.top_k === "number") {
|
|
2233
|
+
requestBody.top_k = rawBody.top_k;
|
|
2234
|
+
}
|
|
2235
|
+
if (stopSequences) {
|
|
2236
|
+
requestBody.stop_sequences = stopSequences;
|
|
2237
|
+
}
|
|
2238
|
+
if (rawBody.metadata) {
|
|
2239
|
+
requestBody.metadata = rawBody.metadata;
|
|
2240
|
+
}
|
|
2241
|
+
if (rawBody.tools) {
|
|
2242
|
+
requestBody.tools = rawBody.tools;
|
|
2243
|
+
}
|
|
2244
|
+
if (rawBody.tool_choice) {
|
|
2245
|
+
requestBody.tool_choice = rawBody.tool_choice;
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
const response = await fetch(buildAnthropicUrl("messages"), {
|
|
2249
|
+
method: "POST",
|
|
2250
|
+
headers: buildAnthropicHeaders(),
|
|
2251
|
+
body: JSON.stringify(requestBody),
|
|
2252
|
+
});
|
|
2253
|
+
|
|
2254
|
+
if (!response.ok) {
|
|
2255
|
+
const errorText = await response.text();
|
|
2256
|
+
return res.status(response.status).json({
|
|
2257
|
+
error: \`Anthropic completions failed: \${response.status}\`,
|
|
2258
|
+
details: errorText,
|
|
2259
|
+
});
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
if (isStreaming) {
|
|
2263
|
+
await relayAnthropicStream(response, res);
|
|
2264
|
+
} else {
|
|
2265
|
+
const data = await response.json();
|
|
2266
|
+
const formatted = convertAnthropicResponseToGenerate(data, requestedModel);
|
|
2267
|
+
res.json(formatted);
|
|
2268
|
+
}
|
|
2269
|
+
} catch (error) {
|
|
2270
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2271
|
+
const status = message.startsWith("Missing ANTHROPIC_API_KEY") ? 400 : 500;
|
|
2272
|
+
res.status(status).json({ error: message });
|
|
2273
|
+
}
|
|
2274
|
+
});
|
|
2275
|
+
|
|
2276
|
+
app.post("/api/anthropic/generate", async (req, res) => {
|
|
2277
|
+
req.url = "/api/anthropic/completions";
|
|
2278
|
+
return app._router.handle(req, res);
|
|
2279
|
+
});
|
|
2280
|
+
|
|
2281
|
+
app.get("/api/anthropic/models", async (_req, res) => {
|
|
2282
|
+
try {
|
|
2283
|
+
requireAnthropicKey();
|
|
2284
|
+
const response = await fetch(buildAnthropicUrl("models"), {
|
|
2285
|
+
method: "GET",
|
|
2286
|
+
headers: buildAnthropicHeaders(),
|
|
2287
|
+
});
|
|
2288
|
+
|
|
2289
|
+
if (!response.ok) {
|
|
2290
|
+
const errorText = await response.text();
|
|
2291
|
+
return res.status(response.status).json({
|
|
2292
|
+
error: \`Anthropic models failed: \${response.status}\`,
|
|
2293
|
+
details: errorText,
|
|
2294
|
+
});
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
const text = await response.text();
|
|
2298
|
+
res.setHeader('Content-Type', 'application/json');
|
|
2299
|
+
res.send(text);
|
|
2300
|
+
} catch (error) {
|
|
2301
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2302
|
+
const status = message.startsWith("Missing ANTHROPIC_API_KEY") ? 400 : 500;
|
|
2303
|
+
res.status(status).json({ error: message });
|
|
2304
|
+
}
|
|
2305
|
+
});
|
|
2306
|
+
|
|
2307
|
+
app.post("/api/anthropic/embed", async (_req, res) => {
|
|
2308
|
+
res.status(501).json({
|
|
2309
|
+
error: "Anthropic embeddings not implemented",
|
|
2310
|
+
message: "Add support for the Anthropic embeddings endpoint if your use case requires it."
|
|
2311
|
+
});
|
|
2312
|
+
});
|
|
2313
|
+
|
|
2314
|
+
// ============================================================================
|
|
2315
|
+
// AZURE OPENAI ROUTES
|
|
2316
|
+
// ============================================================================
|
|
2317
|
+
|
|
2318
|
+
app.get("/api/azure/health", async (_req, res) => {
|
|
2319
|
+
try {
|
|
2320
|
+
const { endpoint } = requireAzureBaseConfig();
|
|
2321
|
+
const deploymentsUrl = buildAzurePath("deployments");
|
|
2322
|
+
const response = await fetch(deploymentsUrl, {
|
|
2323
|
+
headers: { "api-key": AZURE_OPENAI_API_KEY }
|
|
2324
|
+
});
|
|
2325
|
+
const isHealthy = response.ok;
|
|
2326
|
+
res.json({
|
|
2327
|
+
status: isHealthy ? "healthy" : "unhealthy",
|
|
2328
|
+
azure_status: isHealthy,
|
|
2329
|
+
provider: "azure",
|
|
2330
|
+
endpoint
|
|
2331
|
+
});
|
|
2332
|
+
} catch (error) {
|
|
2333
|
+
res.status(503).json({
|
|
2334
|
+
status: "unhealthy",
|
|
2335
|
+
azure_status: false,
|
|
2336
|
+
provider: "azure",
|
|
2337
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2338
|
+
endpoint: AZURE_OPENAI_ENDPOINT
|
|
2339
|
+
});
|
|
2340
|
+
}
|
|
2341
|
+
});
|
|
2342
|
+
|
|
2343
|
+
app.post("/api/azure/chat/completions", async (req, res) => {
|
|
2344
|
+
try {
|
|
2345
|
+
const { apiKey } = requireAzureBaseConfig();
|
|
2346
|
+
const deployment = resolveAzureDeployment(req.body?.model, AZURE_OPENAI_CHAT_DEPLOYMENT, "chat");
|
|
2347
|
+
const isStreaming = req.body?.stream === true;
|
|
2348
|
+
const { provider, model, ...cleanBody } = req.body ?? {};
|
|
2349
|
+
const requestBody = { ...cleanBody };
|
|
2350
|
+
|
|
2351
|
+
const response = await fetch(buildAzureDeploymentUrl(deployment, "chat/completions"), {
|
|
2352
|
+
method: "POST",
|
|
2353
|
+
headers: {
|
|
2354
|
+
"Content-Type": "application/json",
|
|
2355
|
+
"api-key": apiKey
|
|
2356
|
+
},
|
|
2357
|
+
body: JSON.stringify(requestBody)
|
|
2358
|
+
});
|
|
2359
|
+
|
|
2360
|
+
if (!response.ok) {
|
|
2361
|
+
const errorText = await response.text();
|
|
2362
|
+
return res.status(response.status).json({
|
|
2363
|
+
error: \`Azure OpenAI chat failed: \${response.status}\`,
|
|
2364
|
+
details: errorText
|
|
2365
|
+
});
|
|
2366
|
+
}
|
|
2367
|
+
|
|
2368
|
+
if (isStreaming) {
|
|
2369
|
+
await handleStreamingResponse(response, res);
|
|
2370
|
+
} else {
|
|
2371
|
+
const text = await response.text();
|
|
2372
|
+
res.setHeader('Content-Type', 'application/json');
|
|
2373
|
+
res.send(text);
|
|
2374
|
+
}
|
|
2375
|
+
} catch (error) {
|
|
2376
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2377
|
+
const status = message.startsWith("Missing Azure OpenAI") ? 400 : 500;
|
|
2378
|
+
res.status(status).json({ error: message });
|
|
2379
|
+
}
|
|
2380
|
+
});
|
|
2381
|
+
|
|
2382
|
+
app.post("/api/azure/chat", async (req, res) => {
|
|
2383
|
+
req.url = "/api/azure/chat/completions";
|
|
2384
|
+
return app._router.handle(req, res);
|
|
2385
|
+
});
|
|
2386
|
+
|
|
2387
|
+
app.post("/api/azure/completions", async (req, res) => {
|
|
2388
|
+
try {
|
|
2389
|
+
const { apiKey } = requireAzureBaseConfig();
|
|
2390
|
+
const deployment = resolveAzureDeployment(req.body?.model, AZURE_OPENAI_COMPLETIONS_DEPLOYMENT, "completions");
|
|
2391
|
+
const isStreaming = req.body?.stream === true;
|
|
2392
|
+
const { provider, model, ...cleanBody } = req.body ?? {};
|
|
2393
|
+
const requestBody = { ...cleanBody };
|
|
2394
|
+
|
|
2395
|
+
const response = await fetch(buildAzureDeploymentUrl(deployment, "completions"), {
|
|
2396
|
+
method: "POST",
|
|
2397
|
+
headers: {
|
|
2398
|
+
"Content-Type": "application/json",
|
|
2399
|
+
"api-key": apiKey
|
|
2400
|
+
},
|
|
2401
|
+
body: JSON.stringify(requestBody)
|
|
2402
|
+
});
|
|
2403
|
+
|
|
2404
|
+
if (!response.ok) {
|
|
2405
|
+
const errorText = await response.text();
|
|
2406
|
+
return res.status(response.status).json({
|
|
2407
|
+
error: \`Azure OpenAI completions failed: \${response.status}\`,
|
|
2408
|
+
details: errorText
|
|
2409
|
+
});
|
|
2410
|
+
}
|
|
2411
|
+
|
|
2412
|
+
if (isStreaming) {
|
|
2413
|
+
await handleStreamingResponse(response, res);
|
|
2414
|
+
} else {
|
|
2415
|
+
const text = await response.text();
|
|
2416
|
+
res.setHeader('Content-Type', 'application/json');
|
|
2417
|
+
res.send(text);
|
|
2418
|
+
}
|
|
2419
|
+
} catch (error) {
|
|
2420
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2421
|
+
const status = message.startsWith("Missing Azure OpenAI") ? 400 : 500;
|
|
2422
|
+
res.status(status).json({ error: message });
|
|
2423
|
+
}
|
|
2424
|
+
});
|
|
2425
|
+
|
|
2426
|
+
app.post("/api/azure/generate", async (req, res) => {
|
|
2427
|
+
try {
|
|
2428
|
+
const { apiKey } = requireAzureBaseConfig();
|
|
2429
|
+
const deployment = resolveAzureDeployment(req.body?.model, AZURE_OPENAI_CHAT_DEPLOYMENT, "chat");
|
|
2430
|
+
const prompt = req.body?.prompt || "";
|
|
2431
|
+
const isStreaming = req.body?.stream === true;
|
|
2432
|
+
|
|
2433
|
+
const chatBody = {
|
|
2434
|
+
messages: [
|
|
2435
|
+
{
|
|
2436
|
+
role: "user",
|
|
2437
|
+
content: prompt
|
|
2438
|
+
}
|
|
2439
|
+
],
|
|
2440
|
+
stream: isStreaming,
|
|
2441
|
+
max_tokens: req.body?.max_tokens ?? 150,
|
|
2442
|
+
temperature: req.body?.temperature ?? 0.7
|
|
2443
|
+
};
|
|
2444
|
+
|
|
2445
|
+
const response = await fetch(buildAzureDeploymentUrl(deployment, "chat/completions"), {
|
|
2446
|
+
method: "POST",
|
|
2447
|
+
headers: {
|
|
2448
|
+
"Content-Type": "application/json",
|
|
2449
|
+
"api-key": apiKey
|
|
2450
|
+
},
|
|
2451
|
+
body: JSON.stringify(chatBody)
|
|
2452
|
+
});
|
|
2453
|
+
|
|
2454
|
+
if (!response.ok) {
|
|
2455
|
+
const errorText = await response.text();
|
|
2456
|
+
return res.status(response.status).json({
|
|
2457
|
+
error: \`Azure OpenAI generate failed: \${response.status}\`,
|
|
2458
|
+
details: errorText
|
|
2459
|
+
});
|
|
2460
|
+
}
|
|
2461
|
+
|
|
2462
|
+
if (isStreaming) {
|
|
2463
|
+
await handleStreamingResponse(response, res);
|
|
2464
|
+
} else {
|
|
2465
|
+
const text = await response.text();
|
|
2466
|
+
res.setHeader('Content-Type', 'application/json');
|
|
2467
|
+
res.send(text);
|
|
2468
|
+
}
|
|
2469
|
+
} catch (error) {
|
|
2470
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2471
|
+
const status = message.startsWith("Missing Azure OpenAI") ? 400 : 500;
|
|
2472
|
+
res.status(status).json({ error: message });
|
|
2473
|
+
}
|
|
2474
|
+
});
|
|
2475
|
+
|
|
2476
|
+
app.get("/api/azure/models", async (_req, res) => {
|
|
2477
|
+
try {
|
|
2478
|
+
requireAzureBaseConfig();
|
|
2479
|
+
|
|
2480
|
+
const response = await fetch(buildAzurePath("deployments"), {
|
|
2481
|
+
headers: { "api-key": AZURE_OPENAI_API_KEY }
|
|
2482
|
+
});
|
|
2483
|
+
|
|
2484
|
+
if (!response.ok) {
|
|
2485
|
+
const errorText = await response.text();
|
|
2486
|
+
return res.status(response.status).json({
|
|
2487
|
+
error: \`Azure OpenAI models failed: \${response.status}\`,
|
|
2488
|
+
details: errorText
|
|
2489
|
+
});
|
|
2490
|
+
}
|
|
2491
|
+
|
|
2492
|
+
const text = await response.text();
|
|
2493
|
+
res.setHeader('Content-Type', 'application/json');
|
|
2494
|
+
res.send(text);
|
|
2495
|
+
} catch (error) {
|
|
2496
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2497
|
+
const status = message.startsWith("Missing Azure OpenAI") ? 400 : 500;
|
|
2498
|
+
res.status(status).json({ error: message });
|
|
861
2499
|
}
|
|
862
|
-
|
|
863
|
-
};
|
|
2500
|
+
});
|
|
864
2501
|
|
|
865
|
-
|
|
866
|
-
const handleStreamingResponse = async (upstreamResponse, res) => {
|
|
867
|
-
res.setHeader('Content-Type', 'text/event-stream');
|
|
868
|
-
res.setHeader('Cache-Control', 'no-cache');
|
|
869
|
-
res.setHeader('Connection', 'keep-alive');
|
|
870
|
-
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
871
|
-
|
|
2502
|
+
app.post("/api/azure/embed", async (req, res) => {
|
|
872
2503
|
try {
|
|
873
|
-
|
|
874
|
-
const
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
2504
|
+
const { apiKey } = requireAzureBaseConfig();
|
|
2505
|
+
const deployment = resolveAzureDeployment(req.body?.model, AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT, "embeddings");
|
|
2506
|
+
const { provider, model, ...cleanBody } = req.body ?? {};
|
|
2507
|
+
const requestBody = { ...cleanBody };
|
|
2508
|
+
|
|
2509
|
+
const response = await fetch(buildAzureDeploymentUrl(deployment, "embeddings"), {
|
|
2510
|
+
method: "POST",
|
|
2511
|
+
headers: {
|
|
2512
|
+
"Content-Type": "application/json",
|
|
2513
|
+
"api-key": apiKey
|
|
2514
|
+
},
|
|
2515
|
+
body: JSON.stringify(requestBody)
|
|
2516
|
+
});
|
|
2517
|
+
|
|
2518
|
+
if (!response.ok) {
|
|
2519
|
+
const errorText = await response.text();
|
|
2520
|
+
return res.status(response.status).json({
|
|
2521
|
+
error: \`Azure OpenAI embed failed: \${response.status}\`,
|
|
2522
|
+
details: errorText
|
|
2523
|
+
});
|
|
882
2524
|
}
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
console.error('Streaming error:', error);
|
|
887
|
-
// Fallback to non-streaming
|
|
888
|
-
const text = await upstreamResponse.text();
|
|
2525
|
+
|
|
2526
|
+
const text = await response.text();
|
|
2527
|
+
res.setHeader('Content-Type', 'application/json');
|
|
889
2528
|
res.send(text);
|
|
2529
|
+
} catch (error) {
|
|
2530
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2531
|
+
const status = message.startsWith("Missing Azure OpenAI") ? 400 : 500;
|
|
2532
|
+
res.status(status).json({ error: message });
|
|
890
2533
|
}
|
|
891
|
-
};
|
|
2534
|
+
});
|
|
892
2535
|
|
|
893
2536
|
// ============================================================================
|
|
894
|
-
//
|
|
2537
|
+
// XAI ROUTES
|
|
895
2538
|
// ============================================================================
|
|
896
2539
|
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
// Check OpenAI
|
|
2540
|
+
// xAI Health Check
|
|
2541
|
+
app.get("/api/xai/health", async (_req, res) => {
|
|
901
2542
|
try {
|
|
902
|
-
const
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
2543
|
+
const xaiKey = requireXAIKey();
|
|
2544
|
+
const response = await fetch(XAI_BASE_URL + "/models", {
|
|
2545
|
+
headers: { "Authorization": "Bearer " + xaiKey }
|
|
2546
|
+
});
|
|
2547
|
+
const isHealthy = response.ok;
|
|
2548
|
+
res.json({
|
|
2549
|
+
status: isHealthy ? "healthy" : "unhealthy",
|
|
2550
|
+
xai_status: isHealthy,
|
|
2551
|
+
provider: "xai"
|
|
2552
|
+
});
|
|
2553
|
+
} catch (error) {
|
|
2554
|
+
res.status(503).json({
|
|
2555
|
+
status: "unhealthy",
|
|
2556
|
+
xai_status: false,
|
|
2557
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2558
|
+
provider: "xai"
|
|
2559
|
+
});
|
|
2560
|
+
}
|
|
2561
|
+
});
|
|
2562
|
+
|
|
2563
|
+
// xAI Chat Completions
|
|
2564
|
+
app.post("/api/xai/chat/completions", async (req, res) => {
|
|
2565
|
+
try {
|
|
2566
|
+
const xaiKey = requireXAIKey();
|
|
2567
|
+
const isStreaming = req.body?.stream === true;
|
|
2568
|
+
const { provider, ...cleanBody } = req.body ?? {};
|
|
2569
|
+
const requestBody = {
|
|
2570
|
+
...cleanBody,
|
|
2571
|
+
model: req.body?.model?.replace(/^xai:/, "") || "grok-2-latest"
|
|
2572
|
+
};
|
|
2573
|
+
|
|
2574
|
+
const response = await fetch(XAI_BASE_URL + "/chat/completions", {
|
|
2575
|
+
method: "POST",
|
|
2576
|
+
headers: {
|
|
2577
|
+
"Content-Type": "application/json",
|
|
2578
|
+
"Authorization": "Bearer " + xaiKey
|
|
2579
|
+
},
|
|
2580
|
+
body: JSON.stringify(requestBody)
|
|
2581
|
+
});
|
|
2582
|
+
|
|
2583
|
+
if (!response.ok) {
|
|
2584
|
+
const errorText = await response.text();
|
|
2585
|
+
return res.status(response.status).json({
|
|
2586
|
+
error: "xAI chat failed: " + response.status,
|
|
2587
|
+
details: errorText
|
|
911
2588
|
});
|
|
2589
|
+
}
|
|
2590
|
+
|
|
2591
|
+
if (isStreaming) {
|
|
2592
|
+
await handleStreamingResponse(response, res);
|
|
912
2593
|
} else {
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
provider: "openai",
|
|
917
|
-
error: "API key not configured"
|
|
918
|
-
});
|
|
2594
|
+
const text = await response.text();
|
|
2595
|
+
res.setHeader('Content-Type', 'application/json');
|
|
2596
|
+
res.send(text);
|
|
919
2597
|
}
|
|
920
2598
|
} catch (error) {
|
|
921
|
-
|
|
922
|
-
name: "openai",
|
|
923
|
-
status: "unhealthy",
|
|
924
|
-
provider: "openai",
|
|
925
|
-
error: error.message
|
|
926
|
-
});
|
|
2599
|
+
res.status(500).json({ error: error instanceof Error ? error.message : String(error) });
|
|
927
2600
|
}
|
|
2601
|
+
});
|
|
928
2602
|
|
|
929
|
-
|
|
2603
|
+
app.post("/api/xai/chat", async (req, res) => {
|
|
2604
|
+
req.url = "/api/xai/chat/completions";
|
|
2605
|
+
return app._router.handle(req, res);
|
|
2606
|
+
});
|
|
2607
|
+
|
|
2608
|
+
// xAI Completions
|
|
2609
|
+
app.post("/api/xai/completions", async (req, res) => {
|
|
930
2610
|
try {
|
|
931
|
-
|
|
932
|
-
const
|
|
933
|
-
const
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
2611
|
+
const xaiKey = requireXAIKey();
|
|
2612
|
+
const isStreaming = req.body?.stream === true;
|
|
2613
|
+
const { provider, ...cleanBody } = req.body ?? {};
|
|
2614
|
+
const requestBody = {
|
|
2615
|
+
...cleanBody,
|
|
2616
|
+
model: req.body?.model?.replace(/^xai:/, "") || "grok-2-mini"
|
|
2617
|
+
};
|
|
2618
|
+
|
|
2619
|
+
const response = await fetch(XAI_BASE_URL + "/completions", {
|
|
2620
|
+
method: "POST",
|
|
2621
|
+
headers: {
|
|
2622
|
+
"Content-Type": "application/json",
|
|
2623
|
+
"Authorization": "Bearer " + xaiKey
|
|
2624
|
+
},
|
|
2625
|
+
body: JSON.stringify(requestBody)
|
|
940
2626
|
});
|
|
2627
|
+
|
|
2628
|
+
if (!response.ok) {
|
|
2629
|
+
const errorText = await response.text();
|
|
2630
|
+
return res.status(response.status).json({
|
|
2631
|
+
error: "xAI completions failed: " + response.status,
|
|
2632
|
+
details: errorText
|
|
2633
|
+
});
|
|
2634
|
+
}
|
|
2635
|
+
|
|
2636
|
+
if (isStreaming) {
|
|
2637
|
+
await handleStreamingResponse(response, res);
|
|
2638
|
+
} else {
|
|
2639
|
+
const text = await response.text();
|
|
2640
|
+
res.setHeader('Content-Type', 'application/json');
|
|
2641
|
+
res.send(text);
|
|
2642
|
+
}
|
|
941
2643
|
} catch (error) {
|
|
942
|
-
|
|
943
|
-
providers.push({
|
|
944
|
-
name: "ollama",
|
|
945
|
-
status: "offline",
|
|
946
|
-
provider: "ollama",
|
|
947
|
-
error: error.message,
|
|
948
|
-
url: OLLAMA_BASE_URL
|
|
949
|
-
});
|
|
2644
|
+
res.status(500).json({ error: error instanceof Error ? error.message : String(error) });
|
|
950
2645
|
}
|
|
2646
|
+
});
|
|
951
2647
|
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
2648
|
+
// xAI Generate
|
|
2649
|
+
app.post("/api/xai/generate", async (req, res) => {
|
|
2650
|
+
try {
|
|
2651
|
+
const xaiKey = requireXAIKey();
|
|
2652
|
+
const prompt = req.body?.prompt || "";
|
|
2653
|
+
const model = req.body?.model?.replace(/^xai:/, "") || "grok-2-latest";
|
|
2654
|
+
const isStreaming = req.body?.stream === true;
|
|
2655
|
+
|
|
2656
|
+
const chatBody = {
|
|
2657
|
+
model,
|
|
2658
|
+
messages: [
|
|
2659
|
+
{ role: "user", content: prompt }
|
|
2660
|
+
],
|
|
2661
|
+
stream: isStreaming,
|
|
2662
|
+
max_tokens: req.body?.max_tokens || 150,
|
|
2663
|
+
temperature: req.body?.temperature ?? 0.7
|
|
2664
|
+
};
|
|
2665
|
+
|
|
2666
|
+
const response = await fetch(XAI_BASE_URL + "/chat/completions", {
|
|
2667
|
+
method: "POST",
|
|
2668
|
+
headers: {
|
|
2669
|
+
"Content-Type": "application/json",
|
|
2670
|
+
"Authorization": "Bearer " + xaiKey
|
|
2671
|
+
},
|
|
2672
|
+
body: JSON.stringify(chatBody)
|
|
2673
|
+
});
|
|
2674
|
+
|
|
2675
|
+
if (!response.ok) {
|
|
2676
|
+
const errorText = await response.text();
|
|
2677
|
+
return res.status(response.status).json({
|
|
2678
|
+
error: "xAI generate failed: " + response.status,
|
|
2679
|
+
details: errorText
|
|
2680
|
+
});
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
if (isStreaming) {
|
|
2684
|
+
await handleStreamingResponse(response, res);
|
|
2685
|
+
} else {
|
|
2686
|
+
const data = await response.json();
|
|
2687
|
+
const generateResponse = {
|
|
2688
|
+
model,
|
|
2689
|
+
created_at: new Date().toISOString(),
|
|
2690
|
+
response: data.choices?.[0]?.message?.content || "",
|
|
2691
|
+
done: true,
|
|
2692
|
+
context: [],
|
|
2693
|
+
total_duration: 0,
|
|
2694
|
+
load_duration: 0,
|
|
2695
|
+
prompt_eval_count: data.usage?.prompt_tokens || 0,
|
|
2696
|
+
prompt_eval_duration: 0,
|
|
2697
|
+
eval_count: data.usage?.completion_tokens || 0,
|
|
2698
|
+
eval_duration: 0
|
|
2699
|
+
};
|
|
2700
|
+
res.json(generateResponse);
|
|
2701
|
+
}
|
|
2702
|
+
} catch (error) {
|
|
2703
|
+
res.status(500).json({ error: error instanceof Error ? error.message : String(error) });
|
|
2704
|
+
}
|
|
960
2705
|
});
|
|
961
2706
|
|
|
962
|
-
|
|
963
|
-
|
|
2707
|
+
// xAI Models
|
|
2708
|
+
app.get("/api/xai/models", async (_req, res) => {
|
|
2709
|
+
try {
|
|
2710
|
+
const xaiKey = requireXAIKey();
|
|
2711
|
+
const response = await fetch(XAI_BASE_URL + "/models", {
|
|
2712
|
+
headers: { "Authorization": "Bearer " + xaiKey }
|
|
2713
|
+
});
|
|
2714
|
+
|
|
2715
|
+
if (!response.ok) {
|
|
2716
|
+
const errorText = await response.text();
|
|
2717
|
+
return res.status(response.status).json({
|
|
2718
|
+
error: "xAI models failed: " + response.status,
|
|
2719
|
+
details: errorText
|
|
2720
|
+
});
|
|
2721
|
+
}
|
|
2722
|
+
|
|
2723
|
+
const text = await response.text();
|
|
2724
|
+
res.setHeader('Content-Type', 'application/json');
|
|
2725
|
+
res.send(text);
|
|
2726
|
+
} catch (error) {
|
|
2727
|
+
res.status(500).json({ error: error instanceof Error ? error.message : String(error) });
|
|
2728
|
+
}
|
|
964
2729
|
});
|
|
965
2730
|
|
|
966
2731
|
// ============================================================================
|
|
@@ -1477,24 +3242,20 @@ app.post("/api/mcp/generate-image", async (req, res) => {
|
|
|
1477
3242
|
|
|
1478
3243
|
app.all("/api/anthropic/*", (_req, res) => {
|
|
1479
3244
|
res.status(501).json({
|
|
1480
|
-
error: "Anthropic
|
|
1481
|
-
message: "
|
|
1482
|
-
});
|
|
1483
|
-
});
|
|
1484
|
-
|
|
1485
|
-
app.all("/api/azure/*", (_req, res) => {
|
|
1486
|
-
res.status(501).json({
|
|
1487
|
-
error: "Azure OpenAI integration not implemented",
|
|
1488
|
-
message: "This quickstart gateway only supports OpenAI and Ollama providers"
|
|
3245
|
+
error: "Anthropic route not implemented",
|
|
3246
|
+
message: "Extend the quickstart gateway if you need additional Anthropic endpoints beyond the defaults."
|
|
1489
3247
|
});
|
|
1490
3248
|
});
|
|
1491
3249
|
|
|
1492
3250
|
const port = Number(process.env.PORT ?? ${ctx.gatewayPort});
|
|
1493
3251
|
app.listen(port, () => {
|
|
1494
3252
|
console.log("\u26A1 Bandit quickstart gateway ready on http://localhost:" + port);
|
|
1495
|
-
console.log("\u{1F4E1} Supported providers: OpenAI, Ollama");
|
|
3253
|
+
console.log("\u{1F4E1} Supported providers: OpenAI, Azure OpenAI, Anthropic, XAI, Ollama");
|
|
1496
3254
|
console.log("\u{1F517} Provider-specific routes:");
|
|
1497
3255
|
console.log(" \u2022 /api/openai/* - OpenAI endpoints");
|
|
3256
|
+
console.log(" \u2022 /api/azure/* - Azure OpenAI endpoints");
|
|
3257
|
+
console.log(" \u2022 /api/anthropic/* - Anthropic endpoints");
|
|
3258
|
+
console.log(" \u2022 /api/xai/* - XAI endpoints");
|
|
1498
3259
|
console.log(" \u2022 /api/ollama/* - Ollama endpoints");
|
|
1499
3260
|
console.log(" \u2022 /api/health - Overall health check");
|
|
1500
3261
|
});
|
|
@@ -1513,20 +3274,16 @@ dist
|
|
|
1513
3274
|
`
|
|
1514
3275
|
)
|
|
1515
3276
|
);
|
|
1516
|
-
var buildNpmrc = () => ensureTrailingNewline(
|
|
1517
|
-
normalizeLineEndings(`registry=https://registry.npmjs.org/
|
|
1518
|
-
`)
|
|
1519
|
-
);
|
|
1520
3277
|
var buildReadme = (ctx) => ensureTrailingNewline(
|
|
1521
3278
|
normalizeLineEndings(
|
|
1522
3279
|
`# ${ctx.projectTitle} \u2014 Bandit Quickstart
|
|
1523
3280
|
|
|
1524
|
-
This project was generated by the Bandit Engine CLI. It ships with a React + Vite frontend that consumes \`@burtson-labs/bandit-engine
|
|
3281
|
+
This project was generated by the Bandit Engine CLI. It ships with a React + Vite frontend that consumes \`@burtson-labs/bandit-engine\`, a lightweight Express gateway you can adapt for production, and a Next.js App Router API scaffold in \`server/next-app/\`.
|
|
1525
3282
|
|
|
1526
3283
|
## \u{1F680} Next steps
|
|
1527
3284
|
- \`npm install\`
|
|
1528
3285
|
- \`cp .env.example .env\`
|
|
1529
|
-
- Fill in
|
|
3286
|
+
- Fill in your OpenAI, Azure OpenAI, Anthropic, or xAI credentials (or point \`OLLAMA_URL\` at your local server)
|
|
1530
3287
|
- \`npm run dev\`
|
|
1531
3288
|
|
|
1532
3289
|
The command runs the gateway and the frontend together. Visit http://localhost:${ctx.frontendPort} to see the chat and modal in action.
|
|
@@ -1539,7 +3296,8 @@ The command runs the gateway and the frontend together. Visit http://localhost:$
|
|
|
1539
3296
|
## \u{1F4E6} What\u2019s inside
|
|
1540
3297
|
- React + Vite 5 with Material UI theming
|
|
1541
3298
|
- Bandit chat surface + modal wired via \`ChatProvider\`
|
|
1542
|
-
- Express gateway proxying OpenAI or Ollama to keep API keys server-side
|
|
3299
|
+
- Express gateway proxying OpenAI, Azure OpenAI, Anthropic, XAI, or Ollama to keep API keys server-side
|
|
3300
|
+
- Next.js App Router gateway scaffold in 'server/next-app/' for projects that prefer Next
|
|
1543
3301
|
- Friendly defaults you can evolve into your production stack
|
|
1544
3302
|
|
|
1545
3303
|
Need more? Run \`npx @burtson-labs/bandit-engine create --help\` to explore additional options.
|
|
@@ -1554,8 +3312,9 @@ var createQuickstartProject = async (options) => {
|
|
|
1554
3312
|
const packageName = normalizePackageName(rawProjectName);
|
|
1555
3313
|
const projectTitle = toTitleCase(rawProjectName) || "Bandit Quickstart";
|
|
1556
3314
|
await ensureWritableDirectory(resolvedDir, Boolean(options.force));
|
|
1557
|
-
const
|
|
1558
|
-
const
|
|
3315
|
+
const skipPrompts = Boolean(options.skipPrompts);
|
|
3316
|
+
const provider = options.provider ? normalizeProvider(options.provider) : skipPrompts ? "ollama" : await promptForProvider();
|
|
3317
|
+
const promptAnswers = skipPrompts ? {} : await promptForMissingData({
|
|
1559
3318
|
brandingText: options.brandingText,
|
|
1560
3319
|
provider
|
|
1561
3320
|
});
|
|
@@ -1620,17 +3379,84 @@ var ensureWritableDirectory = async (dir, force) => {
|
|
|
1620
3379
|
};
|
|
1621
3380
|
var normalizeProvider = (value) => {
|
|
1622
3381
|
const normalized = (value ?? "openai").toLowerCase();
|
|
1623
|
-
|
|
3382
|
+
if (normalized === "ollama") {
|
|
3383
|
+
return "ollama";
|
|
3384
|
+
}
|
|
3385
|
+
if (normalized === "azure" || normalized === "azure-openai" || normalized === "azureopenai") {
|
|
3386
|
+
return "azure";
|
|
3387
|
+
}
|
|
3388
|
+
if (normalized === "anthropic") {
|
|
3389
|
+
return "anthropic";
|
|
3390
|
+
}
|
|
3391
|
+
if (normalized === "xai" || normalized === "grok") {
|
|
3392
|
+
return "xai";
|
|
3393
|
+
}
|
|
3394
|
+
return "openai";
|
|
1624
3395
|
};
|
|
1625
3396
|
var inferDefaultModelId = (provider) => {
|
|
1626
|
-
|
|
3397
|
+
if (provider === "ollama") {
|
|
3398
|
+
return "ollama:llama3.1";
|
|
3399
|
+
}
|
|
3400
|
+
if (provider === "azure") {
|
|
3401
|
+
return "azure:gpt-4o";
|
|
3402
|
+
}
|
|
3403
|
+
if (provider === "anthropic") {
|
|
3404
|
+
return "anthropic:claude-3-5-haiku-latest";
|
|
3405
|
+
}
|
|
3406
|
+
if (provider === "xai") {
|
|
3407
|
+
return "xai:grok-2-latest";
|
|
3408
|
+
}
|
|
3409
|
+
return "openai:gpt-4o-mini";
|
|
1627
3410
|
};
|
|
1628
3411
|
var inferFallbackModelId = (provider, defaultId) => {
|
|
1629
3412
|
if (provider === "ollama") {
|
|
1630
3413
|
return defaultId === "ollama:llama3" ? "ollama:llama2" : "ollama:llama3";
|
|
1631
3414
|
}
|
|
3415
|
+
if (provider === "azure") {
|
|
3416
|
+
return defaultId === "azure:gpt-4o-mini" ? "azure:gpt-4o" : "azure:gpt-4o-mini";
|
|
3417
|
+
}
|
|
3418
|
+
if (provider === "anthropic") {
|
|
3419
|
+
return defaultId === "anthropic:claude-3-5-haiku-latest" ? "anthropic:claude-3-haiku-20240307" : "anthropic:claude-3-5-haiku-latest";
|
|
3420
|
+
}
|
|
3421
|
+
if (provider === "xai") {
|
|
3422
|
+
return defaultId === "xai:grok-2-mini" ? "xai:grok-2-latest" : "xai:grok-2-mini";
|
|
3423
|
+
}
|
|
1632
3424
|
return defaultId === "openai:gpt-4.1-mini" ? "openai:gpt-4o-mini" : "openai:gpt-4.1-mini";
|
|
1633
3425
|
};
|
|
3426
|
+
var promptForProvider = async () => {
|
|
3427
|
+
const providerOptions = [
|
|
3428
|
+
{ label: "Ollama (self-hosted) \u2014 default", value: "ollama" },
|
|
3429
|
+
{ label: "OpenAI", value: "openai" },
|
|
3430
|
+
{ label: "Azure OpenAI", value: "azure" },
|
|
3431
|
+
{ label: "Anthropic", value: "anthropic" },
|
|
3432
|
+
{ label: "xAI (Grok)", value: "xai" }
|
|
3433
|
+
];
|
|
3434
|
+
const messageLines = [
|
|
3435
|
+
"Which provider should we configure for the gateway?",
|
|
3436
|
+
...providerOptions.map((option, index) => ` ${index + 1}. ${option.label}`),
|
|
3437
|
+
"Enter a number:"
|
|
3438
|
+
];
|
|
3439
|
+
const onCancel = () => {
|
|
3440
|
+
throw new Error("Command cancelled.");
|
|
3441
|
+
};
|
|
3442
|
+
const answers = await (0, import_prompts.default)(
|
|
3443
|
+
{
|
|
3444
|
+
type: "number",
|
|
3445
|
+
name: "providerIndex",
|
|
3446
|
+
message: messageLines.join("\n"),
|
|
3447
|
+
initial: 1,
|
|
3448
|
+
validate: (input) => {
|
|
3449
|
+
if (!Number.isInteger(input)) {
|
|
3450
|
+
return "Enter a whole number.";
|
|
3451
|
+
}
|
|
3452
|
+
return input >= 1 && input <= providerOptions.length ? true : `Enter a number between 1 and ${providerOptions.length}.`;
|
|
3453
|
+
}
|
|
3454
|
+
},
|
|
3455
|
+
{ onCancel }
|
|
3456
|
+
);
|
|
3457
|
+
const selectedIndex = typeof answers.providerIndex === "number" && answers.providerIndex >= 1 ? answers.providerIndex - 1 : 0;
|
|
3458
|
+
return providerOptions[selectedIndex]?.value ?? "ollama";
|
|
3459
|
+
};
|
|
1634
3460
|
var sanitizePort = (value) => {
|
|
1635
3461
|
const port = Number(value);
|
|
1636
3462
|
if (Number.isNaN(port) || port <= 0 || port >= 65535) {
|
|
@@ -1719,9 +3545,12 @@ var writeProject = async (inputs) => {
|
|
|
1719
3545
|
"src/theme.ts": buildThemeTs(),
|
|
1720
3546
|
"public/config.json": buildBrandingConfig(context),
|
|
1721
3547
|
"server/gateway.js": buildGatewayServer(context),
|
|
3548
|
+
"server/next-app/app/api/chat/completions/route.ts": buildNextChatRoute(context),
|
|
3549
|
+
"server/next-app/app/api/health/route.ts": buildNextHealthRoute(),
|
|
3550
|
+
"server/next-app/app/api/models/route.ts": buildNextModelsRoute(context),
|
|
3551
|
+
"server/next-app/README.md": buildNextGatewayReadme(),
|
|
1722
3552
|
".env.example": buildEnvExample(context),
|
|
1723
3553
|
".gitignore": buildGitignore(),
|
|
1724
|
-
".npmrc": buildNpmrc(),
|
|
1725
3554
|
"README.md": buildReadme(context)
|
|
1726
3555
|
};
|
|
1727
3556
|
if (!inputs.logo.isDefault && inputs.logo.fileName) {
|
|
@@ -1766,6 +3595,9 @@ var logIntro = () => {
|
|
|
1766
3595
|
var program = new import_commander.Command();
|
|
1767
3596
|
program.name("bandit").description("Bandit Engine developer utilities").version(package_default.version).showHelpAfterError();
|
|
1768
3597
|
program.command("create").description("Scaffold a Bandit quickstart project with a frontend and gateway").argument("[directory]", "Relative path for your new project", "bandit-quickstart").option("-f, --force", "Overwrite the target directory if it already contains files", false).option("--branding-text <text>", "Assistant display name shown in the UI").option(
|
|
3598
|
+
"--provider <provider>",
|
|
3599
|
+
"Default gateway provider (openai, azure, anthropic, ollama)"
|
|
3600
|
+
).option(
|
|
1769
3601
|
"--frontend-port <port>",
|
|
1770
3602
|
"Frontend dev server port (default: 5183)",
|
|
1771
3603
|
(value) => parseInt(value, 10)
|
|
@@ -1780,6 +3612,7 @@ program.command("create").description("Scaffold a Bandit quickstart project with
|
|
|
1780
3612
|
projectName,
|
|
1781
3613
|
force: Boolean(cmdOptions.force),
|
|
1782
3614
|
brandingText: cmdOptions.brandingText,
|
|
3615
|
+
provider: typeof cmdOptions.provider === "string" ? cmdOptions.provider : void 0,
|
|
1783
3616
|
frontendPort: Number.isFinite(cmdOptions.frontendPort) ? cmdOptions.frontendPort : void 0,
|
|
1784
3617
|
gatewayPort: Number.isFinite(cmdOptions.gatewayPort) ? cmdOptions.gatewayPort : void 0,
|
|
1785
3618
|
skipPrompts
|
|
@@ -1797,6 +3630,11 @@ program.command("create").description("Scaffold a Bandit quickstart project with
|
|
|
1797
3630
|
console.log(" cp .env.example .env");
|
|
1798
3631
|
console.log(" npm run dev");
|
|
1799
3632
|
console.log("");
|
|
3633
|
+
console.log("\u{1F50D} Before you dive in:");
|
|
3634
|
+
console.log(" \u2022 Open .env to confirm the provider credentials and URLs match your setup.");
|
|
3635
|
+
console.log(" \u2022 server/gateway.js is a scaffold Express proxy that keeps API keys server-side\u2014extend it with auth, logging, and your production logic.");
|
|
3636
|
+
console.log(" \u2022 If you prefer Next.js App Router, check server/next-app/ for a starter route handler.");
|
|
3637
|
+
console.log("");
|
|
1800
3638
|
} catch (error) {
|
|
1801
3639
|
const message = error instanceof Error ? error.message : "Failed to create Bandit quickstart project.";
|
|
1802
3640
|
console.error(`
|