@elsium-ai/gateway 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/batch.d.ts.map +1 -1
- package/dist/gateway.d.ts +2 -1
- package/dist/gateway.d.ts.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +297 -210
- package/dist/middleware.d.ts +2 -1
- package/dist/middleware.d.ts.map +1 -1
- package/dist/pricing.d.ts +1 -0
- package/dist/pricing.d.ts.map +1 -1
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/google.d.ts.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/security.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/batch.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../src/batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAErE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAIxC,MAAM,WAAW,WAAW;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;IACvD,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,WAAW;IAC3B,OAAO,EAAE,eAAe,EAAE,CAAA;IAC1B,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;CACvB;
|
|
1
|
+
{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../src/batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAErE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAIxC,MAAM,WAAW,WAAW;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;IACvD,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,WAAW;IAC3B,OAAO,EAAE,eAAe,EAAE,CAAA;IAC1B,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;CACvB;AAsCD,wBAAgB,WAAW,CAC1B,OAAO,EAAE,OAAO,EAChB,MAAM,CAAC,EAAE,WAAW,GAClB;IAAE,OAAO,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;CAAE,CA+ElE"}
|
package/dist/gateway.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CompletionRequest, LLMResponse, Middleware, ProviderConfig, XRayData } from '@elsium-ai/core';
|
|
1
|
+
import type { CompletionRequest, LLMResponse, Middleware, ProviderConfig, StreamMiddleware, XRayData } from '@elsium-ai/core';
|
|
2
2
|
import { type ElsiumStream } from '@elsium-ai/core';
|
|
3
3
|
import type { z } from 'zod';
|
|
4
4
|
import type { LLMProvider } from './provider';
|
|
@@ -10,6 +10,7 @@ export interface GatewayConfig {
|
|
|
10
10
|
timeout?: number;
|
|
11
11
|
maxRetries?: number;
|
|
12
12
|
middleware?: Middleware[];
|
|
13
|
+
streamMiddleware?: StreamMiddleware[];
|
|
13
14
|
xray?: boolean | {
|
|
14
15
|
maxHistory?: number;
|
|
15
16
|
};
|
package/dist/gateway.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateway.d.ts","sourceRoot":"","sources":["../src/gateway.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,iBAAiB,EACjB,WAAW,EACX,UAAU,EAEV,cAAc,EAEd,QAAQ,EACR,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAEN,KAAK,YAAY,EAIjB,MAAM,iBAAiB,CAAA;AACxB,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAI5B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAM7C,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,UAAU,EAAE,CAAA;IACzB,IAAI,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IACxC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;IAC1D,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,YAAY,CAAA;IAChD,QAAQ,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,GAAG;QAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC;QAC3E,IAAI,EAAE,CAAC,CAAA;QACP,QAAQ,EAAE,WAAW,CAAA;KACrB,CAAC,CAAA;IACF,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAA;IAC9B,QAAQ,IAAI,QAAQ,GAAG,IAAI,CAAA;IAC3B,WAAW,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,EAAE,CAAA;CACvC;
|
|
1
|
+
{"version":3,"file":"gateway.d.ts","sourceRoot":"","sources":["../src/gateway.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,iBAAiB,EACjB,WAAW,EACX,UAAU,EAEV,cAAc,EAEd,gBAAgB,EAChB,QAAQ,EACR,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAEN,KAAK,YAAY,EAIjB,MAAM,iBAAiB,CAAA;AACxB,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAI5B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAM7C,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,UAAU,EAAE,CAAA;IACzB,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,CAAA;IACrC,IAAI,CAAC,EAAE,OAAO,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IACxC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;IAC1D,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,YAAY,CAAA;IAChD,QAAQ,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,GAAG;QAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC;QAC3E,IAAI,EAAE,CAAC,CAAA;QACP,QAAQ,EAAE,WAAW,CAAA;KACrB,CAAC,CAAA;IACF,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAA;IAC9B,QAAQ,IAAI,QAAQ,GAAG,IAAI,CAAA;IAC3B,WAAW,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,EAAE,CAAA;CACvC;AA0BD,wBAAgB,uBAAuB,CACtC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,CAAC,MAAM,EAAE,cAAc,KAAK,WAAW,GAC9C,IAAI,CAEN;AAwJD,wBAAgB,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAkItD"}
|
package/dist/index.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ export { registerProvider, getProviderFactory, listProviders, registerProviderMe
|
|
|
5
5
|
export { createAnthropicProvider } from './providers/anthropic';
|
|
6
6
|
export { createOpenAIProvider } from './providers/openai';
|
|
7
7
|
export { createGoogleProvider } from './providers/google';
|
|
8
|
-
export { composeMiddleware, loggingMiddleware, costTrackingMiddleware, xrayMiddleware, } from './middleware';
|
|
8
|
+
export { composeMiddleware, composeStreamMiddleware, loggingMiddleware, costTrackingMiddleware, xrayMiddleware, } from './middleware';
|
|
9
9
|
export type { XRayStore } from './middleware';
|
|
10
10
|
export { securityMiddleware, detectPromptInjection, detectJailbreak, redactSecrets, checkBlockedPatterns, classifyContent, } from './security';
|
|
11
11
|
export type { SecurityMiddlewareConfig, SecurityViolation, SecurityResult, DataClassification, ClassificationResult, } from './security';
|
|
@@ -15,7 +15,7 @@ export { cacheMiddleware, createInMemoryCache } from './cache';
|
|
|
15
15
|
export type { CacheAdapter, CacheStats, CacheMiddlewareConfig } from './cache';
|
|
16
16
|
export { outputGuardrailMiddleware } from './output-guardrails';
|
|
17
17
|
export type { OutputGuardrailConfig, OutputGuardrailRule, OutputViolation, } from './output-guardrails';
|
|
18
|
-
export { calculateCost, registerPricing } from './pricing';
|
|
18
|
+
export { calculateCost, registerPricing, estimateCost } from './pricing';
|
|
19
19
|
export { createBatch } from './batch';
|
|
20
20
|
export type { BatchConfig, BatchResult, BatchResultItem } from './batch';
|
|
21
21
|
export { createProviderMesh } from './router';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAA;AAC5D,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAGvD,YAAY,EACX,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,SAAS,GACT,MAAM,YAAY,CAAA;AACnB,OAAO,EACN,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,EACb,wBAAwB,EACxB,mBAAmB,GACnB,MAAM,YAAY,CAAA;AAGnB,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAGzD,OAAO,EACN,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,cAAc,GACd,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAG7C,OAAO,EACN,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,aAAa,EACb,oBAAoB,EACpB,eAAe,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EACX,wBAAwB,EACxB,iBAAiB,EACjB,cAAc,EACd,kBAAkB,EAClB,oBAAoB,GACpB,MAAM,YAAY,CAAA;AAGnB,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAC/D,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAG1D,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAA;AAC9D,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAG9E,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAA;AAC/D,YAAY,EACX,qBAAqB,EACrB,mBAAmB,EACnB,eAAe,GACf,MAAM,qBAAqB,CAAA;AAG5B,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAA;AAC5D,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAGvD,YAAY,EACX,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,SAAS,GACT,MAAM,YAAY,CAAA;AACnB,OAAO,EACN,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,EACb,wBAAwB,EACxB,mBAAmB,GACnB,MAAM,YAAY,CAAA;AAGnB,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAGzD,OAAO,EACN,iBAAiB,EACjB,uBAAuB,EACvB,iBAAiB,EACjB,sBAAsB,EACtB,cAAc,GACd,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAG7C,OAAO,EACN,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,aAAa,EACb,oBAAoB,EACpB,eAAe,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EACX,wBAAwB,EACxB,iBAAiB,EACjB,cAAc,EACd,kBAAkB,EAClB,oBAAoB,GACpB,MAAM,YAAY,CAAA;AAGnB,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAC/D,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAG1D,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAA;AAC9D,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAG9E,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAA;AAC/D,YAAY,EACX,qBAAqB,EACrB,mBAAmB,EACnB,eAAe,GACf,MAAM,qBAAqB,CAAA;AAG5B,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAGxE,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAGxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AAC7C,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA"}
|
package/dist/index.js
CHANGED
|
@@ -647,6 +647,17 @@ function composeMiddleware(middlewares) {
|
|
|
647
647
|
return dispatch(0);
|
|
648
648
|
};
|
|
649
649
|
}
|
|
650
|
+
function composeStreamMiddleware(middlewares) {
|
|
651
|
+
return (ctx, source, finalNext) => {
|
|
652
|
+
function dispatch(i, currentCtx, currentSource) {
|
|
653
|
+
if (i >= middlewares.length) {
|
|
654
|
+
return finalNext(currentCtx, currentSource);
|
|
655
|
+
}
|
|
656
|
+
return middlewares[i](currentCtx, currentSource, (c, s) => dispatch(i + 1, c, s));
|
|
657
|
+
}
|
|
658
|
+
return dispatch(0, ctx, source);
|
|
659
|
+
};
|
|
660
|
+
}
|
|
650
661
|
function loggingMiddleware(logger) {
|
|
651
662
|
const log3 = logger ?? createLogger({ level: "info" });
|
|
652
663
|
return async (ctx, next) => {
|
|
@@ -851,6 +862,12 @@ function calculateCost(model, usage) {
|
|
|
851
862
|
function registerPricing(model, pricing) {
|
|
852
863
|
PRICING[model] = pricing;
|
|
853
864
|
}
|
|
865
|
+
function estimateCost(model, tokenCount) {
|
|
866
|
+
const pricing = PRICING[resolveModelName(model)];
|
|
867
|
+
if (!pricing)
|
|
868
|
+
return 0;
|
|
869
|
+
return tokenCount / 1e6 * pricing.inputPerMillion;
|
|
870
|
+
}
|
|
854
871
|
|
|
855
872
|
// src/providers/anthropic.ts
|
|
856
873
|
var DEFAULT_BASE_URL = "https://api.anthropic.com";
|
|
@@ -994,6 +1011,52 @@ function createAnthropicProvider(config) {
|
|
|
994
1011
|
input_schema: t.inputSchema
|
|
995
1012
|
}));
|
|
996
1013
|
}
|
|
1014
|
+
function buildOptionalParams(req) {
|
|
1015
|
+
const params = {};
|
|
1016
|
+
if (req.temperature !== undefined)
|
|
1017
|
+
params.temperature = req.temperature;
|
|
1018
|
+
if (req.topP !== undefined)
|
|
1019
|
+
params.top_p = req.topP;
|
|
1020
|
+
if (req.stopSequences?.length)
|
|
1021
|
+
params.stop_sequences = req.stopSequences;
|
|
1022
|
+
return params;
|
|
1023
|
+
}
|
|
1024
|
+
function applyStructuredOutput(body, req, tools) {
|
|
1025
|
+
if (!req.schema)
|
|
1026
|
+
return;
|
|
1027
|
+
const jsonSchema = zodToJsonSchema(req.schema);
|
|
1028
|
+
const structuredTool = {
|
|
1029
|
+
name: "_structured_output",
|
|
1030
|
+
description: "Return structured output matching the required schema",
|
|
1031
|
+
input_schema: jsonSchema
|
|
1032
|
+
};
|
|
1033
|
+
body.tools = [...tools ?? [], structuredTool];
|
|
1034
|
+
body.tool_choice = { type: "tool", name: "_structured_output" };
|
|
1035
|
+
}
|
|
1036
|
+
function buildRequestBody(req) {
|
|
1037
|
+
const { system, messages } = formatMessages(req.messages);
|
|
1038
|
+
const model = req.model ?? "claude-sonnet-4-6";
|
|
1039
|
+
const body = {
|
|
1040
|
+
model,
|
|
1041
|
+
messages,
|
|
1042
|
+
max_tokens: req.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
1043
|
+
...system || req.system ? { system: req.system ?? system } : {},
|
|
1044
|
+
...buildOptionalParams(req),
|
|
1045
|
+
...buildSeedMetadata(req)
|
|
1046
|
+
};
|
|
1047
|
+
const tools = formatTools(req.tools);
|
|
1048
|
+
if (tools)
|
|
1049
|
+
body.tools = tools;
|
|
1050
|
+
applyStructuredOutput(body, req, tools);
|
|
1051
|
+
return body;
|
|
1052
|
+
}
|
|
1053
|
+
function executeWithTimeout(fn, reqSignal) {
|
|
1054
|
+
const controller = new AbortController;
|
|
1055
|
+
const timer = setTimeout(() => controller.abort(), timeout);
|
|
1056
|
+
const signals = [controller.signal, reqSignal].filter(Boolean);
|
|
1057
|
+
const mergedSignal = signals.length > 1 ? AbortSignal.any(signals) : signals[0];
|
|
1058
|
+
return fn(mergedSignal).finally(() => clearTimeout(timer));
|
|
1059
|
+
}
|
|
997
1060
|
function extractContentBlocks(content) {
|
|
998
1061
|
const toolCalls = [];
|
|
999
1062
|
const textParts = [];
|
|
@@ -1045,44 +1108,12 @@ function createAnthropicProvider(config) {
|
|
|
1045
1108
|
authStyle: "x-api-key"
|
|
1046
1109
|
},
|
|
1047
1110
|
async complete(req) {
|
|
1048
|
-
const
|
|
1049
|
-
const model = req.model ?? "claude-sonnet-4-6";
|
|
1050
|
-
const body = {
|
|
1051
|
-
model,
|
|
1052
|
-
messages,
|
|
1053
|
-
max_tokens: req.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
1054
|
-
...system || req.system ? { system: req.system ?? system } : {},
|
|
1055
|
-
...req.temperature !== undefined ? { temperature: req.temperature } : {},
|
|
1056
|
-
...req.topP !== undefined ? { top_p: req.topP } : {},
|
|
1057
|
-
...req.stopSequences?.length ? { stop_sequences: req.stopSequences } : {},
|
|
1058
|
-
...buildSeedMetadata(req)
|
|
1059
|
-
};
|
|
1060
|
-
const tools = formatTools(req.tools);
|
|
1061
|
-
if (tools)
|
|
1062
|
-
body.tools = tools;
|
|
1063
|
-
if (req.schema) {
|
|
1064
|
-
const jsonSchema = zodToJsonSchema(req.schema);
|
|
1065
|
-
const structuredTool = {
|
|
1066
|
-
name: "_structured_output",
|
|
1067
|
-
description: "Return structured output matching the required schema",
|
|
1068
|
-
input_schema: jsonSchema
|
|
1069
|
-
};
|
|
1070
|
-
body.tools = [...tools ?? [], structuredTool];
|
|
1071
|
-
body.tool_choice = { type: "tool", name: "_structured_output" };
|
|
1072
|
-
}
|
|
1111
|
+
const body = buildRequestBody(req);
|
|
1073
1112
|
const startTime = performance.now();
|
|
1074
|
-
const raw = await retry(async () => {
|
|
1075
|
-
const
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
const signals = [controller.signal, req.signal].filter(Boolean);
|
|
1079
|
-
const mergedSignal = signals.length > 1 ? AbortSignal.any(signals) : signals[0];
|
|
1080
|
-
const resp = await request("/messages", body, mergedSignal);
|
|
1081
|
-
return await resp.json();
|
|
1082
|
-
} finally {
|
|
1083
|
-
clearTimeout(timer);
|
|
1084
|
-
}
|
|
1085
|
-
}, {
|
|
1113
|
+
const raw = await retry(() => executeWithTimeout(async (signal) => {
|
|
1114
|
+
const resp = await request("/messages", body, signal);
|
|
1115
|
+
return await resp.json();
|
|
1116
|
+
}, req.signal), {
|
|
1086
1117
|
maxRetries,
|
|
1087
1118
|
baseDelayMs: 1000,
|
|
1088
1119
|
shouldRetry: (e) => e instanceof ElsiumError && e.retryable
|
|
@@ -1091,29 +1122,12 @@ function createAnthropicProvider(config) {
|
|
|
1091
1122
|
return parseResponse(raw, latencyMs);
|
|
1092
1123
|
},
|
|
1093
1124
|
stream(req) {
|
|
1094
|
-
const
|
|
1095
|
-
|
|
1096
|
-
const
|
|
1097
|
-
model,
|
|
1098
|
-
messages,
|
|
1099
|
-
max_tokens: req.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
1100
|
-
stream: true,
|
|
1101
|
-
...system || req.system ? { system: req.system ?? system } : {},
|
|
1102
|
-
...req.temperature !== undefined ? { temperature: req.temperature } : {},
|
|
1103
|
-
...req.topP !== undefined ? { top_p: req.topP } : {},
|
|
1104
|
-
...req.stopSequences?.length ? { stop_sequences: req.stopSequences } : {},
|
|
1105
|
-
...buildSeedMetadata(req)
|
|
1106
|
-
};
|
|
1107
|
-
const tools = formatTools(req.tools);
|
|
1108
|
-
if (tools)
|
|
1109
|
-
body.tools = tools;
|
|
1125
|
+
const body = buildRequestBody(req);
|
|
1126
|
+
body.stream = true;
|
|
1127
|
+
const model = body.model ?? "claude-sonnet-4-6";
|
|
1110
1128
|
return createStream(async (emit) => {
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
try {
|
|
1114
|
-
const signals = [controller.signal, req.signal].filter(Boolean);
|
|
1115
|
-
const mergedSignal = signals.length > 1 ? AbortSignal.any(signals) : signals[0];
|
|
1116
|
-
const resp = await request("/messages", body, mergedSignal);
|
|
1129
|
+
await executeWithTimeout(async (signal) => {
|
|
1130
|
+
const resp = await request("/messages", body, signal);
|
|
1117
1131
|
if (!resp.body)
|
|
1118
1132
|
throw new ElsiumError({
|
|
1119
1133
|
code: "STREAM_ERROR",
|
|
@@ -1122,9 +1136,7 @@ function createAnthropicProvider(config) {
|
|
|
1122
1136
|
retryable: false
|
|
1123
1137
|
});
|
|
1124
1138
|
await processAnthropicSSEStream(resp.body, model, emit);
|
|
1125
|
-
}
|
|
1126
|
-
clearTimeout(timer);
|
|
1127
|
-
}
|
|
1139
|
+
}, req.signal);
|
|
1128
1140
|
});
|
|
1129
1141
|
},
|
|
1130
1142
|
async listModels() {
|
|
@@ -1278,31 +1290,38 @@ function createGoogleProvider(config) {
|
|
|
1278
1290
|
}
|
|
1279
1291
|
return { role, parts };
|
|
1280
1292
|
}
|
|
1293
|
+
function convertGeminiImagePart(p) {
|
|
1294
|
+
if (p.source.type === "base64") {
|
|
1295
|
+
return { inlineData: { mimeType: p.source.mediaType, data: p.source.data } };
|
|
1296
|
+
}
|
|
1297
|
+
return { fileData: { mimeType: "image/jpeg", fileUri: p.source.url } };
|
|
1298
|
+
}
|
|
1299
|
+
function convertGeminiMediaPart(p) {
|
|
1300
|
+
if (p.source.type === "base64") {
|
|
1301
|
+
return { inlineData: { mimeType: p.source.mediaType, data: p.source.data } };
|
|
1302
|
+
}
|
|
1303
|
+
const urlSource = p.source;
|
|
1304
|
+
return { fileData: { mimeType: "application/octet-stream", fileUri: urlSource.url } };
|
|
1305
|
+
}
|
|
1306
|
+
function convertGeminiContentPart(p) {
|
|
1307
|
+
if (p.type === "text") {
|
|
1308
|
+
return { text: p.text };
|
|
1309
|
+
}
|
|
1310
|
+
if (p.type === "image") {
|
|
1311
|
+
return convertGeminiImagePart(p);
|
|
1312
|
+
}
|
|
1313
|
+
if (p.type === "audio" || p.type === "document") {
|
|
1314
|
+
return convertGeminiMediaPart(p);
|
|
1315
|
+
}
|
|
1316
|
+
return null;
|
|
1317
|
+
}
|
|
1281
1318
|
function formatGeminiMultipartContent(msg, role) {
|
|
1319
|
+
const content = msg.content;
|
|
1282
1320
|
const parts = [];
|
|
1283
|
-
for (const p of
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
const img = p;
|
|
1288
|
-
if (img.source.type === "base64") {
|
|
1289
|
-
parts.push({ inlineData: { mimeType: img.source.mediaType, data: img.source.data } });
|
|
1290
|
-
} else {
|
|
1291
|
-
parts.push({ fileData: { mimeType: "image/jpeg", fileUri: img.source.url } });
|
|
1292
|
-
}
|
|
1293
|
-
} else if (p.type === "audio" || p.type === "document") {
|
|
1294
|
-
const media = p;
|
|
1295
|
-
if (media.source.type === "base64") {
|
|
1296
|
-
parts.push({
|
|
1297
|
-
inlineData: { mimeType: media.source.mediaType, data: media.source.data }
|
|
1298
|
-
});
|
|
1299
|
-
} else {
|
|
1300
|
-
const urlSource = media.source;
|
|
1301
|
-
parts.push({
|
|
1302
|
-
fileData: { mimeType: "application/octet-stream", fileUri: urlSource.url }
|
|
1303
|
-
});
|
|
1304
|
-
}
|
|
1305
|
-
}
|
|
1321
|
+
for (const p of content) {
|
|
1322
|
+
const converted = convertGeminiContentPart(p);
|
|
1323
|
+
if (converted)
|
|
1324
|
+
parts.push(converted);
|
|
1306
1325
|
}
|
|
1307
1326
|
return { role, parts };
|
|
1308
1327
|
}
|
|
@@ -1669,40 +1688,48 @@ function createOpenAIProvider(config) {
|
|
|
1669
1688
|
}
|
|
1670
1689
|
return openaiMsg;
|
|
1671
1690
|
}
|
|
1691
|
+
function convertImagePart(part) {
|
|
1692
|
+
if (part.source.type === "base64") {
|
|
1693
|
+
const url = `data:${part.source.mediaType};base64,${part.source.data}`;
|
|
1694
|
+
return { type: "image_url", image_url: { url } };
|
|
1695
|
+
}
|
|
1696
|
+
return { type: "image_url", image_url: { url: part.source.url } };
|
|
1697
|
+
}
|
|
1698
|
+
function convertAudioPart(part) {
|
|
1699
|
+
if (part.source.type === "base64") {
|
|
1700
|
+
const format = part.source.mediaType.split("/")[1] ?? "wav";
|
|
1701
|
+
return { type: "input_audio", input_audio: { data: part.source.data, format } };
|
|
1702
|
+
}
|
|
1703
|
+
return { type: "text", text: "[audio: url source requires file upload]" };
|
|
1704
|
+
}
|
|
1705
|
+
function convertDocumentPart(part) {
|
|
1706
|
+
if (part.source.type === "base64") {
|
|
1707
|
+
return {
|
|
1708
|
+
type: "text",
|
|
1709
|
+
text: `[document: ${part.source.mediaType} content attached as base64]`
|
|
1710
|
+
};
|
|
1711
|
+
}
|
|
1712
|
+
return { type: "text", text: `[document: ${part.source.url}]` };
|
|
1713
|
+
}
|
|
1714
|
+
function convertContentPart(part) {
|
|
1715
|
+
if (part.type === "text")
|
|
1716
|
+
return { type: "text", text: part.text };
|
|
1717
|
+
if (part.type === "image")
|
|
1718
|
+
return convertImagePart(part);
|
|
1719
|
+
if (part.type === "audio")
|
|
1720
|
+
return convertAudioPart(part);
|
|
1721
|
+
if (part.type === "document")
|
|
1722
|
+
return convertDocumentPart(part);
|
|
1723
|
+
return null;
|
|
1724
|
+
}
|
|
1672
1725
|
function formatUserContent(msg) {
|
|
1673
1726
|
if (typeof msg.content === "string")
|
|
1674
1727
|
return msg.content;
|
|
1675
1728
|
const parts = [];
|
|
1676
1729
|
for (const part of msg.content) {
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
if (part.source.type === "base64") {
|
|
1681
|
-
const url = `data:${part.source.mediaType};base64,${part.source.data}`;
|
|
1682
|
-
parts.push({ type: "image_url", image_url: { url } });
|
|
1683
|
-
} else {
|
|
1684
|
-
parts.push({ type: "image_url", image_url: { url: part.source.url } });
|
|
1685
|
-
}
|
|
1686
|
-
} else if (part.type === "audio") {
|
|
1687
|
-
if (part.source.type === "base64") {
|
|
1688
|
-
const format = part.source.mediaType.split("/")[1] ?? "wav";
|
|
1689
|
-
parts.push({
|
|
1690
|
-
type: "input_audio",
|
|
1691
|
-
input_audio: { data: part.source.data, format }
|
|
1692
|
-
});
|
|
1693
|
-
} else {
|
|
1694
|
-
parts.push({ type: "text", text: "[audio: url source requires file upload]" });
|
|
1695
|
-
}
|
|
1696
|
-
} else if (part.type === "document") {
|
|
1697
|
-
if (part.source.type === "base64") {
|
|
1698
|
-
parts.push({
|
|
1699
|
-
type: "text",
|
|
1700
|
-
text: `[document: ${part.source.mediaType} content attached as base64]`
|
|
1701
|
-
});
|
|
1702
|
-
} else {
|
|
1703
|
-
parts.push({ type: "text", text: `[document: ${part.source.url}]` });
|
|
1704
|
-
}
|
|
1705
|
-
}
|
|
1730
|
+
const converted = convertContentPart(part);
|
|
1731
|
+
if (converted)
|
|
1732
|
+
parts.push(converted);
|
|
1706
1733
|
}
|
|
1707
1734
|
return parts;
|
|
1708
1735
|
}
|
|
@@ -1737,6 +1764,49 @@ function createOpenAIProvider(config) {
|
|
|
1737
1764
|
}
|
|
1738
1765
|
}));
|
|
1739
1766
|
}
|
|
1767
|
+
function buildOptionalParams(req) {
|
|
1768
|
+
const params = {};
|
|
1769
|
+
if (req.temperature !== undefined)
|
|
1770
|
+
params.temperature = req.temperature;
|
|
1771
|
+
if (req.seed !== undefined)
|
|
1772
|
+
params.seed = req.seed;
|
|
1773
|
+
if (req.topP !== undefined)
|
|
1774
|
+
params.top_p = req.topP;
|
|
1775
|
+
if (req.stopSequences?.length)
|
|
1776
|
+
params.stop = req.stopSequences;
|
|
1777
|
+
return params;
|
|
1778
|
+
}
|
|
1779
|
+
function applyResponseFormat(body, req) {
|
|
1780
|
+
if (!req.schema)
|
|
1781
|
+
return;
|
|
1782
|
+
const jsonSchema = zodToJsonSchema(req.schema);
|
|
1783
|
+
body.response_format = {
|
|
1784
|
+
type: "json_schema",
|
|
1785
|
+
json_schema: {
|
|
1786
|
+
name: "structured_output",
|
|
1787
|
+
strict: true,
|
|
1788
|
+
schema: jsonSchema
|
|
1789
|
+
}
|
|
1790
|
+
};
|
|
1791
|
+
}
|
|
1792
|
+
function buildRequestBody(req) {
|
|
1793
|
+
const messages = formatMessages(req.messages);
|
|
1794
|
+
const model = req.model ?? "gpt-4o";
|
|
1795
|
+
if (req.system) {
|
|
1796
|
+
messages.unshift({ role: "system", content: req.system });
|
|
1797
|
+
}
|
|
1798
|
+
const body = {
|
|
1799
|
+
model,
|
|
1800
|
+
messages,
|
|
1801
|
+
max_tokens: req.maxTokens ?? DEFAULT_MAX_TOKENS2,
|
|
1802
|
+
...buildOptionalParams(req)
|
|
1803
|
+
};
|
|
1804
|
+
const tools = formatTools(req.tools);
|
|
1805
|
+
if (tools)
|
|
1806
|
+
body.tools = tools;
|
|
1807
|
+
applyResponseFormat(body, req);
|
|
1808
|
+
return body;
|
|
1809
|
+
}
|
|
1740
1810
|
function parseResponse(raw, latencyMs) {
|
|
1741
1811
|
const traceId = generateTraceId();
|
|
1742
1812
|
const choice = raw.choices[0];
|
|
@@ -1781,34 +1851,7 @@ function createOpenAIProvider(config) {
|
|
|
1781
1851
|
authStyle: "bearer"
|
|
1782
1852
|
},
|
|
1783
1853
|
async complete(req) {
|
|
1784
|
-
const
|
|
1785
|
-
const model = req.model ?? "gpt-4o";
|
|
1786
|
-
if (req.system) {
|
|
1787
|
-
messages.unshift({ role: "system", content: req.system });
|
|
1788
|
-
}
|
|
1789
|
-
const body = {
|
|
1790
|
-
model,
|
|
1791
|
-
messages,
|
|
1792
|
-
max_tokens: req.maxTokens ?? DEFAULT_MAX_TOKENS2,
|
|
1793
|
-
...req.temperature !== undefined ? { temperature: req.temperature } : {},
|
|
1794
|
-
...req.seed !== undefined ? { seed: req.seed } : {},
|
|
1795
|
-
...req.topP !== undefined ? { top_p: req.topP } : {},
|
|
1796
|
-
...req.stopSequences?.length ? { stop: req.stopSequences } : {}
|
|
1797
|
-
};
|
|
1798
|
-
const tools = formatTools(req.tools);
|
|
1799
|
-
if (tools)
|
|
1800
|
-
body.tools = tools;
|
|
1801
|
-
if (req.schema) {
|
|
1802
|
-
const jsonSchema = zodToJsonSchema(req.schema);
|
|
1803
|
-
body.response_format = {
|
|
1804
|
-
type: "json_schema",
|
|
1805
|
-
json_schema: {
|
|
1806
|
-
name: "structured_output",
|
|
1807
|
-
strict: true,
|
|
1808
|
-
schema: jsonSchema
|
|
1809
|
-
}
|
|
1810
|
-
};
|
|
1811
|
-
}
|
|
1854
|
+
const body = buildRequestBody(req);
|
|
1812
1855
|
const startTime = performance.now();
|
|
1813
1856
|
const raw = await retry(async () => {
|
|
1814
1857
|
const controller = new AbortController;
|
|
@@ -1830,25 +1873,10 @@ function createOpenAIProvider(config) {
|
|
|
1830
1873
|
return parseResponse(raw, latencyMs);
|
|
1831
1874
|
},
|
|
1832
1875
|
stream(req) {
|
|
1833
|
-
const
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
}
|
|
1838
|
-
const body = {
|
|
1839
|
-
model,
|
|
1840
|
-
messages,
|
|
1841
|
-
max_tokens: req.maxTokens ?? DEFAULT_MAX_TOKENS2,
|
|
1842
|
-
stream: true,
|
|
1843
|
-
stream_options: { include_usage: true },
|
|
1844
|
-
...req.temperature !== undefined ? { temperature: req.temperature } : {},
|
|
1845
|
-
...req.seed !== undefined ? { seed: req.seed } : {},
|
|
1846
|
-
...req.topP !== undefined ? { top_p: req.topP } : {},
|
|
1847
|
-
...req.stopSequences?.length ? { stop: req.stopSequences } : {}
|
|
1848
|
-
};
|
|
1849
|
-
const tools = formatTools(req.tools);
|
|
1850
|
-
if (tools)
|
|
1851
|
-
body.tools = tools;
|
|
1876
|
+
const body = buildRequestBody(req);
|
|
1877
|
+
body.stream = true;
|
|
1878
|
+
body.stream_options = { include_usage: true };
|
|
1879
|
+
const model = body.model ?? "gpt-4o";
|
|
1852
1880
|
return createStream(async (emit) => {
|
|
1853
1881
|
const controller = new AbortController;
|
|
1854
1882
|
const timer = setTimeout(() => controller.abort(), timeout);
|
|
@@ -1968,15 +1996,32 @@ var PROVIDER_FACTORIES = {
|
|
|
1968
1996
|
openai: createOpenAIProvider,
|
|
1969
1997
|
google: createGoogleProvider
|
|
1970
1998
|
};
|
|
1999
|
+
registerProviderMetadata("anthropic", {
|
|
2000
|
+
baseUrl: "https://api.anthropic.com/v1/messages",
|
|
2001
|
+
capabilities: ["tools", "vision", "streaming", "system"],
|
|
2002
|
+
authStyle: "x-api-key"
|
|
2003
|
+
});
|
|
2004
|
+
registerProviderMetadata("openai", {
|
|
2005
|
+
baseUrl: "https://api.openai.com/v1/chat/completions",
|
|
2006
|
+
capabilities: ["tools", "vision", "streaming", "system", "json_mode"],
|
|
2007
|
+
authStyle: "bearer"
|
|
2008
|
+
});
|
|
2009
|
+
registerProviderMetadata("google", {
|
|
2010
|
+
baseUrl: "https://generativelanguage.googleapis.com/v1beta/models",
|
|
2011
|
+
capabilities: ["tools", "vision", "streaming", "system"],
|
|
2012
|
+
authStyle: "bearer"
|
|
2013
|
+
});
|
|
1971
2014
|
function registerProviderFactory(name, factory) {
|
|
1972
2015
|
PROVIDER_FACTORIES[name] = factory;
|
|
1973
2016
|
}
|
|
1974
2017
|
function validateGatewayConfig(config) {
|
|
1975
|
-
const factory = PROVIDER_FACTORIES[config.provider];
|
|
2018
|
+
const factory = PROVIDER_FACTORIES[config.provider] ?? getProviderFactory(config.provider);
|
|
1976
2019
|
if (!factory) {
|
|
2020
|
+
const available = [...Object.keys(PROVIDER_FACTORIES), ...listProviders()];
|
|
2021
|
+
const unique = [...new Set(available)];
|
|
1977
2022
|
throw new ElsiumError({
|
|
1978
2023
|
code: "CONFIG_ERROR",
|
|
1979
|
-
message: `Unknown provider: ${config.provider}. Available: ${
|
|
2024
|
+
message: `Unknown provider: ${config.provider}. Available: ${unique.join(", ")}`,
|
|
1980
2025
|
retryable: false
|
|
1981
2026
|
});
|
|
1982
2027
|
}
|
|
@@ -2054,6 +2099,24 @@ async function accumulateStreamEvents(stream, emit) {
|
|
|
2054
2099
|
}
|
|
2055
2100
|
return { textContent, usage, stopReason, id };
|
|
2056
2101
|
}
|
|
2102
|
+
function extractFromToolCalls(response) {
|
|
2103
|
+
if (response.stopReason !== "tool_use" || !response.message.toolCalls?.length) {
|
|
2104
|
+
return;
|
|
2105
|
+
}
|
|
2106
|
+
const structuredCall = response.message.toolCalls.find((tc) => tc.name === "_structured_output");
|
|
2107
|
+
return structuredCall?.arguments;
|
|
2108
|
+
}
|
|
2109
|
+
function extractJsonFromText(response) {
|
|
2110
|
+
let text = typeof response.message.content === "string" ? response.message.content : "";
|
|
2111
|
+
text = text.replace(/^```(?:json)?\s*\n?([\s\S]*?)\n?\s*```$/gm, "$1").trim();
|
|
2112
|
+
const jsonMatch = text.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
|
|
2113
|
+
if (!jsonMatch) {
|
|
2114
|
+
throw ElsiumError.validation("LLM response did not contain valid JSON", {
|
|
2115
|
+
response: text
|
|
2116
|
+
});
|
|
2117
|
+
}
|
|
2118
|
+
return JSON.parse(jsonMatch[0]);
|
|
2119
|
+
}
|
|
2057
2120
|
function gateway(config) {
|
|
2058
2121
|
const factory = validateGatewayConfig(config);
|
|
2059
2122
|
const provider = factory({
|
|
@@ -2075,6 +2138,7 @@ function gateway(config) {
|
|
|
2075
2138
|
allMiddleware.push(xm);
|
|
2076
2139
|
}
|
|
2077
2140
|
const composedMiddleware = allMiddleware.length ? composeMiddleware(allMiddleware) : null;
|
|
2141
|
+
const composedStreamMiddleware = config.streamMiddleware?.length ? composeStreamMiddleware(config.streamMiddleware) : null;
|
|
2078
2142
|
async function executeWithMiddleware(request) {
|
|
2079
2143
|
const req = { ...request, model: request.model ?? defaultModel };
|
|
2080
2144
|
if (!composedMiddleware) {
|
|
@@ -2099,11 +2163,11 @@ function gateway(config) {
|
|
|
2099
2163
|
validateRequestLimits(request, maxMessages, maxInputTokens);
|
|
2100
2164
|
const req = { ...request, model: request.model ?? defaultModel };
|
|
2101
2165
|
if (composedMiddleware) {
|
|
2102
|
-
const
|
|
2166
|
+
const ctx2 = buildMiddlewareContext(req, provider.name, defaultModel, request.metadata ?? {});
|
|
2103
2167
|
return createStream(async (emit) => {
|
|
2104
|
-
await composedMiddleware(
|
|
2168
|
+
await composedMiddleware(ctx2, async (c) => {
|
|
2105
2169
|
const result = await accumulateStreamEvents(provider.stream(c.request), emit);
|
|
2106
|
-
const latencyMs = Math.round(performance.now() -
|
|
2170
|
+
const latencyMs = Math.round(performance.now() - ctx2.startTime);
|
|
2107
2171
|
return {
|
|
2108
2172
|
id: result.id,
|
|
2109
2173
|
message: { role: "assistant", content: result.textContent },
|
|
@@ -2113,12 +2177,21 @@ function gateway(config) {
|
|
|
2113
2177
|
provider: provider.name,
|
|
2114
2178
|
stopReason: result.stopReason,
|
|
2115
2179
|
latencyMs,
|
|
2116
|
-
traceId:
|
|
2180
|
+
traceId: ctx2.traceId
|
|
2117
2181
|
};
|
|
2118
2182
|
});
|
|
2119
2183
|
});
|
|
2120
2184
|
}
|
|
2121
|
-
|
|
2185
|
+
const rawStream = provider.stream(req);
|
|
2186
|
+
if (!composedStreamMiddleware)
|
|
2187
|
+
return rawStream;
|
|
2188
|
+
const ctx = buildMiddlewareContext(req, provider.name, defaultModel, request.metadata ?? {});
|
|
2189
|
+
return createStream(async (emit) => {
|
|
2190
|
+
const processed = composedStreamMiddleware(ctx, rawStream, (_c, s) => s);
|
|
2191
|
+
for await (const event of processed) {
|
|
2192
|
+
emit(event);
|
|
2193
|
+
}
|
|
2194
|
+
});
|
|
2122
2195
|
},
|
|
2123
2196
|
async generate(request) {
|
|
2124
2197
|
const { schema, ...rest } = request;
|
|
@@ -2135,23 +2208,7 @@ function gateway(config) {
|
|
|
2135
2208
|
|
|
2136
2209
|
`)
|
|
2137
2210
|
});
|
|
2138
|
-
|
|
2139
|
-
if (response.stopReason === "tool_use" && response.message.toolCalls?.length) {
|
|
2140
|
-
const structuredCall = response.message.toolCalls.find((tc) => tc.name === "_structured_output");
|
|
2141
|
-
if (structuredCall) {
|
|
2142
|
-
parsed = structuredCall.arguments;
|
|
2143
|
-
}
|
|
2144
|
-
}
|
|
2145
|
-
if (parsed === undefined) {
|
|
2146
|
-
const text = typeof response.message.content === "string" ? response.message.content : "";
|
|
2147
|
-
const jsonMatch = text.match(/\{[\s\S]*\}/);
|
|
2148
|
-
if (!jsonMatch) {
|
|
2149
|
-
throw ElsiumError.validation("LLM response did not contain valid JSON", {
|
|
2150
|
-
response: text
|
|
2151
|
-
});
|
|
2152
|
-
}
|
|
2153
|
-
parsed = JSON.parse(jsonMatch[0]);
|
|
2154
|
-
}
|
|
2211
|
+
const parsed = extractFromToolCalls(response) ?? extractJsonFromText(response);
|
|
2155
2212
|
const result = schema.safeParse(parsed);
|
|
2156
2213
|
if (!result.success) {
|
|
2157
2214
|
throw ElsiumError.validation("LLM response did not match schema", {
|
|
@@ -2219,6 +2276,21 @@ var SECRET_PATTERNS = [
|
|
|
2219
2276
|
detail: "API public key detected",
|
|
2220
2277
|
replacement: "[REDACTED_API_KEY]"
|
|
2221
2278
|
},
|
|
2279
|
+
{
|
|
2280
|
+
pattern: /\bghp_[a-zA-Z0-9]{36,}\b/g,
|
|
2281
|
+
detail: "GitHub personal access token detected",
|
|
2282
|
+
replacement: "[REDACTED_GITHUB_TOKEN]"
|
|
2283
|
+
},
|
|
2284
|
+
{
|
|
2285
|
+
pattern: /\bgho_[a-zA-Z0-9]{36,}\b/g,
|
|
2286
|
+
detail: "GitHub OAuth token detected",
|
|
2287
|
+
replacement: "[REDACTED_GITHUB_TOKEN]"
|
|
2288
|
+
},
|
|
2289
|
+
{
|
|
2290
|
+
pattern: /\bgithub_pat_[a-zA-Z0-9_]{20,}\b/g,
|
|
2291
|
+
detail: "GitHub fine-grained token detected",
|
|
2292
|
+
replacement: "[REDACTED_GITHUB_TOKEN]"
|
|
2293
|
+
},
|
|
2222
2294
|
{
|
|
2223
2295
|
pattern: /\bapi_key[=:]\s*["']?[a-zA-Z0-9_-]{16,}["']?/gi,
|
|
2224
2296
|
detail: "API key assignment detected",
|
|
@@ -2754,6 +2826,35 @@ function outputGuardrailMiddleware(config) {
|
|
|
2754
2826
|
}
|
|
2755
2827
|
// src/batch.ts
|
|
2756
2828
|
var log6 = createLogger();
|
|
2829
|
+
function makeCancelledItem(index) {
|
|
2830
|
+
return { index, success: false, error: "Batch cancelled" };
|
|
2831
|
+
}
|
|
2832
|
+
function makeFailedItem(index, error) {
|
|
2833
|
+
return { index, success: false, error };
|
|
2834
|
+
}
|
|
2835
|
+
async function attemptRequest(gateway2, request, retryPerItem) {
|
|
2836
|
+
let lastError;
|
|
2837
|
+
for (let attempt = 0;attempt <= retryPerItem; attempt++) {
|
|
2838
|
+
try {
|
|
2839
|
+
const response = await gateway2.complete(request);
|
|
2840
|
+
return { response };
|
|
2841
|
+
} catch (err2) {
|
|
2842
|
+
lastError = err2 instanceof Error ? err2.message : String(err2);
|
|
2843
|
+
const isRetryable = attempt < retryPerItem && err2 instanceof ElsiumError && err2.retryable;
|
|
2844
|
+
if (!isRetryable)
|
|
2845
|
+
break;
|
|
2846
|
+
}
|
|
2847
|
+
}
|
|
2848
|
+
return { error: lastError };
|
|
2849
|
+
}
|
|
2850
|
+
function cancelRemaining(results, fromIndex, total) {
|
|
2851
|
+
let cancelled = 0;
|
|
2852
|
+
for (let i = fromIndex;i < total; i++) {
|
|
2853
|
+
results[i] = makeCancelledItem(i);
|
|
2854
|
+
cancelled++;
|
|
2855
|
+
}
|
|
2856
|
+
return cancelled;
|
|
2857
|
+
}
|
|
2757
2858
|
function createBatch(gateway2, config) {
|
|
2758
2859
|
const concurrency = config?.concurrency ?? 5;
|
|
2759
2860
|
const retryPerItem = config?.retryPerItem ?? 0;
|
|
@@ -2769,40 +2870,24 @@ function createBatch(gateway2, config) {
|
|
|
2769
2870
|
const signal = config?.signal;
|
|
2770
2871
|
async function processItem(index) {
|
|
2771
2872
|
if (signal?.aborted) {
|
|
2772
|
-
results[index] =
|
|
2773
|
-
index,
|
|
2774
|
-
success: false,
|
|
2775
|
-
error: "Batch cancelled"
|
|
2776
|
-
};
|
|
2873
|
+
results[index] = makeCancelledItem(index);
|
|
2777
2874
|
totalFailed++;
|
|
2778
2875
|
return;
|
|
2779
2876
|
}
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
} catch (err2) {
|
|
2788
|
-
lastError = err2 instanceof Error ? err2.message : String(err2);
|
|
2789
|
-
if (attempt < retryPerItem && err2 instanceof ElsiumError && err2.retryable) {
|
|
2790
|
-
continue;
|
|
2791
|
-
}
|
|
2792
|
-
break;
|
|
2793
|
-
}
|
|
2877
|
+
const result = await attemptRequest(gateway2, requests[index], retryPerItem);
|
|
2878
|
+
if (result.response) {
|
|
2879
|
+
results[index] = { index, success: true, response: result.response };
|
|
2880
|
+
totalSucceeded++;
|
|
2881
|
+
} else {
|
|
2882
|
+
results[index] = makeFailedItem(index, result.error);
|
|
2883
|
+
totalFailed++;
|
|
2794
2884
|
}
|
|
2795
|
-
results[index] = { index, success: false, error: lastError };
|
|
2796
|
-
totalFailed++;
|
|
2797
2885
|
}
|
|
2798
2886
|
return new Promise((resolve) => {
|
|
2799
2887
|
function scheduleNext() {
|
|
2800
2888
|
while (running < concurrency && nextIndex < requests.length) {
|
|
2801
2889
|
if (signal?.aborted) {
|
|
2802
|
-
|
|
2803
|
-
results[i] = { index: i, success: false, error: "Batch cancelled" };
|
|
2804
|
-
totalFailed++;
|
|
2805
|
-
}
|
|
2890
|
+
totalFailed += cancelRemaining(results, nextIndex, requests.length);
|
|
2806
2891
|
nextIndex = requests.length;
|
|
2807
2892
|
break;
|
|
2808
2893
|
}
|
|
@@ -3100,6 +3185,7 @@ export {
|
|
|
3100
3185
|
getProviderMetadata,
|
|
3101
3186
|
getProviderFactory,
|
|
3102
3187
|
gateway,
|
|
3188
|
+
estimateCost,
|
|
3103
3189
|
detectPromptInjection,
|
|
3104
3190
|
detectJailbreak,
|
|
3105
3191
|
createProviderMesh,
|
|
@@ -3110,6 +3196,7 @@ export {
|
|
|
3110
3196
|
createBatch,
|
|
3111
3197
|
createAnthropicProvider,
|
|
3112
3198
|
costTrackingMiddleware,
|
|
3199
|
+
composeStreamMiddleware,
|
|
3113
3200
|
composeMiddleware,
|
|
3114
3201
|
classifyContent,
|
|
3115
3202
|
checkBlockedPatterns,
|
package/dist/middleware.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { Logger, Middleware, XRayData } from '@elsium-ai/core';
|
|
1
|
+
import type { Logger, Middleware, StreamMiddleware, XRayData } from '@elsium-ai/core';
|
|
2
2
|
export declare function composeMiddleware(middlewares: Middleware[]): Middleware;
|
|
3
|
+
export declare function composeStreamMiddleware(middlewares: StreamMiddleware[]): StreamMiddleware;
|
|
3
4
|
export declare function loggingMiddleware(logger?: Logger): Middleware;
|
|
4
5
|
export declare function costTrackingMiddleware(): Middleware & {
|
|
5
6
|
getTotalCost(): number;
|
package/dist/middleware.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"middleware.d.ts","sourceRoot":"","sources":["../src/middleware.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEX,MAAM,EACN,UAAU,
|
|
1
|
+
{"version":3,"file":"middleware.d.ts","sourceRoot":"","sources":["../src/middleware.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEX,MAAM,EACN,UAAU,EAIV,gBAAgB,EAChB,QAAQ,EACR,MAAM,iBAAiB,CAAA;AAIxB,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,UAAU,CAoBvE;AAED,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,gBAAgB,EAAE,GAAG,gBAAgB,CAczF;AAED,wBAAgB,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,CAyB7D;AAED,wBAAgB,sBAAsB,IAAI,UAAU,GAAG;IACtD,YAAY,IAAI,MAAM,CAAA;IACtB,cAAc,IAAI,MAAM,CAAA;IACxB,YAAY,IAAI,MAAM,CAAA;IACtB,KAAK,IAAI,IAAI,CAAA;CACb,CAuBA;AAkBD,MAAM,WAAW,SAAS;IACzB,QAAQ,IAAI,QAAQ,GAAG,IAAI,CAAA;IAC3B,WAAW,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,EAAE,CAAA;IACvC,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAA;IACnD,KAAK,IAAI,IAAI,CAAA;CACb;AAkFD,wBAAgB,cAAc,CAAC,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,GAAG,SAAS,CA4B5F"}
|
package/dist/pricing.d.ts
CHANGED
|
@@ -2,4 +2,5 @@ import type { CostBreakdown, TokenUsage } from '@elsium-ai/core';
|
|
|
2
2
|
import type { ModelPricing } from './provider';
|
|
3
3
|
export declare function calculateCost(model: string, usage: TokenUsage): CostBreakdown;
|
|
4
4
|
export declare function registerPricing(model: string, pricing: ModelPricing): void;
|
|
5
|
+
export declare function estimateCost(model: string, tokenCount: number): number;
|
|
5
6
|
//# sourceMappingURL=pricing.d.ts.map
|
package/dist/pricing.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pricing.d.ts","sourceRoot":"","sources":["../src/pricing.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAEhE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AA0C9C,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,aAAa,CAwB7E;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,IAAI,CAE1E"}
|
|
1
|
+
{"version":3,"file":"pricing.d.ts","sourceRoot":"","sources":["../src/pricing.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAEhE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AA0C9C,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,aAAa,CAwB7E;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,IAAI,CAE1E;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAItE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../src/providers/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,EAMN,KAAK,cAAc,EAUnB,MAAM,iBAAiB,CAAA;AAExB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AA2C9C,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,cAAc,GAAG,WAAW,
|
|
1
|
+
{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../src/providers/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,EAMN,KAAK,cAAc,EAUnB,MAAM,iBAAiB,CAAA;AAExB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AA2C9C,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,cAAc,GAAG,WAAW,CAqV3E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"google.d.ts","sourceRoot":"","sources":["../../src/providers/google.ts"],"names":[],"mappings":"AAAA,OAAO,EAMN,KAAK,cAAc,EAUnB,MAAM,iBAAiB,CAAA;AAExB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAqC9C,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,WAAW,
|
|
1
|
+
{"version":3,"file":"google.d.ts","sourceRoot":"","sources":["../../src/providers/google.ts"],"names":[],"mappings":"AAAA,OAAO,EAMN,KAAK,cAAc,EAUnB,MAAM,iBAAiB,CAAA;AAExB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAqC9C,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,WAAW,CAoSxE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAON,KAAK,cAAc,EAUnB,MAAM,iBAAiB,CAAA;AAExB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAqD9C,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,WAAW,CAwUxE"}
|
package/dist/security.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"security.d.ts","sourceRoot":"","sources":["../src/security.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,UAAU,EAAqC,MAAM,iBAAiB,CAAA;AAKjG,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,kBAAkB,GAAG,WAAW,GAAG,iBAAiB,GAAG,iBAAiB,CAAA;IAC9E,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAA;CACnC;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,OAAO,CAAA;IACb,UAAU,EAAE,iBAAiB,EAAE,CAAA;CAC/B;AAED,MAAM,WAAW,wBAAwB;IACxC,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;IAC1B,QAAQ,CAAC,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,CAAA;IACpE,WAAW,CAAC,EAAE,CAAC,SAAS,EAAE,iBAAiB,KAAK,IAAI,CAAA;CACpD;
|
|
1
|
+
{"version":3,"file":"security.d.ts","sourceRoot":"","sources":["../src/security.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,UAAU,EAAqC,MAAM,iBAAiB,CAAA;AAKjG,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,kBAAkB,GAAG,WAAW,GAAG,iBAAiB,GAAG,iBAAiB,CAAA;IAC9E,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAA;CACnC;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,OAAO,CAAA;IACb,UAAU,EAAE,iBAAiB,EAAE,CAAA;CAC/B;AAED,MAAM,WAAW,wBAAwB;IACxC,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;IAC1B,QAAQ,CAAC,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,CAAA;IACpE,WAAW,CAAC,EAAE,CAAC,SAAS,EAAE,iBAAiB,KAAK,IAAI,CAAA;CACpD;AAsJD,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,UAAU,GAAG,cAAc,GAAG,YAAY,CAAA;AAEtF,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,kBAAkB,CAAA;IACzB,aAAa,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,CA4BlE;AAQD,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CASvE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CASjE;AAoCD,wBAAgB,aAAa,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,GAClE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,iBAAiB,EAAE,CAAA;CAAE,CAYlD;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,iBAAiB,EAAE,CAa1F;AAwDD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,wBAAwB,GAAG,UAAU,CA+B/E"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elsium-ai/gateway",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Multi-provider LLM gateway for ElsiumAI",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Eric Utrera <ebutrera9103@gmail.com>",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"dev": "bun --watch src/index.ts"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"@elsium-ai/core": "^0.
|
|
29
|
+
"@elsium-ai/core": "^0.4.0",
|
|
30
30
|
"zod": "^3.24.0"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|