llm-proxy 1.0.19 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -65,9 +65,6 @@ Below is a flow diagram illustrating how llm-proxy processes requests.
|
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
|
|
68
|
-
|
|
69
|
-
-- diagram goes here
|
|
70
|
-
|
|
71
68
|
## Contributing
|
|
72
69
|
|
|
73
70
|
Contributions are welcome! Please follow the standard GitHub flow for submitting issues and pull requests.
|
|
@@ -2,6 +2,7 @@ import { LLMResponse, Providers } from "../types";
|
|
|
2
2
|
export declare class OutputFormatAdapter {
|
|
3
3
|
static adaptResponse(response: any, provider: Providers): LLMResponse;
|
|
4
4
|
private static adaptAnthropicBedrockResponse;
|
|
5
|
+
private static adaptAnthropicBedrockStreamResponse;
|
|
5
6
|
private static mapRole;
|
|
6
7
|
private static extractContent;
|
|
7
8
|
}
|
|
@@ -4,19 +4,33 @@ exports.OutputFormatAdapter = void 0;
|
|
|
4
4
|
const types_1 = require("../types");
|
|
5
5
|
class OutputFormatAdapter {
|
|
6
6
|
static adaptResponse(response, provider) {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
7
|
+
try {
|
|
8
|
+
switch (provider) {
|
|
9
|
+
case types_1.Providers.OPENAI:
|
|
10
|
+
return response;
|
|
11
|
+
case types_1.Providers.ANTHROPIC_BEDROCK:
|
|
12
|
+
if (response.type === "message_start") {
|
|
13
|
+
return this.adaptAnthropicBedrockStreamResponse(response);
|
|
14
|
+
}
|
|
15
|
+
return this.adaptAnthropicBedrockResponse(response);
|
|
16
|
+
default:
|
|
17
|
+
throw new Error(`Unsupported provider: ${provider}`);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
const errorMessage = error instanceof Error ? error.message : "Unknown error";
|
|
22
|
+
throw new Error(`Failed to adapt response: ${errorMessage}`);
|
|
14
23
|
}
|
|
15
24
|
}
|
|
16
25
|
static adaptAnthropicBedrockResponse(response) {
|
|
17
|
-
|
|
26
|
+
var _a, _b, _c, _d;
|
|
27
|
+
// TODO: define type
|
|
28
|
+
if (!response || !response.id || !response.model || !response.content) {
|
|
29
|
+
throw new Error("Invalid Anthropic Bedrock response structure");
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
18
32
|
id: response.id,
|
|
19
|
-
object: "
|
|
33
|
+
object: "chat.completion",
|
|
20
34
|
created: Date.now(),
|
|
21
35
|
model: response.model,
|
|
22
36
|
choices: response.content.map((contentBlock, index) => ({
|
|
@@ -26,20 +40,88 @@ class OutputFormatAdapter {
|
|
|
26
40
|
content: this.extractContent(contentBlock),
|
|
27
41
|
},
|
|
28
42
|
logprobs: null,
|
|
29
|
-
finish_reason: response.stop_reason,
|
|
43
|
+
finish_reason: response.stop_reason || null,
|
|
30
44
|
})),
|
|
31
45
|
usage: {
|
|
32
|
-
prompt_tokens: response.usage.input_tokens,
|
|
33
|
-
completion_tokens: response.usage.output_tokens,
|
|
34
|
-
total_tokens: response.usage.input_tokens +
|
|
46
|
+
prompt_tokens: ((_a = response.usage) === null || _a === void 0 ? void 0 : _a.input_tokens) || 0,
|
|
47
|
+
completion_tokens: ((_b = response.usage) === null || _b === void 0 ? void 0 : _b.output_tokens) || 0,
|
|
48
|
+
total_tokens: (((_c = response.usage) === null || _c === void 0 ? void 0 : _c.input_tokens) || 0) +
|
|
49
|
+
(((_d = response.usage) === null || _d === void 0 ? void 0 : _d.output_tokens) || 0),
|
|
35
50
|
prompt_tokens_details: { cached_tokens: 0 },
|
|
36
51
|
completion_tokens_details: { reasoning_tokens: 0 },
|
|
37
52
|
},
|
|
38
53
|
system_fingerprint: "anthropic_translation",
|
|
39
54
|
};
|
|
40
|
-
|
|
55
|
+
}
|
|
56
|
+
static adaptAnthropicBedrockStreamResponse(chunk) {
|
|
57
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
58
|
+
// TODO: define type
|
|
59
|
+
try {
|
|
60
|
+
// For message start chunks
|
|
61
|
+
if (chunk.message) {
|
|
62
|
+
return {
|
|
63
|
+
id: chunk.message.id || "stream_chunk",
|
|
64
|
+
object: "chat.completion.chunk",
|
|
65
|
+
created: Date.now(),
|
|
66
|
+
model: chunk.message.model || "anthropic_stream",
|
|
67
|
+
choices: [
|
|
68
|
+
{
|
|
69
|
+
index: 0,
|
|
70
|
+
delta: {
|
|
71
|
+
content: ((_b = (_a = chunk.message.content) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.type) === "text"
|
|
72
|
+
? chunk.message.content[0]
|
|
73
|
+
.text
|
|
74
|
+
: "",
|
|
75
|
+
},
|
|
76
|
+
logprobs: null,
|
|
77
|
+
finish_reason: chunk.message.stop_reason || null,
|
|
78
|
+
},
|
|
79
|
+
],
|
|
80
|
+
usage: {
|
|
81
|
+
prompt_tokens: ((_c = chunk.message.usage) === null || _c === void 0 ? void 0 : _c.input_tokens) || 0,
|
|
82
|
+
completion_tokens: ((_d = chunk.message.usage) === null || _d === void 0 ? void 0 : _d.output_tokens) || 0,
|
|
83
|
+
total_tokens: (((_e = chunk.message.usage) === null || _e === void 0 ? void 0 : _e.input_tokens) || 0) +
|
|
84
|
+
(((_f = chunk.message.usage) === null || _f === void 0 ? void 0 : _f.output_tokens) || 0),
|
|
85
|
+
prompt_tokens_details: { cached_tokens: 0 },
|
|
86
|
+
completion_tokens_details: { reasoning_tokens: 0 },
|
|
87
|
+
},
|
|
88
|
+
system_fingerprint: "anthropic_translation",
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
// For content chunks
|
|
92
|
+
return {
|
|
93
|
+
id: "stream_chunk",
|
|
94
|
+
object: "chat.completion.chunk",
|
|
95
|
+
created: Date.now(),
|
|
96
|
+
model: "anthropic_stream",
|
|
97
|
+
choices: [
|
|
98
|
+
{
|
|
99
|
+
index: 0,
|
|
100
|
+
delta: {
|
|
101
|
+
content: ((_g = chunk.content_block) === null || _g === void 0 ? void 0 : _g.text) || "",
|
|
102
|
+
},
|
|
103
|
+
logprobs: null,
|
|
104
|
+
finish_reason: null,
|
|
105
|
+
},
|
|
106
|
+
],
|
|
107
|
+
usage: {
|
|
108
|
+
prompt_tokens: 0,
|
|
109
|
+
completion_tokens: 1,
|
|
110
|
+
total_tokens: 1,
|
|
111
|
+
prompt_tokens_details: { cached_tokens: 0 },
|
|
112
|
+
completion_tokens_details: { reasoning_tokens: 0 },
|
|
113
|
+
},
|
|
114
|
+
system_fingerprint: "anthropic_translation",
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
throw new Error(`Failed to adapt stream response: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
119
|
+
}
|
|
41
120
|
}
|
|
42
121
|
static mapRole(content) {
|
|
122
|
+
if (!content || !content.type) {
|
|
123
|
+
throw new Error("Invalid content structure: missing type");
|
|
124
|
+
}
|
|
43
125
|
switch (content.type) {
|
|
44
126
|
case types_1.BedrockAnthropicContentType.TOOL_USE:
|
|
45
127
|
case types_1.BedrockAnthropicContentType.TOOL_RESULT:
|
|
@@ -51,15 +133,18 @@ class OutputFormatAdapter {
|
|
|
51
133
|
}
|
|
52
134
|
}
|
|
53
135
|
static extractContent(content) {
|
|
136
|
+
if (!content || !content.type) {
|
|
137
|
+
throw new Error("Invalid content structure: missing type");
|
|
138
|
+
}
|
|
54
139
|
switch (content.type) {
|
|
55
140
|
case types_1.BedrockAnthropicContentType.TEXT:
|
|
56
|
-
return content.text;
|
|
141
|
+
return content.text || "";
|
|
57
142
|
case types_1.BedrockAnthropicContentType.TOOL_RESULT:
|
|
58
143
|
return content.content || "";
|
|
59
144
|
case types_1.BedrockAnthropicContentType.TOOL_USE:
|
|
60
145
|
return content.id || "";
|
|
61
146
|
default:
|
|
62
|
-
|
|
147
|
+
throw new Error(`Unsupported content type: ${content.type}`);
|
|
63
148
|
}
|
|
64
149
|
}
|
|
65
150
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OutputFormatAdapter.js","sourceRoot":"","sources":["../../src/middleware/OutputFormatAdapter.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"OutputFormatAdapter.js","sourceRoot":"","sources":["../../src/middleware/OutputFormatAdapter.ts"],"names":[],"mappings":";;;AAAA,oCAYkB;AAElB,MAAa,mBAAmB;IAC9B,MAAM,CAAC,aAAa,CAAC,QAAa,EAAE,QAAmB;QACrD,IAAI,CAAC;YACH,QAAQ,QAAQ,EAAE,CAAC;gBACjB,KAAK,iBAAS,CAAC,MAAM;oBACnB,OAAO,QAA0B,CAAC;gBACpC,KAAK,iBAAS,CAAC,iBAAiB;oBAC9B,IAAI,QAAQ,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;wBACtC,OAAO,IAAI,CAAC,mCAAmC,CAAC,QAAQ,CAAC,CAAC;oBAC5D,CAAC;oBACD,OAAO,IAAI,CAAC,6BAA6B,CACvC,QAAoC,CACrC,CAAC;gBACJ;oBACE,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,6BAA6B,CAC1C,QAAkC;;QAElC,oBAAoB;QACpB,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QAED,OAAO;YACL,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,iBAAiB;YACzB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtD,KAAK;gBACL,OAAO,EAAE;oBACP,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;oBAChC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC;iBAC3C;gBACD,QAAQ,EAAE,IAAI;gBACd,aAAa,EAAE,QAAQ,CAAC,WAAW,IAAI,IAAI;aAC5C,CAAC,CAAC;YACH,KAAK,EAAE;gBACL,aAAa,EAAE,CAAA,MAAA,QAAQ,CAAC,KAAK,0CAAE,YAAY,KAAI,CAAC;gBAChD,iBAAiB,EAAE,CAAA,MAAA,QAAQ,CAAC,KAAK,0CAAE,aAAa,KAAI,CAAC;gBACrD,YAAY,EACV,CAAC,CAAA,MAAA,QAAQ,CAAC,KAAK,0CAAE,YAAY,KAAI,CAAC,CAAC;oBACnC,CAAC,CAAA,MAAA,QAAQ,CAAC,KAAK,0CAAE,aAAa,KAAI,CAAC,CAAC;gBACtC,qBAAqB,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE;gBAC3C,yBAAyB,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;aACnD;YACD,kBAAkB,EAAE,uBAAuB;SAC5C,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,mCAAmC,CAChD,KAAkC;;QAElC,oBAAoB;QACpB,IAAI,CAAC;YACH,2BAA2B;YAC3B,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAClB,OAAO;oBACL,EAAE,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,cAAc;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;oBACnB,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,kBAAkB;oBAChD,OAAO,EAAE;wBACP;4BACE,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE;gCACL,OAAO,EACL,CAAA,MAAA,MAAA,KAAK,CAAC,OAAO,CAAC,OAAO,0CAAG,CAAC,CAAC,0CAAE,IAAI,MAAK,MAAM;oCACzC,CAAC,CAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAiC;yCACtD,IAAI;oCACT,CAAC,CAAC,EAAE;6BACT;4BACD,QAAQ,EAAE,IAAI;4BACd,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,WAAW,IAAI,IAAI;yBACjD;qBACF;oBACD,KAAK,EAAE;wBACL,aAAa,EAAE,CAAA,MAAA,KAAK,CAAC,OAAO,CAAC,KAAK,0CAAE,YAAY,KAAI,CAAC;wBACrD,iBAAiB,EAAE,CAAA,MAAA,KAAK,CAAC,OAAO,CAAC,KAAK,0CAAE,aAAa,KAAI,CAAC;wBAC1D,YAAY,EACV,CAAC,CAAA,MAAA,KAAK,CAAC,OAAO,CAAC,KAAK,0CAAE,YAAY,KAAI,CAAC,CAAC;4BACxC,CAAC,CAAA,MAAA,KAAK,CAAC,OAAO,CAAC,KAAK,0CAAE,aAAa,KAAI,CAAC,CAAC;wBAC3C,qBAAqB,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE;wBAC3C,yBAAyB,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;qBACnD;oBACD,kBAAkB,EAAE,uBAAuB;iBAC5C,CAAC;YACJ,CAAC;YAED,qBAAqB;YACrB,OAAO;gBACL,EAAE,EAAE,cAAc;gBAClB,MAAM,EAAE,uBAAuB;gBAC/B,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;gBACnB,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE;oBACP;wBACE,KAAK,EAAE,CAAC;wBACR,KAAK,EAAE;4BACL,OAAO,EAAE,CAAA,MAAA,KAAK,CAAC,aAAa,0CAAE,IAAI,KAAI,EAAE;yBACzC;wBACD,QAAQ,EAAE,IAAI;wBACd,aAAa,EAAE,IAAI;qBACpB;iBACF;gBACD,KAAK,EAAE;oBACL,aAAa,EAAE,CAAC;oBAChB,iBAAiB,EAAE,CAAC;oBACpB,YAAY,EAAE,CAAC;oBACf,qBAAqB,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE;oBAC3C,yBAAyB,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;iBACnD;gBACD,kBAAkB,EAAE,uBAAuB;aAC5C,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,oCACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,OAAO,CAAC,OAAgC;QACrD,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,mCAA2B,CAAC,QAAQ,CAAC;YAC1C,KAAK,mCAA2B,CAAC,WAAW;gBAC1C,OAAO,MAAM,CAAC;YAChB,KAAK,mCAA2B,CAAC,IAAI;gBACnC,OAAO,WAAW,CAAC;YACrB;gBACE,OAAO,SAAS,CAAC;QACrB,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,OAAgC;QAC5D,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,mCAA2B,CAAC,IAAI;gBACnC,OAAQ,OAAuC,CAAC,IAAI,IAAI,EAAE,CAAC;YAC7D,KAAK,mCAA2B,CAAC,WAAW;gBAC1C,OAAQ,OAA6C,CAAC,OAAO,IAAI,EAAE,CAAC;YACtE,KAAK,mCAA2B,CAAC,QAAQ;gBACvC,OAAQ,OAA0C,CAAC,EAAE,IAAI,EAAE,CAAC;YAC9D;gBACE,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;CACF;AAnKD,kDAmKC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "llm-proxy",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.21",
|
|
4
4
|
"description": "An LLM Proxy that allows the user to interact with different language models from different providers using unified request and response formats.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|