llm-proxy 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/utils/llmAsJudge.d.ts +34 -0
- package/dist/utils/llmAsJudge.js +125 -0
- package/dist/utils/llmAsJudge.js.map +1 -0
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -17,8 +17,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
17
17
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
18
18
|
};
|
|
19
19
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
|
+
exports.llmAsJudge = void 0;
|
|
20
21
|
exports.generateLLMResponse = generateLLMResponse;
|
|
21
22
|
exports.generateLLMStreamResponse = generateLLMStreamResponse;
|
|
23
|
+
var llmAsJudge_1 = require("./utils/llmAsJudge");
|
|
24
|
+
Object.defineProperty(exports, "llmAsJudge", { enumerable: true, get: function () { return llmAsJudge_1.llmAsJudge; } });
|
|
22
25
|
const types_1 = require("./types");
|
|
23
26
|
const OpenAIService_1 = __importDefault(require("./services/OpenAIService"));
|
|
24
27
|
const AwsBedrockAnthropicService_1 = __importDefault(require("./services/AwsBedrockAnthropicService"));
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AA+BA,kDA2FC;AAKD,8DAoKC;AAnSD,iDAAgD;AAAvC,wGAAA,UAAU,OAAA;AACnB,mCAA8D;AAC9D,6EAAqD;AACrD,uGAA+E;AAC/E,iFAAyD;AACzD,yFAAiE;AACjE,2FAAmE;AACnE,iGAAyE;AACzE,iGAAyE;AAsBzE,2CAA2C;AACpC,KAAK,UAAU,mBAAmB,CACvC,MAAiC;IAEjC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,GACxE,MAAM,CAAC;IAET,MAAM,EAAE,8BAA8B,EAAE,GAAG,WAAW,CAAC;IACvD,MAAM,EAAE,2BAA2B,EAAE,OAAO,EAAE,GAC5C,8BAA8B,IAAI,EAAE,CAAC;IAEvC,mDAAmD;IACnD,MAAM,QAAQ,GAAG,wBAAc,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE5D,uDAAuD;IACvD,IAAI,OAIuB,CAAC;IAC5B,IAAI,QAAQ,KAAK,iBAAS,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAC3D,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,uBAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;SAAM,IAAI,QAAQ,KAAK,iBAAS,CAAC,iBAAiB,EAAE,CAAC;QACpD,MAAM,EAAE,SAAS,EAAE,GAAG,WAAW,CAAC;QAClC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAC9D,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,oCAA0B,CACtC,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,eAAe,EACzB,SAAS,CAAC,MAAM,CACjB,CAAC;IACJ,CAAC;SAAM,IAAI,QAAQ,KAAK,iBAAS,CAAC,iBAAiB,EAAE,CAAC;QACpD,MAAM,EAAE,SAAS,EAAE,GAAG,WAAW,CAAC;QAClC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAC9D,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,iCAAuB,CACnC,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,eAAe,EACzB,SAAS,CAAC,MAAM,CACjB,CAAC;IACJ,CAAC;SAAM,IAAI,QAAQ,KAAK,iBAAS,CAAC,0BAA0B,EAAE,CAAC;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7C,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CACP,wFAAwF,CACzF,CACF,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,iCAAuB,CAAC,2BAA2B,EAAE,OAAO,CAAC,CAAC;IAC9E,CAAC;SAAM,CAAC;QACN,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,uDAAuD;IACvD,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,GAAG,4BAAkB,CAAC,aAAa,CACxE,QAAQ,EACR,QAAQ,EACR,KAAK,CACN,CAAC;IAEF,kCAAkC;IAClC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,kBAAkB,CAAC;QAChD,QAAQ,EAAE,eAAsB,EAAE,qBAAqB;QACvD,KAAK;QACL,GAAG,CAAC,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,WAAW,EAAE,WAAW,IAAI,CAAC;QAC7B,KAAK,EAAE,SAAS;QAChB,YAAY,EAAE,YAAY,IAAI,EAAE;KACjC,CAAC,CAAC;IAEH,uCAAuC;IACvC,MAAM,eAAe,GACnB,QAAQ,KAAK,iBAAS,CAAC,MAAM;QAC7B,QAAQ,KAAK,iBAAS,CAAC,0BAA0B;QAC/C,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,6BAAmB,CAAC,aAAa,CAAC;YAChC,QAAQ;YACR,QAAQ;YACR,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;IACT,OAAO,eAAiC,CAAC;AAC3C,CAAC;AAED,uCAAuC;AACvC,uCAAuC;AACvC,uCAAuC;AAChC,KAAK,UAAU,yBAAyB,CAC7C,MAAiC;IAEjC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,GACxE,MAAM,CAAC;IAET,MAAM,EAAE,8BAA8B,EAAE,GAAG,WAAW,CAAC;IACvD,MAAM,EAAE,2BAA2B,EAAE,OAAO,EAAE,GAC5C,8BAA8B,IAAI,EAAE,CAAC;IAEvC,mDAAmD;IACnD,MAAM,QAAQ,GAAG,wBAAc,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE5D,uDAAuD;IACvD,IAAI,OAIuB,CAAC;IAC5B,IAAI,QAAQ,KAAK,iBAAS,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAC3D,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,uBAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;SAAM,IAAI,QAAQ,KAAK,iBAAS,CAAC,iBAAiB,EAAE,CAAC;QACpD,MAAM,EAAE,SAAS,EAAE,GAAG,WAAW,CAAC;QAClC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAC9D,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,oCAA0B,CACtC,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,eAAe,EACzB,SAAS,CAAC,MAAM,CACjB,CAAC;IACJ,CAAC;SAAM,IAAI,QAAQ,KAAK,iBAAS,CAAC,iBAAiB,EAAE,CAAC;QACpD,MAAM,EAAE,SAAS,EAAE,GAAG,WAAW,CAAC;QAClC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAC9D,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,iCAAuB,CACnC,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,eAAe,EACzB,SAAS,CAAC,MAAM,CACjB,CAAC;IACJ,CAAC;SAAM,IAAI,QAAQ,KAAK,iBAAS,CAAC,0BAA0B,EAAE,CAAC;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7C,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CACP,wFAAwF,CACzF,CACF,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,iCAAuB,CAAC,2BAA2B,EAAE,OAAO,CAAC,CAAC;IAC9E,CAAC;SAAM,CAAC;QACN,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,uDAAuD;IACvD,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,GAAG,4BAAkB,CAAC,aAAa,CACxE,QAAQ,EACR,QAAQ,EACR,KAAK,CACN,CAAC;IAEF,4CAA4C;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC;QAC9C,QAAQ,EAAE,eAAsB,EAAE,qBAAqB;QACvD,KAAK;QACL,GAAG,CAAC,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,WAAW,EAAE,WAAW,IAAI,CAAC;QAC7B,KAAK,EAAE,SAAS;QAChB,YAAY,EAAE,YAAY,IAAI,EAAE;KACjC,CAAC,CAAC;IAEH,gDAAgD;IAChD,KAAK,SAAS,CAAC,CAAC,eAAe;QAC7B,MAAM,cAAc,GAClB,QAAQ,KAAK,iBAAS,CAAC,MAAM;YAC7B,QAAQ,KAAK,iBAAS,CAAC,0BAA0B,CAAC;QAEpD,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,MAAM,KAAuB,CAAC;YAChC,CAAC;YACD,OAAO;QACT,CAAC;QAED,wCAAwC;QACxC,MAAM,MAAM,GAAU,EAAE,CAAC,CAAC,wCAAwC;QAClE,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,MAAM,iBAAiB,GAAU,EAAE,CAAC,CAAC,uCAAuC;QAE5E,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,+BAA+B;gBAC/B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEnB,oDAAoD;gBACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACxB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,CAAC;oBAEtC,yBAAyB;oBACzB,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;wBACjE,cAAc,GAAG,IAAI,CAAC;oBACxB,CAAC;oBAED,6DAA6D;oBAC7D,IAAI,cAAc,EAAE,CAAC;wBACnB,iBAAiB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;wBAClC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;oBACpB,CAAC;yBAAM,CAAC;wBACN,wBAAwB;wBACxB,MAAM,CAAC,MAAM,6BAAmB,CAAC,aAAa,CAAC;4BAC7C,QAAQ,EAAE,KAAK;4BACf,QAAQ;4BACR,QAAQ,EAAE,IAAI;4BACd,cAAc,EAAE,KAAK;yBACtB,CAAC,CAAmB,CAAC;wBAEtB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,yCAAyC;oBAC3D,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,sCAAsC;gBACtC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,wDAAwD;YACxD,MAAM,YAAY,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBACzD,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,UAAU,CAAC;gBACjC,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,CAAC,MAAM,6BAAmB,CAAC,aAAa,CAAC;gBACxD,QAAQ,EAAE,YAAY;gBACtB,QAAQ;gBACR,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,IAAI;aACrB,CAAC,CAAmB,CAAC;YAEtB,MAAM,QAAQ,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,mEAAmE;YACnE,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAG,CAAC,MAAM,6BAAmB,CAAC,aAAa,CAAC;oBACxD,QAAQ,EAAE,KAAK;oBACf,QAAQ;oBACR,QAAQ,EAAE,IAAI;iBACf,CAAC,CAAmB,CAAC;gBACtB,MAAM,QAAQ,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,eAAe,EAAE,CAAC;AAC3B,CAAC;AAED,0CAAwB"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { OpenAIMessages } from "../index";
|
|
2
|
+
interface JudgeCredentials {
|
|
3
|
+
apiKey?: string;
|
|
4
|
+
awsConfig?: {
|
|
5
|
+
accessKeyId: string;
|
|
6
|
+
secretAccessKey: string;
|
|
7
|
+
region: string;
|
|
8
|
+
};
|
|
9
|
+
openAICompatibleProviderConfig?: {
|
|
10
|
+
openAICompatibleProviderKey: string;
|
|
11
|
+
baseUrl: string;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
interface JudgeParams {
|
|
15
|
+
messages: OpenAIMessages;
|
|
16
|
+
judgePrompt: string;
|
|
17
|
+
model: string;
|
|
18
|
+
provider: string;
|
|
19
|
+
credentials: JudgeCredentials;
|
|
20
|
+
maxTokens?: number;
|
|
21
|
+
temperature?: number;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Uses LLM-as-a-judge to evaluate an assistant's response or trajectory
|
|
25
|
+
* Following LangChain multi-turn simulation patterns
|
|
26
|
+
* Returns: { passed: boolean, score: number | null, judgeResponse: string, tokenUsage }
|
|
27
|
+
*/
|
|
28
|
+
export declare function llmAsJudge({ messages, judgePrompt, model, credentials, maxTokens, temperature, }: JudgeParams): Promise<{
|
|
29
|
+
passed: boolean;
|
|
30
|
+
score: number | null;
|
|
31
|
+
judgeResponse: string;
|
|
32
|
+
tokenUsage?: any;
|
|
33
|
+
}>;
|
|
34
|
+
export {};
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.llmAsJudge = llmAsJudge;
|
|
4
|
+
const index_1 = require("../index");
|
|
5
|
+
/**
|
|
6
|
+
* Uses LLM-as-a-judge to evaluate an assistant's response or trajectory
|
|
7
|
+
* Following LangChain multi-turn simulation patterns
|
|
8
|
+
* Returns: { passed: boolean, score: number | null, judgeResponse: string, tokenUsage }
|
|
9
|
+
*/
|
|
10
|
+
async function llmAsJudge({ messages, judgePrompt, model, credentials, maxTokens = 500, temperature = 0, }) {
|
|
11
|
+
var _a, _b, _c;
|
|
12
|
+
try {
|
|
13
|
+
// Format the trajectory for the judge
|
|
14
|
+
// Following LangChain pattern: evaluate the entire conversation trajectory
|
|
15
|
+
const trajectoryText = messages
|
|
16
|
+
.map((msg, idx) => {
|
|
17
|
+
const role = msg.role === "system"
|
|
18
|
+
? "System"
|
|
19
|
+
: msg.role === "user"
|
|
20
|
+
? "User"
|
|
21
|
+
: msg.role === "assistant"
|
|
22
|
+
? "Assistant"
|
|
23
|
+
: msg.role === "tool" || msg.role === "function"
|
|
24
|
+
? "Tool"
|
|
25
|
+
: "Unknown";
|
|
26
|
+
const content = typeof msg.content === "string"
|
|
27
|
+
? msg.content
|
|
28
|
+
: JSON.stringify(msg.content);
|
|
29
|
+
return `Turn ${idx + 1} [${role}]: ${content}`;
|
|
30
|
+
})
|
|
31
|
+
.join("\n\n");
|
|
32
|
+
// Build judge prompt with trajectory context
|
|
33
|
+
const enhancedPrompt = judgePrompt
|
|
34
|
+
.replace(/{{messages\[-1\]}}/g, ((_a = messages[messages.length - 1]) === null || _a === void 0 ? void 0 : _a.content) || "")
|
|
35
|
+
.replace(/{{messages}}/g, trajectoryText)
|
|
36
|
+
.replace(/\{\{trajectory\}\}/g, trajectoryText);
|
|
37
|
+
// Build judge messages with the enhanced prompt
|
|
38
|
+
const judgeMessages = [
|
|
39
|
+
{
|
|
40
|
+
role: "system",
|
|
41
|
+
content: enhancedPrompt,
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
role: "user",
|
|
45
|
+
content: `Please evaluate the following conversation trajectory:\n\n${trajectoryText}\n\nRespond with JSON format: {"passed": true/false, "score": 0.0-1.0, "reasoning": "your explanation"}`,
|
|
46
|
+
},
|
|
47
|
+
];
|
|
48
|
+
const response = await (0, index_1.generateLLMResponse)({
|
|
49
|
+
messages: judgeMessages,
|
|
50
|
+
model,
|
|
51
|
+
max_tokens: maxTokens,
|
|
52
|
+
temperature,
|
|
53
|
+
credentials,
|
|
54
|
+
});
|
|
55
|
+
const judgeResponse = ((_c = (_b = response.choices[0]) === null || _b === void 0 ? void 0 : _b.message) === null || _c === void 0 ? void 0 : _c.content) || "";
|
|
56
|
+
// Parse judge response to extract passed/score
|
|
57
|
+
// Expected format: JSON like {"passed": true, "score": 0.95, "reasoning": "..."}
|
|
58
|
+
let passed = false;
|
|
59
|
+
let score = null;
|
|
60
|
+
try {
|
|
61
|
+
const jsonMatch = judgeResponse.match(/\{[\s\S]*\}/);
|
|
62
|
+
if (jsonMatch) {
|
|
63
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
64
|
+
passed =
|
|
65
|
+
parsed.passed === true ||
|
|
66
|
+
parsed.passed === "true" ||
|
|
67
|
+
parsed.passed === true;
|
|
68
|
+
// Handle score - normalize to 0-1 range if needed
|
|
69
|
+
if (parsed.score !== undefined && parsed.score !== null) {
|
|
70
|
+
score = parseFloat(String(parsed.score));
|
|
71
|
+
// If score is > 1, assume it's 0-10 scale and normalize
|
|
72
|
+
if (score > 1 && score <= 10) {
|
|
73
|
+
score = score / 10;
|
|
74
|
+
}
|
|
75
|
+
// Ensure score is between 0 and 1
|
|
76
|
+
score = Math.max(0, Math.min(1, score));
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
// Fallback: check for keywords and extract score
|
|
81
|
+
const lowerResponse = judgeResponse.toLowerCase();
|
|
82
|
+
passed =
|
|
83
|
+
lowerResponse.includes("pass") ||
|
|
84
|
+
lowerResponse.includes("correct") ||
|
|
85
|
+
lowerResponse.includes("yes") ||
|
|
86
|
+
lowerResponse.includes("true") ||
|
|
87
|
+
lowerResponse.includes("satisfied") ||
|
|
88
|
+
lowerResponse.includes("successful");
|
|
89
|
+
// Try to extract score from text (look for 0-1 or 0-10 scale)
|
|
90
|
+
const scoreMatch = judgeResponse.match(/score[:\s]+([0-9.]+)/i);
|
|
91
|
+
if (scoreMatch) {
|
|
92
|
+
score = parseFloat(scoreMatch[1] || "0");
|
|
93
|
+
if (score > 1 && score <= 10) {
|
|
94
|
+
score = score / 10;
|
|
95
|
+
}
|
|
96
|
+
score = Math.max(0, Math.min(1, score));
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
catch (parseError) {
|
|
101
|
+
// If parsing fails, use keyword detection
|
|
102
|
+
const lowerResponse = judgeResponse.toLowerCase();
|
|
103
|
+
passed =
|
|
104
|
+
lowerResponse.includes("pass") ||
|
|
105
|
+
lowerResponse.includes("correct") ||
|
|
106
|
+
lowerResponse.includes("yes") ||
|
|
107
|
+
lowerResponse.includes("satisfied");
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
passed,
|
|
111
|
+
score,
|
|
112
|
+
judgeResponse,
|
|
113
|
+
tokenUsage: response.usage,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
return {
|
|
118
|
+
passed: false,
|
|
119
|
+
score: null,
|
|
120
|
+
judgeResponse: `Error in judge evaluation: ${error instanceof Error ? error.message : String(error)}`,
|
|
121
|
+
tokenUsage: undefined,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=llmAsJudge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llmAsJudge.js","sourceRoot":"","sources":["../../src/utils/llmAsJudge.ts"],"names":[],"mappings":";;AA8BA,gCAwIC;AAtKD,oCAA+D;AAyB/D;;;;GAIG;AACI,KAAK,UAAU,UAAU,CAAC,EAC/B,QAAQ,EACR,WAAW,EACX,KAAK,EACL,WAAW,EACX,SAAS,GAAG,GAAG,EACf,WAAW,GAAG,CAAC,GACH;;IAMZ,IAAI,CAAC;QACH,sCAAsC;QACtC,2EAA2E;QAC3E,MAAM,cAAc,GAAG,QAAQ;aAC5B,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAChB,MAAM,IAAI,GACR,GAAG,CAAC,IAAI,KAAK,QAAQ;gBACnB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM;oBACnB,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,WAAW;wBACxB,CAAC,CAAC,WAAW;wBACb,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU;4BAC9C,CAAC,CAAC,MAAM;4BACR,CAAC,CAAC,SAAS,CAAC;YACtB,MAAM,OAAO,GACX,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;gBAC7B,CAAC,CAAC,GAAG,CAAC,OAAO;gBACb,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAClC,OAAO,QAAQ,GAAG,GAAG,CAAC,KAAK,IAAI,MAAM,OAAO,EAAE,CAAC;QACjD,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,6CAA6C;QAC7C,MAAM,cAAc,GAAG,WAAW;aAC/B,OAAO,CACN,qBAAqB,EACrB,CAAA,MAAA,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,0CAAE,OAAO,KAAI,EAAE,CAC7C;aACA,OAAO,CAAC,eAAe,EAAE,cAAc,CAAC;aACxC,OAAO,CAAC,qBAAqB,EAAE,cAAc,CAAC,CAAC;QAElD,gDAAgD;QAChD,MAAM,aAAa,GAAmB;YACpC;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,cAAc;aACxB;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,6DAA6D,cAAc,yGAAyG;aAC9L;SACF,CAAC;QAEF,MAAM,QAAQ,GAAG,MAAM,IAAA,2BAAmB,EAAC;YACzC,QAAQ,EAAE,aAAa;YACvB,KAAK;YACL,UAAU,EAAE,SAAS;YACrB,WAAW;YACX,WAAW;SACZ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,CAAA,MAAA,MAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,OAAO,0CAAE,OAAO,KAAI,EAAE,CAAC;QAElE,+CAA+C;QAC/C,iFAAiF;QACjF,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,KAAK,GAAkB,IAAI,CAAC;QAEhC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YACrD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,MAAM;oBACJ,MAAM,CAAC,MAAM,KAAK,IAAI;wBACtB,MAAM,CAAC,MAAM,KAAK,MAAM;wBACxB,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC;gBAEzB,kDAAkD;gBAClD,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;oBACxD,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;oBACzC,wDAAwD;oBACxD,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,EAAE,CAAC;wBAC7B,KAAK,GAAG,KAAK,GAAG,EAAE,CAAC;oBACrB,CAAC;oBACD,kCAAkC;oBAClC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,MAAM,aAAa,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC;gBAClD,MAAM;oBACJ,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAC9B,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC;wBACjC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC;wBAC7B,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAC9B,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC;wBACnC,aAAa,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gBAEvC,8DAA8D;gBAC9D,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;gBAChE,IAAI,UAAU,EAAE,CAAC;oBACf,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;oBACzC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,EAAE,CAAC;wBAC7B,KAAK,GAAG,KAAK,GAAG,EAAE,CAAC;oBACrB,CAAC;oBACD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,0CAA0C;YAC1C,MAAM,aAAa,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC;YAClD,MAAM;gBACJ,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC9B,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC;oBACjC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAC7B,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QACxC,CAAC;QAED,OAAO;YACL,MAAM;YACN,KAAK;YACL,aAAa;YACb,UAAU,EAAE,QAAQ,CAAC,KAAK;SAC3B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,IAAI;YACX,aAAa,EAAE,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACrG,UAAU,EAAE,SAAS;SACtB,CAAC;IACJ,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "llm-proxy",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.0",
|
|
4
4
|
"description": "An LLM Proxy that allows the user to interact with different language models from different providers using unified request and response formats.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|