@agentuity/evals 0.1.24 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_utils.js +2 -2
- package/dist/_utils.js.map +1 -1
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -5
- package/src/_utils.ts +2 -2
- package/src/types.ts +10 -4
package/dist/_utils.js
CHANGED
|
@@ -66,10 +66,10 @@ export function createPresetEval(config) {
|
|
|
66
66
|
description: description ?? config.description,
|
|
67
67
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
68
68
|
handler: (async (ctx, input, output) => {
|
|
69
|
-
const evalInput = middleware
|
|
69
|
+
const evalInput = middleware?.transformInput
|
|
70
70
|
? middleware.transformInput(input)
|
|
71
71
|
: input;
|
|
72
|
-
const evalOutput = middleware
|
|
72
|
+
const evalOutput = middleware?.transformOutput
|
|
73
73
|
? middleware.transformOutput(output)
|
|
74
74
|
: output;
|
|
75
75
|
return config.handler(ctx, evalInput, evalOutput, currentOptions);
|
package/dist/_utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"_utils.js","sourceRoot":"","sources":["../src/_utils.ts"],"names":[],"mappings":"AACA,OAAO,EACN,uBAAuB,GAIvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,CAAC,EAAE,MAAM,mBAAmB,CAAC;AACtC,OAAO,EAAE,YAAY,EAAsB,MAAM,IAAI,CAAC;AAEtD,sEAAsE;AACtE,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC,CAAC;AAIH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,SAAiC;IACpF,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,CACtC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,GAAG,IAAI,EAAE,KAAK,CAAC,EAChE,QAAQ,CACR,CAAC;AACH,CAAC;AAQD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAkC;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAElD,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAErD,IAAI,CAAC;YACJ,4DAA4D;YAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3F,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,OAAO,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,8BAA8B;YAC9B,IAAI,OAAO,KAAK,UAAU,GAAG,CAAC;gBAAE,MAAM;QACvC,CAAC;IACF,CAAC;IAED,MAAM,IAAI,KAAK,CACd,8CAA8C,UAAU,cAAc,SAAS,EAAE,OAAO,EAAE,CAC1F,CAAC;AACH,CAAC;AA4BD,MAAM,UAAU,gBAAgB,CAI9B,MAUD;IAcA,OAAO,CAAC,SAAS,EAAE,EAAE;QACpB,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,eAAe,EAAE,GAAG,SAAS,IAAI,EAAE,CAAC;QAC9E,MAAM,cAAc,GAAG,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,eAAe,EAAc,CAAC;QAE7E,OAAO;YACN,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,IAAI;YACzB,WAAW,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;YAC9C,8DAA8D;YAC9D,OAAO,EAAE,CAAC,KAAK,EAAE,GAAgB,EAAE,KAAU,EAAE,MAAW,EAAE,EAAE;gBAC7D,MAAM,SAAS,GAAG,UAAU;
|
|
1
|
+
{"version":3,"file":"_utils.js","sourceRoot":"","sources":["../src/_utils.ts"],"names":[],"mappings":"AACA,OAAO,EACN,uBAAuB,GAIvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,CAAC,EAAE,MAAM,mBAAmB,CAAC;AACtC,OAAO,EAAE,YAAY,EAAsB,MAAM,IAAI,CAAC;AAEtD,sEAAsE;AACtE,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC,CAAC;AAIH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,SAAiC;IACpF,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,CACtC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,GAAG,IAAI,EAAE,KAAK,CAAC,EAChE,QAAQ,CACR,CAAC;AACH,CAAC;AAQD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAkC;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAElD,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAErD,IAAI,CAAC;YACJ,4DAA4D;YAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3F,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,OAAO,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,8BAA8B;YAC9B,IAAI,OAAO,KAAK,UAAU,GAAG,CAAC;gBAAE,MAAM;QACvC,CAAC;IACF,CAAC;IAED,MAAM,IAAI,KAAK,CACd,8CAA8C,UAAU,cAAc,SAAS,EAAE,OAAO,EAAE,CAC1F,CAAC;AACH,CAAC;AA4BD,MAAM,UAAU,gBAAgB,CAI9B,MAUD;IAcA,OAAO,CAAC,SAAS,EAAE,EAAE;QACpB,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,eAAe,EAAE,GAAG,SAAS,IAAI,EAAE,CAAC;QAC9E,MAAM,cAAc,GAAG,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,eAAe,EAAc,CAAC;QAE7E,OAAO;YACN,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,IAAI;YACzB,WAAW,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;YAC9C,8DAA8D;YAC9D,OAAO,EAAE,CAAC,KAAK,EAAE,GAAgB,EAAE,KAAU,EAAE,MAAW,EAAE,EAAE;gBAC7D,MAAM,SAAS,GAAG,UAAU,EAAE,cAAc;oBAC3C,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC;oBAClC,CAAC,CAAE,KAAuC,CAAC;gBAC5C,MAAM,UAAU,GAAG,UAAU,EAAE,eAAe;oBAC7C,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC;oBACpC,CAAC,CAAE,MAAyC,CAAC;gBAC9C,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;gBAClE,8DAA8D;YAC/D,CAAC,CAAQ;YACT,OAAO,EAAE,cAAc;SACvB,CAAC;IACH,CAAC,CAAC;AACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export type BaseEvalOptions = {
|
|
|
6
6
|
/**
|
|
7
7
|
* Middleware to transform agent input/output to preset eval input/output.
|
|
8
8
|
* Allows reusing preset evals across agents with different schemas.
|
|
9
|
+
* At least one transform must be provided.
|
|
9
10
|
*
|
|
10
11
|
* @template TAgentInput - Agent's input type (inferred from generics or `any`)
|
|
11
12
|
* @template TAgentOutput - Agent's output type (inferred from generics or `any`)
|
|
@@ -14,6 +15,9 @@ export type BaseEvalOptions = {
|
|
|
14
15
|
*/
|
|
15
16
|
export type EvalMiddleware<TAgentInput, TAgentOutput, TEvalInput, TEvalOutput> = {
|
|
16
17
|
transformInput: (agentInput: TAgentInput) => TEvalInput;
|
|
18
|
+
transformOutput?: (agentOutput: TAgentOutput) => TEvalOutput;
|
|
19
|
+
} | {
|
|
20
|
+
transformInput?: (agentInput: TAgentInput) => TEvalInput;
|
|
17
21
|
transformOutput: (agentOutput: TAgentOutput) => TEvalOutput;
|
|
18
22
|
};
|
|
19
23
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,eAAO,MAAM,kBAAkB,EAAE,aAA0C,CAAC;AAE5E,MAAM,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,aAAa,CAAC;CACrB,CAAC;AAEF
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,eAAO,MAAM,kBAAkB,EAAE,aAA0C,CAAC;AAE5E,MAAM,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,aAAa,CAAC;CACrB,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,IAC1E;IACA,cAAc,EAAE,CAAC,UAAU,EAAE,WAAW,KAAK,UAAU,CAAC;IACxD,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,YAAY,KAAK,WAAW,CAAC;CAC5D,GACD;IACA,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,WAAW,KAAK,UAAU,CAAC;IACzD,eAAe,EAAE,CAAC,WAAW,EAAE,YAAY,KAAK,WAAW,CAAC;CAC3D,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentuity/evals",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.25",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"author": "Agentuity employees and contributors",
|
|
6
6
|
"type": "module",
|
|
@@ -26,14 +26,14 @@
|
|
|
26
26
|
"prepublishOnly": "bun run clean && bun run build"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"@agentuity/core": "0.1.
|
|
30
|
-
"@agentuity/runtime": "0.1.
|
|
31
|
-
"@agentuity/schema": "0.1.
|
|
29
|
+
"@agentuity/core": "0.1.25",
|
|
30
|
+
"@agentuity/runtime": "0.1.25",
|
|
31
|
+
"@agentuity/schema": "0.1.25",
|
|
32
32
|
"@ai-sdk/groq": "latest",
|
|
33
33
|
"ai": "^6.0.0"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
|
-
"@agentuity/test-utils": "0.1.
|
|
36
|
+
"@agentuity/test-utils": "0.1.25",
|
|
37
37
|
"@types/bun": "latest",
|
|
38
38
|
"bun-types": "latest",
|
|
39
39
|
"typescript": "^5.9.0"
|
package/src/_utils.ts
CHANGED
|
@@ -150,10 +150,10 @@ export function createPresetEval<
|
|
|
150
150
|
description: description ?? config.description,
|
|
151
151
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
152
152
|
handler: (async (ctx: EvalContext, input: any, output: any) => {
|
|
153
|
-
const evalInput = middleware
|
|
153
|
+
const evalInput = middleware?.transformInput
|
|
154
154
|
? middleware.transformInput(input)
|
|
155
155
|
: (input as InferSchemaOutput<TEvalInput>);
|
|
156
|
-
const evalOutput = middleware
|
|
156
|
+
const evalOutput = middleware?.transformOutput
|
|
157
157
|
? middleware.transformOutput(output)
|
|
158
158
|
: (output as InferSchemaOutput<TEvalOutput>);
|
|
159
159
|
return config.handler(ctx, evalInput, evalOutput, currentOptions);
|
package/src/types.ts
CHANGED
|
@@ -10,13 +10,19 @@ export type BaseEvalOptions = {
|
|
|
10
10
|
/**
|
|
11
11
|
* Middleware to transform agent input/output to preset eval input/output.
|
|
12
12
|
* Allows reusing preset evals across agents with different schemas.
|
|
13
|
+
* At least one transform must be provided.
|
|
13
14
|
*
|
|
14
15
|
* @template TAgentInput - Agent's input type (inferred from generics or `any`)
|
|
15
16
|
* @template TAgentOutput - Agent's output type (inferred from generics or `any`)
|
|
16
17
|
* @template TEvalInput - Eval's expected input type
|
|
17
18
|
* @template TEvalOutput - Eval's expected output type
|
|
18
19
|
*/
|
|
19
|
-
export type EvalMiddleware<TAgentInput, TAgentOutput, TEvalInput, TEvalOutput> =
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
export type EvalMiddleware<TAgentInput, TAgentOutput, TEvalInput, TEvalOutput> =
|
|
21
|
+
| {
|
|
22
|
+
transformInput: (agentInput: TAgentInput) => TEvalInput;
|
|
23
|
+
transformOutput?: (agentOutput: TAgentOutput) => TEvalOutput;
|
|
24
|
+
}
|
|
25
|
+
| {
|
|
26
|
+
transformInput?: (agentInput: TAgentInput) => TEvalInput;
|
|
27
|
+
transformOutput: (agentOutput: TAgentOutput) => TEvalOutput;
|
|
28
|
+
};
|