@dalmia/calibrate-mcp 0.0.40 → 0.0.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mcp-server.js +1218 -682
- package/bin/mcp-server.js.map +35 -33
- package/esm/funcs/agentTestsListRunsForAgent.js +1 -0
- package/esm/funcs/agentTestsListRunsForAgent.js.map +1 -1
- package/esm/landing-page.js +1 -1
- package/esm/lib/config.d.ts +2 -2
- package/esm/lib/config.js +2 -2
- package/esm/mcp-server/mcp-server.js +1 -1
- package/esm/mcp-server/server.js +1 -1
- package/esm/models/agentcreate.d.ts +26 -0
- package/esm/models/agentcreate.d.ts.map +1 -1
- package/esm/models/agentcreate.js +18 -0
- package/esm/models/agentcreate.js.map +1 -1
- package/esm/models/agentresponse.d.ts +26 -0
- package/esm/models/agentresponse.d.ts.map +1 -1
- package/esm/models/agentresponse.js +17 -0
- package/esm/models/agentresponse.js.map +1 -1
- package/esm/models/agentsummary.d.ts +26 -0
- package/esm/models/agentsummary.d.ts.map +1 -1
- package/esm/models/agentsummary.js +17 -0
- package/esm/models/agentsummary.js.map +1 -1
- package/esm/models/agenttestrunlistitem.d.ts +2 -0
- package/esm/models/agenttestrunlistitem.d.ts.map +1 -1
- package/esm/models/agenttestrunlistitem.js +2 -0
- package/esm/models/agenttestrunlistitem.js.map +1 -1
- package/esm/models/annotationtaskresponse.js +1 -1
- package/esm/models/annotationtaskresponse.js.map +1 -1
- package/esm/models/bulktestitem.d.ts +2 -1
- package/esm/models/bulktestitem.d.ts.map +1 -1
- package/esm/models/bulktestitem.js +4 -2
- package/esm/models/bulktestitem.js.map +1 -1
- package/esm/models/bulktestupload.d.ts +4 -0
- package/esm/models/bulktestupload.d.ts.map +1 -1
- package/esm/models/bulktestupload.js +5 -2
- package/esm/models/bulktestupload.js.map +1 -1
- package/esm/models/evaluatorcreate.d.ts +4 -0
- package/esm/models/evaluatorcreate.d.ts.map +1 -1
- package/esm/models/evaluatorcreate.js +5 -2
- package/esm/models/evaluatorcreate.js.map +1 -1
- package/esm/models/evaluatordetailresponsecompact.d.ts +5 -0
- package/esm/models/evaluatordetailresponsecompact.d.ts.map +1 -1
- package/esm/models/evaluatordetailresponsecompact.js +6 -2
- package/esm/models/evaluatordetailresponsecompact.js.map +1 -1
- package/esm/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.d.ts +1 -0
- package/esm/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.d.ts.map +1 -1
- package/esm/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.js +1 -0
- package/esm/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.js.map +1 -1
- package/esm/models/listevaluatorsevaluatorsgetop.d.ts +2 -0
- package/esm/models/listevaluatorsevaluatorsgetop.d.ts.map +1 -1
- package/esm/models/listevaluatorsevaluatorsgetop.js +2 -0
- package/esm/models/listevaluatorsevaluatorsgetop.js.map +1 -1
- package/esm/models/routersevaluatorsevaluatorresponse.d.ts +5 -0
- package/esm/models/routersevaluatorsevaluatorresponse.d.ts.map +1 -1
- package/esm/models/routersevaluatorsevaluatorresponse.js +6 -2
- package/esm/models/routersevaluatorsevaluatorresponse.js.map +1 -1
- package/esm/models/testcreate.d.ts +4 -0
- package/esm/models/testcreate.d.ts.map +1 -1
- package/esm/models/testcreate.js +6 -3
- package/esm/models/testcreate.js.map +1 -1
- package/esm/models/testlistresponse.d.ts +4 -0
- package/esm/models/testlistresponse.d.ts.map +1 -1
- package/esm/models/testlistresponse.js +5 -2
- package/esm/models/testlistresponse.js.map +1 -1
- package/esm/models/testresponse.d.ts +4 -0
- package/esm/models/testresponse.d.ts.map +1 -1
- package/esm/models/testresponse.js +5 -2
- package/esm/models/testresponse.js.map +1 -1
- package/esm/models/testupdate.d.ts +2 -0
- package/esm/models/testupdate.d.ts.map +1 -1
- package/esm/models/testupdate.js +4 -2
- package/esm/models/testupdate.js.map +1 -1
- package/esm/models/traceingest.d.ts +6 -1
- package/esm/models/traceingest.d.ts.map +1 -1
- package/esm/models/traceingest.js +8 -1
- package/esm/models/traceingest.js.map +1 -1
- package/esm/models/tracetoolcall.d.ts +1 -0
- package/esm/models/tracetoolcall.d.ts.map +1 -1
- package/esm/models/tracetoolcall.js +1 -0
- package/esm/models/tracetoolcall.js.map +1 -1
- package/package.json +1 -1
- package/src/funcs/agentTestsListRunsForAgent.ts +1 -0
- package/src/landing-page.ts +1 -1
- package/src/lib/config.ts +2 -2
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +1 -1
- package/src/models/agentcreate.ts +36 -0
- package/src/models/agentresponse.ts +35 -0
- package/src/models/agentsummary.ts +35 -0
- package/src/models/agenttestrunlistitem.ts +6 -0
- package/src/models/annotationtaskresponse.ts +1 -1
- package/src/models/bulktestitem.ts +10 -5
- package/src/models/bulktestupload.ts +6 -2
- package/src/models/evaluatorcreate.ts +6 -2
- package/src/models/evaluatordetailresponsecompact.ts +10 -2
- package/src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts +4 -0
- package/src/models/listevaluatorsevaluatorsgetop.ts +2 -0
- package/src/models/routersevaluatorsevaluatorresponse.ts +10 -2
- package/src/models/testcreate.ts +7 -3
- package/src/models/testlistresponse.ts +6 -2
- package/src/models/testresponse.ts +6 -2
- package/src/models/testupdate.ts +4 -2
- package/src/models/traceingest.ts +18 -3
- package/src/models/tracetoolcall.ts +4 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluatorcreate.js","sourceRoot":"","sources":["../../src/models/evaluatorcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,gCAAgC,GACjC,MAAM,6BAA6B,CAAC;AAErC
|
|
1
|
+
{"version":3,"file":"evaluatorcreate.js","sourceRoot":"","sources":["../../src/models/evaluatorcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,gCAAgC,GACjC,MAAM,6BAA6B,CAAC;AAErC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG;IAC1C,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,WAAW;CACb,CAAC;AAiBX,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,CAAC,IAAI,CAAC;IAC3D,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,WAAW;CACZ,CAAC,CAAC,QAAQ,CACT,uSAAuS,CACxS,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,CAAC,IAAI,CAAC;IACtD,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAC,CAAC,IAAI,CAAC;IACxD,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAWF,MAAM,CAAC,MAAM,yBAAyB,GAA+B,CAAC,CAAC,MAAM,CAAC;IAC5E,SAAS,EAAE,iCAAiC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CACnE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,kCAAkC,CACnC;IACD,cAAc,EAAE,sCAAsC,CAAC,OAAO,CAAC,KAAK,CAAC;SAClE,QAAQ,CACP,uSAAuS,CACxS;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;IACzE,WAAW,EAAE,mCAAmC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,QAAQ,CACzE,0HAA0H,CAC3H;IACD,OAAO,EAAE,gCAAgC,CAAC,QAAQ,CAChD,8EAA8E,CAC/E;CACF,CAAC,CAAC"}
|
|
@@ -11,6 +11,7 @@ import { EvaluatorVersionCompact } from "./evaluatorversioncompact.js";
|
|
|
11
11
|
* - `llm`: a reply with its conversation history
|
|
12
12
|
* - `llm-general`: a standalone input and output pair
|
|
13
13
|
* - `conversation`: a full conversation
|
|
14
|
+
* - `tool-call`: whether the agent called the right tool, labelled by a person
|
|
14
15
|
*/
|
|
15
16
|
export declare const EvaluatorDetailResponseCompactEvaluatorType: {
|
|
16
17
|
readonly Tts: "tts";
|
|
@@ -18,6 +19,7 @@ export declare const EvaluatorDetailResponseCompactEvaluatorType: {
|
|
|
18
19
|
readonly Llm: "llm";
|
|
19
20
|
readonly LlmGeneral: "llm-general";
|
|
20
21
|
readonly Conversation: "conversation";
|
|
22
|
+
readonly ToolCall: "tool-call";
|
|
21
23
|
};
|
|
22
24
|
/**
|
|
23
25
|
* What the evaluator judges:
|
|
@@ -29,6 +31,7 @@ export declare const EvaluatorDetailResponseCompactEvaluatorType: {
|
|
|
29
31
|
* - `llm`: a reply with its conversation history
|
|
30
32
|
* - `llm-general`: a standalone input and output pair
|
|
31
33
|
* - `conversation`: a full conversation
|
|
34
|
+
* - `tool-call`: whether the agent called the right tool, labelled by a person
|
|
32
35
|
*/
|
|
33
36
|
export type EvaluatorDetailResponseCompactEvaluatorType = ClosedEnum<typeof EvaluatorDetailResponseCompactEvaluatorType>;
|
|
34
37
|
export declare const EvaluatorDetailResponseCompactEvaluatorType$zodSchema: z.ZodEnum<{
|
|
@@ -37,6 +40,7 @@ export declare const EvaluatorDetailResponseCompactEvaluatorType$zodSchema: z.Zo
|
|
|
37
40
|
stt: "stt";
|
|
38
41
|
llm: "llm";
|
|
39
42
|
"llm-general": "llm-general";
|
|
43
|
+
"tool-call": "tool-call";
|
|
40
44
|
}>;
|
|
41
45
|
/**
|
|
42
46
|
* The modality the judge reads:
|
|
@@ -96,6 +100,7 @@ export type EvaluatorDetailResponseCompact = {
|
|
|
96
100
|
data_type: EvaluatorDetailResponseCompactDataType;
|
|
97
101
|
output_type: EvaluatorDetailResponseCompactOutputType;
|
|
98
102
|
is_default: boolean;
|
|
103
|
+
is_protected: boolean;
|
|
99
104
|
slug?: string | null | undefined;
|
|
100
105
|
source_default_slug?: string | null | undefined;
|
|
101
106
|
live_version_id?: string | null | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluatordetailresponsecompact.d.ts","sourceRoot":"","sources":["../../src/models/evaluatordetailresponsecompact.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,uBAAuB,EAExB,MAAM,8BAA8B,CAAC;AAEtC
|
|
1
|
+
{"version":3,"file":"evaluatordetailresponsecompact.d.ts","sourceRoot":"","sources":["../../src/models/evaluatordetailresponsecompact.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,uBAAuB,EAExB,MAAM,8BAA8B,CAAC;AAEtC;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,2CAA2C;;;;;;;CAO9C,CAAC;AACX;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,2CAA2C,GAAG,UAAU,CAClE,OAAO,2CAA2C,CACnD,CAAC;AAEF,eAAO,MAAM,qDAAqD;;;;;;;EASjE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,sCAAsC;;;CAGzC,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,sCAAsC,GAAG,UAAU,CAC7D,OAAO,sCAAsC,CAC9C,CAAC;AAEF,eAAO,MAAM,gDAAgD;;;EAGQ,CAAC;AAEtE;;;;;;;GAOG;AACH,eAAO,MAAM,wCAAwC;;;CAG3C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,wCAAwC,GAAG,UAAU,CAC/D,OAAO,wCAAwC,CAChD,CAAC;AAEF,eAAO,MAAM,kDAAkD;;;EAK9D,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,cAAc,EAAE,2CAA2C,CAAC;IAC5D,SAAS,EAAE,sCAAsC,CAAC;IAClD,WAAW,EAAE,wCAAwC,CAAC;IACtD,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC;CAC1C,CAAC;AAEF,eAAO,MAAM,wCAAwC,EAAE,CAAC,CAAC,OAAO,CAC9D,8BAA8B,CAyC9B,CAAC"}
|
|
@@ -13,6 +13,7 @@ import { EvaluatorVersionCompact$zodSchema, } from "./evaluatorversioncompact.js
|
|
|
13
13
|
* - `llm`: a reply with its conversation history
|
|
14
14
|
* - `llm-general`: a standalone input and output pair
|
|
15
15
|
* - `conversation`: a full conversation
|
|
16
|
+
* - `tool-call`: whether the agent called the right tool, labelled by a person
|
|
16
17
|
*/
|
|
17
18
|
export const EvaluatorDetailResponseCompactEvaluatorType = {
|
|
18
19
|
Tts: "tts",
|
|
@@ -20,6 +21,7 @@ export const EvaluatorDetailResponseCompactEvaluatorType = {
|
|
|
20
21
|
Llm: "llm",
|
|
21
22
|
LlmGeneral: "llm-general",
|
|
22
23
|
Conversation: "conversation",
|
|
24
|
+
ToolCall: "tool-call",
|
|
23
25
|
};
|
|
24
26
|
export const EvaluatorDetailResponseCompactEvaluatorType$zodSchema = z.enum([
|
|
25
27
|
"tts",
|
|
@@ -27,7 +29,8 @@ export const EvaluatorDetailResponseCompactEvaluatorType$zodSchema = z.enum([
|
|
|
27
29
|
"llm",
|
|
28
30
|
"llm-general",
|
|
29
31
|
"conversation",
|
|
30
|
-
|
|
32
|
+
"tool-call",
|
|
33
|
+
]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n");
|
|
31
34
|
/**
|
|
32
35
|
* The modality the judge reads:
|
|
33
36
|
*
|
|
@@ -65,8 +68,9 @@ export const EvaluatorDetailResponseCompact$zodSchema = z.object({
|
|
|
65
68
|
data_type: EvaluatorDetailResponseCompactDataType$zodSchema.describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
|
|
66
69
|
description: z.string().nullable().optional().describe("What the evaluator checks"),
|
|
67
70
|
evaluator_type: EvaluatorDetailResponseCompactEvaluatorType$zodSchema
|
|
68
|
-
.describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
|
|
71
|
+
.describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n"),
|
|
69
72
|
is_default: z.boolean().describe("True when the evaluator is a built-in default or your workspace's editable copy of one. False for an evaluator you created yourself"),
|
|
73
|
+
is_protected: z.boolean().describe("True when the evaluator is locked. A locked evaluator cannot be deleted, and only its name, description and rubric can change"),
|
|
70
74
|
live_version_id: z.string().nullable().optional().describe("ID of the version that is currently live"),
|
|
71
75
|
name: z.string().describe("Evaluator name"),
|
|
72
76
|
output_type: EvaluatorDetailResponseCompactOutputType$zodSchema.describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n"),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluatordetailresponsecompact.js","sourceRoot":"","sources":["../../src/models/evaluatordetailresponsecompact.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,iCAAiC,GAClC,MAAM,8BAA8B,CAAC;AAEtC
|
|
1
|
+
{"version":3,"file":"evaluatordetailresponsecompact.js","sourceRoot":"","sources":["../../src/models/evaluatordetailresponsecompact.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,iCAAiC,GAClC,MAAM,8BAA8B,CAAC;AAEtC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,2CAA2C,GAAG;IACzD,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,WAAW;CACb,CAAC;AAiBX,MAAM,CAAC,MAAM,qDAAqD,GAAG,CAAC,CAAC,IAAI,CAAC;IAC1E,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,WAAW;CACZ,CAAC,CAAC,QAAQ,CACT,uSAAuS,CACxS,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAAG;IACpD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,gDAAgD,GAAG,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG;IACtD,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,kDAAkD,GAAG,CAAC,CAAC,IAAI,CAAC;IACvE,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAmBF,MAAM,CAAC,MAAM,wCAAwC,GAEjD,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,SAAS,EAAE,gDAAgD,CAAC,QAAQ,CAClE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,2BAA2B,CAC5B;IACD,cAAc,EAAE,qDAAqD;SAClE,QAAQ,CACP,uSAAuS,CACxS;IACH,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAC9B,qIAAqI,CACtI;IACD,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAChC,+HAA+H,CAChI;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,0CAA0C,CAC3C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC3C,WAAW,EAAE,kDAAkD,CAAC,QAAQ,CACtE,0HAA0H,CAC3H;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,8CAA8C,CAC/C;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,qJAAqJ,CACtJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,oDAAoD,CACrD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;IACzC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,QAAQ,CAC3D,oCAAoC,CACrC;CACF,CAAC,CAAC"}
|
|
@@ -31,6 +31,7 @@ export type GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest = {
|
|
|
31
31
|
type?: GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetType | null | undefined;
|
|
32
32
|
status?: TaskStatus | null | undefined;
|
|
33
33
|
has_failures?: boolean | null | undefined;
|
|
34
|
+
around?: string | null | undefined;
|
|
34
35
|
limit?: number | null | undefined;
|
|
35
36
|
offset?: number | undefined;
|
|
36
37
|
xAPIKey?: string | null | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getagenttestrunsagenttestsagentagentuuidrunsgetop.d.ts","sourceRoot":"","sources":["../../src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,qCAAqC,EAEtC,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AAEnE;;;;;;GAMG;AACH,eAAO,MAAM,mDAAmD;;;CAGtD,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,mDAAmD,GAAG,UAAU,CAC1E,OAAO,mDAAmD,CAC3D,CAAC;AAEF,eAAO,MAAM,6DAA6D;;;EAMvE,CAAC;AAEJ,MAAM,MAAM,sDAAsD,GAAG;IACnE,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,mDAAmD,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9E,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,YAAY,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1C,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,gEAAgE,EAC3E,CAAC,CAAC,OAAO,CAAC,sDAAsD,
|
|
1
|
+
{"version":3,"file":"getagenttestrunsagenttestsagentagentuuidrunsgetop.d.ts","sourceRoot":"","sources":["../../src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,qCAAqC,EAEtC,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AAEnE;;;;;;GAMG;AACH,eAAO,MAAM,mDAAmD;;;CAGtD,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,mDAAmD,GAAG,UAAU,CAC1E,OAAO,mDAAmD,CAC3D,CAAC;AAEF,eAAO,MAAM,6DAA6D;;;EAMvE,CAAC;AAEJ,MAAM,MAAM,sDAAsD,GAAG;IACnE,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,mDAAmD,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9E,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,YAAY,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1C,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,gEAAgE,EAC3E,CAAC,CAAC,OAAO,CAAC,sDAAsD,CAsB9D,CAAC;AAEL,MAAM,MAAM,uDAAuD,GAC/D,qCAAqC,GACrC,mBAAmB,CAAC;AAExB,eAAO,MAAM,iEAAiE,EAC5E,CAAC,CAAC,OAAO,CAAC,uDAAuD,CAG/D,CAAC"}
|
|
@@ -23,6 +23,7 @@ export const GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetType$zodSchema = z
|
|
|
23
23
|
]).describe("Filter by run type. Omit to return both:\n- `llm-unit-test`: single runs of an agent's tests\n- `llm-benchmark`: multi-model comparisons");
|
|
24
24
|
export const GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest$zodSchema = z.object({
|
|
25
25
|
agent_uuid: z.string().describe("Agent whose test runs to list"),
|
|
26
|
+
around: z.string().describe("ID of a run to jump to, returning the page that contains it instead of the page at `offset`").nullable().optional(),
|
|
26
27
|
has_failures: z.boolean().describe("Filter by whether the run has any failing test case or model. `true` returns only runs with failures (or errors), `false` only clean runs. Omit for both").nullable().optional(),
|
|
27
28
|
limit: z.int().describe("Maximum number of items to return. Omit for no limit (all items)").nullable().optional(),
|
|
28
29
|
offset: z.int().default(0).describe("Number of items to skip before returning results"),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getagenttestrunsagenttestsagentagentuuidrunsgetop.js","sourceRoot":"","sources":["../../src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,+CAA+C,GAChD,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mDAAmD,GAAG;IACjE,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,6DAA6D,GAAG,CAAC;KAC3E,IAAI,CAAC;IACJ,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,0IAA0I,CAC3I,CAAC;
|
|
1
|
+
{"version":3,"file":"getagenttestrunsagenttestsagentagentuuidrunsgetop.js","sourceRoot":"","sources":["../../src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,+CAA+C,GAChD,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mDAAmD,GAAG;IACjE,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,6DAA6D,GAAG,CAAC;KAC3E,IAAI,CAAC;IACJ,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,0IAA0I,CAC3I,CAAC;AAaJ,MAAM,CAAC,MAAM,gEAAgE,GACP,CAAC,CAAC,MAAM,CAAC;IAC3E,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IAChE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACzB,6FAA6F,CAC9F,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAChC,0JAA0J,CAC3J,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CACrB,kEAAkE,CACnE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CACjC,kDAAkD,CACnD;IACD,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,6CAA6C,CAC9C;IACD,IAAI,EAAE,6DAA6D;SAChE,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7B,0IAA0I,CAC3I;IACH,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAML,MAAM,CAAC,MAAM,iEAAiE,GACP,CAAC,CAAC,KAAK,CAAC;IAC3E,+CAA+C;IAC/C,6BAA6B;CAC9B,CAAC,CAAC"}
|
|
@@ -11,6 +11,7 @@ export declare const ListEvaluatorsEvaluatorsGetEvaluatorType: {
|
|
|
11
11
|
readonly Llm: "llm";
|
|
12
12
|
readonly LlmGeneral: "llm-general";
|
|
13
13
|
readonly Conversation: "conversation";
|
|
14
|
+
readonly ToolCall: "tool-call";
|
|
14
15
|
};
|
|
15
16
|
/**
|
|
16
17
|
* Filter by what the evaluator judges. Omit for all types
|
|
@@ -22,6 +23,7 @@ export declare const ListEvaluatorsEvaluatorsGetEvaluatorType$zodSchema: z.ZodEn
|
|
|
22
23
|
stt: "stt";
|
|
23
24
|
llm: "llm";
|
|
24
25
|
"llm-general": "llm-general";
|
|
26
|
+
"tool-call": "tool-call";
|
|
25
27
|
}>;
|
|
26
28
|
/**
|
|
27
29
|
* Filter by modality. Omit for all
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"listevaluatorsevaluatorsgetop.d.ts","sourceRoot":"","sources":["../../src/models/listevaluatorsevaluatorsgetop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kCAAkC,EAEnC,MAAM,yCAAyC,CAAC;AAEjD;;GAEG;AACH,eAAO,MAAM,wCAAwC
|
|
1
|
+
{"version":3,"file":"listevaluatorsevaluatorsgetop.d.ts","sourceRoot":"","sources":["../../src/models/listevaluatorsevaluatorsgetop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kCAAkC,EAEnC,MAAM,yCAAyC,CAAC;AAEjD;;GAEG;AACH,eAAO,MAAM,wCAAwC;;;;;;;CAO3C,CAAC;AACX;;GAEG;AACH,MAAM,MAAM,wCAAwC,GAAG,UAAU,CAC/D,OAAO,wCAAwC,CAChD,CAAC;AAEF,eAAO,MAAM,kDAAkD;;;;;;;EAOO,CAAC;AAEvE;;GAEG;AACH,eAAO,MAAM,mCAAmC;;;CAGtC,CAAC;AACX;;GAEG;AACH,MAAM,MAAM,mCAAmC,GAAG,UAAU,CAC1D,OAAO,mCAAmC,CAC3C,CAAC;AAEF,eAAO,MAAM,6CAA6C;;;EAGX,CAAC;AAEhD,MAAM,MAAM,kCAAkC,GAAG;IAC/C,cAAc,CAAC,EAAE,wCAAwC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7E,SAAS,CAAC,EAAE,mCAAmC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnE,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4CAA4C,EAAE,CAAC,CAAC,OAAO,CAClE,kCAAkC,CAqBlC,CAAC;AAEH,MAAM,MAAM,mCAAmC,GAC3C,kCAAkC,GAClC,mBAAmB,CAAC;AAExB,eAAO,MAAM,6CAA6C,EAAE,CAAC,CAAC,OAAO,CACnE,mCAAmC,CAInC,CAAC"}
|
|
@@ -13,6 +13,7 @@ export const ListEvaluatorsEvaluatorsGetEvaluatorType = {
|
|
|
13
13
|
Llm: "llm",
|
|
14
14
|
LlmGeneral: "llm-general",
|
|
15
15
|
Conversation: "conversation",
|
|
16
|
+
ToolCall: "tool-call",
|
|
16
17
|
};
|
|
17
18
|
export const ListEvaluatorsEvaluatorsGetEvaluatorType$zodSchema = z.enum([
|
|
18
19
|
"tts",
|
|
@@ -20,6 +21,7 @@ export const ListEvaluatorsEvaluatorsGetEvaluatorType$zodSchema = z.enum([
|
|
|
20
21
|
"llm",
|
|
21
22
|
"llm-general",
|
|
22
23
|
"conversation",
|
|
24
|
+
"tool-call",
|
|
23
25
|
]).describe("Filter by what the evaluator judges. Omit for all types");
|
|
24
26
|
/**
|
|
25
27
|
* Filter by modality. Omit for all
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"listevaluatorsevaluatorsgetop.js","sourceRoot":"","sources":["../../src/models/listevaluatorsevaluatorsgetop.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,4CAA4C,GAC7C,MAAM,yCAAyC,CAAC;AAEjD;;GAEG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG;IACtD,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;
|
|
1
|
+
{"version":3,"file":"listevaluatorsevaluatorsgetop.js","sourceRoot":"","sources":["../../src/models/listevaluatorsevaluatorsgetop.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,4CAA4C,GAC7C,MAAM,yCAAyC,CAAC;AAEjD;;GAEG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG;IACtD,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,WAAW;CACb,CAAC;AAQX,MAAM,CAAC,MAAM,kDAAkD,GAAG,CAAC,CAAC,IAAI,CAAC;IACvE,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,WAAW;CACZ,CAAC,CAAC,QAAQ,CAAC,yDAAyD,CAAC,CAAC;AAEvE;;GAEG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG;IACjD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAQX,MAAM,CAAC,MAAM,6CAA6C,GAAG,CAAC,CAAC,IAAI,CAAC;IAClE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,kCAAkC,CAAC,CAAC;AAYhD,MAAM,CAAC,MAAM,4CAA4C,GAErD,CAAC,CAAC,MAAM,CAAC;IACX,SAAS,EAAE,6CAA6C,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC3E,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,cAAc,EAAE,kDAAkD,CAAC,QAAQ,EAAE;SAC1E,QAAQ,EAAE,CAAC,QAAQ,CAClB,yDAAyD,CAC1D;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAClD,iJAAiJ,CAClJ;IACD,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CACrB,kEAAkE,CACnE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CACjC,kDAAkD,CACnD;IACD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACpB,+DAA+D,CAChE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAMH,MAAM,CAAC,MAAM,6CAA6C,GAEtD,CAAC,CAAC,KAAK,CAAC;IACV,4CAA4C;IAC5C,6BAA6B;CAC9B,CAAC,CAAC"}
|
|
@@ -11,6 +11,7 @@ import { EvaluatorLiveVersionSummary } from "./evaluatorliveversionsummary.js";
|
|
|
11
11
|
* - `llm`: a reply with its conversation history
|
|
12
12
|
* - `llm-general`: a standalone input and output pair
|
|
13
13
|
* - `conversation`: a full conversation
|
|
14
|
+
* - `tool-call`: whether the agent called the right tool, labelled by a person
|
|
14
15
|
*/
|
|
15
16
|
export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType: {
|
|
16
17
|
readonly Tts: "tts";
|
|
@@ -18,6 +19,7 @@ export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType: {
|
|
|
18
19
|
readonly Llm: "llm";
|
|
19
20
|
readonly LlmGeneral: "llm-general";
|
|
20
21
|
readonly Conversation: "conversation";
|
|
22
|
+
readonly ToolCall: "tool-call";
|
|
21
23
|
};
|
|
22
24
|
/**
|
|
23
25
|
* What the evaluator judges:
|
|
@@ -29,6 +31,7 @@ export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType: {
|
|
|
29
31
|
* - `llm`: a reply with its conversation history
|
|
30
32
|
* - `llm-general`: a standalone input and output pair
|
|
31
33
|
* - `conversation`: a full conversation
|
|
34
|
+
* - `tool-call`: whether the agent called the right tool, labelled by a person
|
|
32
35
|
*/
|
|
33
36
|
export type RoutersEvaluatorsEvaluatorResponseEvaluatorType = ClosedEnum<typeof RoutersEvaluatorsEvaluatorResponseEvaluatorType>;
|
|
34
37
|
export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema: z.ZodEnum<{
|
|
@@ -37,6 +40,7 @@ export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema:
|
|
|
37
40
|
stt: "stt";
|
|
38
41
|
llm: "llm";
|
|
39
42
|
"llm-general": "llm-general";
|
|
43
|
+
"tool-call": "tool-call";
|
|
40
44
|
}>;
|
|
41
45
|
/**
|
|
42
46
|
* The modality the judge reads:
|
|
@@ -96,6 +100,7 @@ export type RoutersEvaluatorsEvaluatorResponse = {
|
|
|
96
100
|
data_type: RoutersEvaluatorsEvaluatorResponseDataType;
|
|
97
101
|
output_type: RoutersEvaluatorsEvaluatorResponseOutputType;
|
|
98
102
|
is_default: boolean;
|
|
103
|
+
is_protected: boolean;
|
|
99
104
|
slug?: string | null | undefined;
|
|
100
105
|
source_default_slug?: string | null | undefined;
|
|
101
106
|
live_version_id?: string | null | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"routersevaluatorsevaluatorresponse.d.ts","sourceRoot":"","sources":["../../src/models/routersevaluatorsevaluatorresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,2BAA2B,EAE5B,MAAM,kCAAkC,CAAC;AAE1C
|
|
1
|
+
{"version":3,"file":"routersevaluatorsevaluatorresponse.d.ts","sourceRoot":"","sources":["../../src/models/routersevaluatorsevaluatorresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,2BAA2B,EAE5B,MAAM,kCAAkC,CAAC;AAE1C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,+CAA+C;;;;;;;CAOlD,CAAC;AACX;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,+CAA+C,GAAG,UAAU,CACtE,OAAO,+CAA+C,CACvD,CAAC;AAEF,eAAO,MAAM,yDAAyD;;;;;;;EAWrE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,0CAA0C;;;CAG7C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,0CAA0C,GAAG,UAAU,CACjE,OAAO,0CAA0C,CAClD,CAAC;AAEF,eAAO,MAAM,oDAAoD;;;EAGI,CAAC;AAEtE;;;;;;;GAOG;AACH,eAAO,MAAM,4CAA4C;;;CAG/C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,4CAA4C,GAAG,UAAU,CACnE,OAAO,4CAA4C,CACpD,CAAC;AAEF,eAAO,MAAM,sDAAsD;;;EAKlE,CAAC;AAEF,MAAM,MAAM,kCAAkC,GAAG;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,cAAc,EAAE,+CAA+C,CAAC;IAChE,SAAS,EAAE,0CAA0C,CAAC;IACtD,WAAW,EAAE,4CAA4C,CAAC;IAC1D,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,2BAA2B,GAAG,IAAI,GAAG,SAAS,CAAC;CAC/D,CAAC;AAEF,eAAO,MAAM,4CAA4C,EAAE,CAAC,CAAC,OAAO,CAClE,kCAAkC,CAwClC,CAAC"}
|
|
@@ -13,6 +13,7 @@ import { EvaluatorLiveVersionSummary$zodSchema, } from "./evaluatorliveversionsu
|
|
|
13
13
|
* - `llm`: a reply with its conversation history
|
|
14
14
|
* - `llm-general`: a standalone input and output pair
|
|
15
15
|
* - `conversation`: a full conversation
|
|
16
|
+
* - `tool-call`: whether the agent called the right tool, labelled by a person
|
|
16
17
|
*/
|
|
17
18
|
export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
|
|
18
19
|
Tts: "tts",
|
|
@@ -20,6 +21,7 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
|
|
|
20
21
|
Llm: "llm",
|
|
21
22
|
LlmGeneral: "llm-general",
|
|
22
23
|
Conversation: "conversation",
|
|
24
|
+
ToolCall: "tool-call",
|
|
23
25
|
};
|
|
24
26
|
export const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema = z.enum([
|
|
25
27
|
"tts",
|
|
@@ -27,7 +29,8 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema = z.enum(
|
|
|
27
29
|
"llm",
|
|
28
30
|
"llm-general",
|
|
29
31
|
"conversation",
|
|
30
|
-
|
|
32
|
+
"tool-call",
|
|
33
|
+
]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n");
|
|
31
34
|
/**
|
|
32
35
|
* The modality the judge reads:
|
|
33
36
|
*
|
|
@@ -65,8 +68,9 @@ export const RoutersEvaluatorsEvaluatorResponse$zodSchema = z.object({
|
|
|
65
68
|
data_type: RoutersEvaluatorsEvaluatorResponseDataType$zodSchema.describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
|
|
66
69
|
description: z.string().nullable().optional().describe("What the evaluator checks"),
|
|
67
70
|
evaluator_type: RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema
|
|
68
|
-
.describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
|
|
71
|
+
.describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n"),
|
|
69
72
|
is_default: z.boolean().describe("True when the evaluator is a built-in default or your workspace's editable copy of one. False for an evaluator you created yourself"),
|
|
73
|
+
is_protected: z.boolean().describe("True when the evaluator is locked. A locked evaluator cannot be deleted, and only its name, description and rubric can change"),
|
|
70
74
|
live_version: EvaluatorLiveVersionSummary$zodSchema.nullable().optional()
|
|
71
75
|
.describe("The version that is currently live"),
|
|
72
76
|
live_version_id: z.string().nullable().optional().describe("ID of the version that is currently live"),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"routersevaluatorsevaluatorresponse.js","sourceRoot":"","sources":["../../src/models/routersevaluatorsevaluatorresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,qCAAqC,GACtC,MAAM,kCAAkC,CAAC;AAE1C
|
|
1
|
+
{"version":3,"file":"routersevaluatorsevaluatorresponse.js","sourceRoot":"","sources":["../../src/models/routersevaluatorsevaluatorresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,qCAAqC,GACtC,MAAM,kCAAkC,CAAC;AAE1C;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,+CAA+C,GAAG;IAC7D,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,WAAW;CACb,CAAC;AAiBX,MAAM,CAAC,MAAM,yDAAyD,GAAG,CAAC,CAAC,IAAI,CAC7E;IACE,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,WAAW;CACZ,CACF,CAAC,QAAQ,CACR,uSAAuS,CACxS,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0CAA0C,GAAG;IACxD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,oDAAoD,GAAG,CAAC,CAAC,IAAI,CAAC;IACzE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,4CAA4C,GAAG;IAC1D,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,sDAAsD,GAAG,CAAC,CAAC,IAAI,CAAC;IAC3E,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAmBF,MAAM,CAAC,MAAM,4CAA4C,GAErD,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,SAAS,EAAE,oDAAoD,CAAC,QAAQ,CACtE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,2BAA2B,CAC5B;IACD,cAAc,EAAE,yDAAyD;SACtE,QAAQ,CACP,uSAAuS,CACxS;IACH,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAC9B,qIAAqI,CACtI;IACD,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAChC,+HAA+H,CAChI;IACD,YAAY,EAAE,qCAAqC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACtE,QAAQ,CAAC,oCAAoC,CAAC;IACjD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,0CAA0C,CAC3C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC3C,WAAW,EAAE,sDAAsD,CAAC,QAAQ,CAC1E,0HAA0H,CAC3H;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,8CAA8C,CAC/C;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,qJAAqJ,CACtJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,oDAAoD,CACrD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;CAC1C,CAAC,CAAC"}
|
|
@@ -9,11 +9,13 @@ import { RoutersTestsEvaluatorRef } from "./routerstestsevaluatorref.js";
|
|
|
9
9
|
* - `response`: judges the generated reply
|
|
10
10
|
* - `tool_call`: diffs the generated tool calls
|
|
11
11
|
* - `conversation`: judges the full conversation
|
|
12
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
12
13
|
*/
|
|
13
14
|
export declare const TestCreateType: {
|
|
14
15
|
readonly Response: "response";
|
|
15
16
|
readonly ToolCall: "tool_call";
|
|
16
17
|
readonly Conversation: "conversation";
|
|
18
|
+
readonly General: "general";
|
|
17
19
|
};
|
|
18
20
|
/**
|
|
19
21
|
* What the test judges:
|
|
@@ -23,12 +25,14 @@ export declare const TestCreateType: {
|
|
|
23
25
|
* - `response`: judges the generated reply
|
|
24
26
|
* - `tool_call`: diffs the generated tool calls
|
|
25
27
|
* - `conversation`: judges the full conversation
|
|
28
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
26
29
|
*/
|
|
27
30
|
export type TestCreateType = ClosedEnum<typeof TestCreateType>;
|
|
28
31
|
export declare const TestCreateType$zodSchema: z.ZodEnum<{
|
|
29
32
|
response: "response";
|
|
30
33
|
tool_call: "tool_call";
|
|
31
34
|
conversation: "conversation";
|
|
35
|
+
general: "general";
|
|
32
36
|
}>;
|
|
33
37
|
export type TestCreate = {
|
|
34
38
|
name: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testcreate.d.ts","sourceRoot":"","sources":["../../src/models/testcreate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,wBAAwB,EAEzB,MAAM,+BAA+B,CAAC;AAEvC
|
|
1
|
+
{"version":3,"file":"testcreate.d.ts","sourceRoot":"","sources":["../../src/models/testcreate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,wBAAwB,EAEzB,MAAM,+BAA+B,CAAC;AAEvC;;;;;;;;;GASG;AACH,eAAO,MAAM,cAAc;;;;;CAKjB,CAAC;AACX;;;;;;;;;GASG;AACH,MAAM,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAE/D,eAAO,MAAM,wBAAwB;;;;;EAOpC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,UAAU,CAAC,EAAE,KAAK,CAAC,wBAAwB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACjE,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAYrD,CAAC"}
|
package/esm/models/testcreate.js
CHANGED
|
@@ -11,22 +11,25 @@ import { RoutersTestsEvaluatorRef$zodSchema, } from "./routerstestsevaluatorref.
|
|
|
11
11
|
* - `response`: judges the generated reply
|
|
12
12
|
* - `tool_call`: diffs the generated tool calls
|
|
13
13
|
* - `conversation`: judges the full conversation
|
|
14
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
14
15
|
*/
|
|
15
16
|
export const TestCreateType = {
|
|
16
17
|
Response: "response",
|
|
17
18
|
ToolCall: "tool_call",
|
|
18
19
|
Conversation: "conversation",
|
|
20
|
+
General: "general",
|
|
19
21
|
};
|
|
20
22
|
export const TestCreateType$zodSchema = z.enum([
|
|
21
23
|
"response",
|
|
22
24
|
"tool_call",
|
|
23
25
|
"conversation",
|
|
24
|
-
|
|
26
|
+
"general",
|
|
27
|
+
]).describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n");
|
|
25
28
|
export const TestCreate$zodSchema = z.object({
|
|
26
|
-
config: z.record(z.string(), z.any()).nullable().optional().describe("The calibrate test config
|
|
29
|
+
config: z.record(z.string(), z.any()).nullable().optional().describe("The calibrate test config.\n\n- `history`: the conversation up to the agent's turn, required for `response` and `conversation` tests and for a `tool_call` test aimed at a conversational agent. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `input`: a standalone prompt with no conversation around it, required for `general` tests and for a `tool_call` test aimed at a `general` agent. A string, not a conversation.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\nA `tool_call` test carries exactly one of `history` or `input`, and which one it carries decides the agent it can be linked to.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n- `general`: judge a standalone, non-conversational input/output pair, graded by the linked evaluators. `{\"type\": \"general\"}`\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n \"evaluation\": {\"type\": \"response\"},\n \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example, for a conversational agent. Swap `history` for `input` to aim it at a `general` agent:\n```json\n{\n \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n \"evaluation\": {\n \"type\": \"tool_call\",\n \"tool_calls\": [\n {\n \"tool\": \"book_room\",\n \"arguments\": {\n \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n },\n \"accept_any_arguments\": false\n }\n ]\n }\n}\n```\n\n`general` example:\n```json\n{\n \"input\": \"Summarize this article: ...\",\n \"evaluation\": {\"type\": \"general\"},\n \"settings\": {\"language\": \"en\"}\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update"),
|
|
27
30
|
evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
|
|
28
31
|
.describe("Evaluators to link. Used by `response` and `conversation` tests"),
|
|
29
32
|
name: z.string().describe("Name of the test, unique within the workspace"),
|
|
30
|
-
type: TestCreateType$zodSchema.describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n"),
|
|
33
|
+
type: TestCreateType$zodSchema.describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n"),
|
|
31
34
|
});
|
|
32
35
|
//# sourceMappingURL=testcreate.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testcreate.js","sourceRoot":"","sources":["../../src/models/testcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,kCAAkC,GACnC,MAAM,+BAA+B,CAAC;AAEvC
|
|
1
|
+
{"version":3,"file":"testcreate.js","sourceRoot":"","sources":["../../src/models/testcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,kCAAkC,GACnC,MAAM,+BAA+B,CAAC;AAEvC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,QAAQ,EAAE,UAAU;IACpB,QAAQ,EAAE,WAAW;IACrB,YAAY,EAAE,cAAc;IAC5B,OAAO,EAAE,SAAS;CACV,CAAC;AAaX,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC7C,UAAU;IACV,WAAW;IACX,cAAc;IACd,SAAS;CACV,CAAC,CAAC,QAAQ,CACT,8SAA8S,CAC/S,CAAC;AASF,MAAM,CAAC,MAAM,oBAAoB,GAA0B,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,4rFAA4rF,CAC7rF;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC1E,QAAQ,CACP,iEAAiE,CAClE;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;IAC1E,IAAI,EAAE,wBAAwB,CAAC,QAAQ,CACrC,8SAA8S,CAC/S;CACF,CAAC,CAAC"}
|
|
@@ -9,11 +9,13 @@ import { TestListConfig } from "./testlistconfig.js";
|
|
|
9
9
|
* - `response`: judges the generated reply
|
|
10
10
|
* - `tool_call`: diffs the generated tool calls
|
|
11
11
|
* - `conversation`: judges the full conversation
|
|
12
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
12
13
|
*/
|
|
13
14
|
export declare const TestListResponseType: {
|
|
14
15
|
readonly Response: "response";
|
|
15
16
|
readonly ToolCall: "tool_call";
|
|
16
17
|
readonly Conversation: "conversation";
|
|
18
|
+
readonly General: "general";
|
|
17
19
|
};
|
|
18
20
|
/**
|
|
19
21
|
* What the test judges:
|
|
@@ -23,12 +25,14 @@ export declare const TestListResponseType: {
|
|
|
23
25
|
* - `response`: judges the generated reply
|
|
24
26
|
* - `tool_call`: diffs the generated tool calls
|
|
25
27
|
* - `conversation`: judges the full conversation
|
|
28
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
26
29
|
*/
|
|
27
30
|
export type TestListResponseType = ClosedEnum<typeof TestListResponseType>;
|
|
28
31
|
export declare const TestListResponseType$zodSchema: z.ZodEnum<{
|
|
29
32
|
response: "response";
|
|
30
33
|
tool_call: "tool_call";
|
|
31
34
|
conversation: "conversation";
|
|
35
|
+
general: "general";
|
|
32
36
|
}>;
|
|
33
37
|
/**
|
|
34
38
|
* Trimmed test shape for list/index endpoints. The full config and hydrated
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testlistresponse.d.ts","sourceRoot":"","sources":["../../src/models/testlistresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAE/E
|
|
1
|
+
{"version":3,"file":"testlistresponse.d.ts","sourceRoot":"","sources":["../../src/models/testlistresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAE/E;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB;;;;;CAKvB,CAAC;AACX;;;;;;;;;GASG;AACH,MAAM,MAAM,oBAAoB,GAAG,UAAU,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAE3E,eAAO,MAAM,8BAA8B;;;;;EAO1C,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,oBAAoB,CAAC;IAC3B,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI,GAAG,SAAS,CAAC;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAiBlE,CAAC"}
|
|
@@ -11,22 +11,25 @@ import { TestListConfig$zodSchema } from "./testlistconfig.js";
|
|
|
11
11
|
* - `response`: judges the generated reply
|
|
12
12
|
* - `tool_call`: diffs the generated tool calls
|
|
13
13
|
* - `conversation`: judges the full conversation
|
|
14
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
14
15
|
*/
|
|
15
16
|
export const TestListResponseType = {
|
|
16
17
|
Response: "response",
|
|
17
18
|
ToolCall: "tool_call",
|
|
18
19
|
Conversation: "conversation",
|
|
20
|
+
General: "general",
|
|
19
21
|
};
|
|
20
22
|
export const TestListResponseType$zodSchema = z.enum([
|
|
21
23
|
"response",
|
|
22
24
|
"tool_call",
|
|
23
25
|
"conversation",
|
|
24
|
-
|
|
26
|
+
"general",
|
|
27
|
+
]).describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n");
|
|
25
28
|
export const TestListResponse$zodSchema = z.object({
|
|
26
29
|
config: TestListConfig$zodSchema.nullable().optional().describe("Trimmed config carrying only the test's description. Fetch the test by ID for the full config and evaluators"),
|
|
27
30
|
created_at: z.string().describe("When the test was created (ISO 8601 UTC)"),
|
|
28
31
|
name: z.string().describe("Name of the test"),
|
|
29
|
-
type: TestListResponseType$zodSchema.describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n"),
|
|
32
|
+
type: TestListResponseType$zodSchema.describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n"),
|
|
30
33
|
updated_at: z.string().describe("When the test was last updated (ISO 8601 UTC)"),
|
|
31
34
|
uuid: z.string().describe("Unique ID for the test"),
|
|
32
35
|
}).describe("Trimmed test shape for list/index endpoints. The full config and hydrated\nevaluators live on the detail endpoint (`GET /tests/{uuid}`); the list keeps\nonly what list and attach-dropdown views render.");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testlistresponse.js","sourceRoot":"","sources":["../../src/models/testlistresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAE/E
|
|
1
|
+
{"version":3,"file":"testlistresponse.js","sourceRoot":"","sources":["../../src/models/testlistresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAE/E;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,QAAQ,EAAE,UAAU;IACpB,QAAQ,EAAE,WAAW;IACrB,YAAY,EAAE,cAAc;IAC5B,OAAO,EAAE,SAAS;CACV,CAAC;AAaX,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC,IAAI,CAAC;IACnD,UAAU;IACV,WAAW;IACX,cAAc;IACd,SAAS;CACV,CAAC,CAAC,QAAQ,CACT,8SAA8S,CAC/S,CAAC;AAkBF,MAAM,CAAC,MAAM,0BAA0B,GAAgC,CAAC,CAAC,MAAM,CAC7E;IACE,MAAM,EAAE,wBAAwB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,8GAA8G,CAC/G;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAC3E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC7C,IAAI,EAAE,8BAA8B,CAAC,QAAQ,CAC3C,8SAA8S,CAC/S;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;CACpD,CACF,CAAC,QAAQ,CACR,2MAA2M,CAC5M,CAAC"}
|
|
@@ -8,11 +8,13 @@ import { ClosedEnum } from "../types/enums.js";
|
|
|
8
8
|
* - `response`: judges the generated reply
|
|
9
9
|
* - `tool_call`: diffs the generated tool calls
|
|
10
10
|
* - `conversation`: judges the full conversation
|
|
11
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
11
12
|
*/
|
|
12
13
|
export declare const TestResponseType: {
|
|
13
14
|
readonly Response: "response";
|
|
14
15
|
readonly ToolCall: "tool_call";
|
|
15
16
|
readonly Conversation: "conversation";
|
|
17
|
+
readonly General: "general";
|
|
16
18
|
};
|
|
17
19
|
/**
|
|
18
20
|
* What the test judges:
|
|
@@ -22,12 +24,14 @@ export declare const TestResponseType: {
|
|
|
22
24
|
* - `response`: judges the generated reply
|
|
23
25
|
* - `tool_call`: diffs the generated tool calls
|
|
24
26
|
* - `conversation`: judges the full conversation
|
|
27
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
25
28
|
*/
|
|
26
29
|
export type TestResponseType = ClosedEnum<typeof TestResponseType>;
|
|
27
30
|
export declare const TestResponseType$zodSchema: z.ZodEnum<{
|
|
28
31
|
response: "response";
|
|
29
32
|
tool_call: "tool_call";
|
|
30
33
|
conversation: "conversation";
|
|
34
|
+
general: "general";
|
|
31
35
|
}>;
|
|
32
36
|
export type TestResponse = {
|
|
33
37
|
uuid: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testresponse.d.ts","sourceRoot":"","sources":["../../src/models/testresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C
|
|
1
|
+
{"version":3,"file":"testresponse.d.ts","sourceRoot":"","sources":["../../src/models/testresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;;;;;;;;GASG;AACH,eAAO,MAAM,gBAAgB;;;;;CAKnB,CAAC;AACX;;;;;;;;;GASG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEnE,eAAO,MAAM,0BAA0B;;;;;EAOtC,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,gBAAgB,CAAC;IACvB,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;CACtD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAgBzD,CAAC"}
|
|
@@ -10,23 +10,26 @@ import * as z from "zod";
|
|
|
10
10
|
* - `response`: judges the generated reply
|
|
11
11
|
* - `tool_call`: diffs the generated tool calls
|
|
12
12
|
* - `conversation`: judges the full conversation
|
|
13
|
+
* - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
|
|
13
14
|
*/
|
|
14
15
|
export const TestResponseType = {
|
|
15
16
|
Response: "response",
|
|
16
17
|
ToolCall: "tool_call",
|
|
17
18
|
Conversation: "conversation",
|
|
19
|
+
General: "general",
|
|
18
20
|
};
|
|
19
21
|
export const TestResponseType$zodSchema = z.enum([
|
|
20
22
|
"response",
|
|
21
23
|
"tool_call",
|
|
22
24
|
"conversation",
|
|
23
|
-
|
|
25
|
+
"general",
|
|
26
|
+
]).describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n");
|
|
24
27
|
export const TestResponse$zodSchema = z.object({
|
|
25
28
|
config: z.record(z.string(), z.any()).nullable().optional().describe("The stored config: `history`, `evaluation`, and an optional `settings`"),
|
|
26
29
|
created_at: z.string().describe("When the test was created (ISO 8601 UTC)"),
|
|
27
30
|
evaluators: z.array(z.record(z.string(), z.any())).optional().describe("Linked evaluators, resolved to their current live version at read time"),
|
|
28
31
|
name: z.string().describe("Name of the test"),
|
|
29
|
-
type: TestResponseType$zodSchema.describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n"),
|
|
32
|
+
type: TestResponseType$zodSchema.describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n"),
|
|
30
33
|
updated_at: z.string().describe("When the test was last updated (ISO 8601 UTC)"),
|
|
31
34
|
uuid: z.string().describe("Unique ID for the test"),
|
|
32
35
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testresponse.js","sourceRoot":"","sources":["../../src/models/testresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB
|
|
1
|
+
{"version":3,"file":"testresponse.js","sourceRoot":"","sources":["../../src/models/testresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,QAAQ,EAAE,UAAU;IACpB,QAAQ,EAAE,WAAW;IACrB,YAAY,EAAE,cAAc;IAC5B,OAAO,EAAE,SAAS;CACV,CAAC;AAaX,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,IAAI,CAAC;IAC/C,UAAU;IACV,WAAW;IACX,cAAc;IACd,SAAS;CACV,CAAC,CAAC,QAAQ,CACT,8SAA8S,CAC/S,CAAC;AAYF,MAAM,CAAC,MAAM,sBAAsB,GAA4B,CAAC,CAAC,MAAM,CAAC;IACtE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,wEAAwE,CACzE;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAC3E,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,wEAAwE,CACzE;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC7C,IAAI,EAAE,0BAA0B,CAAC,QAAQ,CACvC,8SAA8S,CAC/S;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;CACpD,CAAC,CAAC"}
|
|
@@ -5,12 +5,14 @@ export declare const TestUpdateType: {
|
|
|
5
5
|
readonly Response: "response";
|
|
6
6
|
readonly ToolCall: "tool_call";
|
|
7
7
|
readonly Conversation: "conversation";
|
|
8
|
+
readonly General: "general";
|
|
8
9
|
};
|
|
9
10
|
export type TestUpdateType = ClosedEnum<typeof TestUpdateType>;
|
|
10
11
|
export declare const TestUpdateType$zodSchema: z.ZodEnum<{
|
|
11
12
|
response: "response";
|
|
12
13
|
tool_call: "tool_call";
|
|
13
14
|
conversation: "conversation";
|
|
15
|
+
general: "general";
|
|
14
16
|
}>;
|
|
15
17
|
export type TestUpdate = {
|
|
16
18
|
name?: string | null | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testupdate.d.ts","sourceRoot":"","sources":["../../src/models/testupdate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,wBAAwB,EAEzB,MAAM,+BAA+B,CAAC;AAEvC,eAAO,MAAM,cAAc
|
|
1
|
+
{"version":3,"file":"testupdate.d.ts","sourceRoot":"","sources":["../../src/models/testupdate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,wBAAwB,EAEzB,MAAM,+BAA+B,CAAC;AAEvC,eAAO,MAAM,cAAc;;;;;CAKjB,CAAC;AACX,MAAM,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAE/D,eAAO,MAAM,wBAAwB;;;;;EAKnC,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,IAAI,CAAC,EAAE,cAAc,GAAG,IAAI,GAAG,SAAS,CAAC;IACzC,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,UAAU,CAAC,EAAE,KAAK,CAAC,wBAAwB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACjE,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAcrD,CAAC"}
|