@dalmia/calibrate-mcp 0.0.41 → 0.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mcp-server.js +592 -138
- package/bin/mcp-server.js.map +14 -14
- package/esm/landing-page.js +1 -1
- package/esm/lib/config.d.ts +2 -2
- package/esm/lib/config.js +2 -2
- package/esm/mcp-server/mcp-server.js +1 -1
- package/esm/mcp-server/server.js +1 -1
- package/esm/models/annotationtaskresponse.js +1 -1
- package/esm/models/annotationtaskresponse.js.map +1 -1
- package/esm/models/evaluatorcreate.d.ts +4 -0
- package/esm/models/evaluatorcreate.d.ts.map +1 -1
- package/esm/models/evaluatorcreate.js +5 -2
- package/esm/models/evaluatorcreate.js.map +1 -1
- package/esm/models/evaluatordetailresponsecompact.d.ts +5 -0
- package/esm/models/evaluatordetailresponsecompact.d.ts.map +1 -1
- package/esm/models/evaluatordetailresponsecompact.js +6 -2
- package/esm/models/evaluatordetailresponsecompact.js.map +1 -1
- package/esm/models/listevaluatorsevaluatorsgetop.d.ts +2 -0
- package/esm/models/listevaluatorsevaluatorsgetop.d.ts.map +1 -1
- package/esm/models/listevaluatorsevaluatorsgetop.js +2 -0
- package/esm/models/listevaluatorsevaluatorsgetop.js.map +1 -1
- package/esm/models/routersevaluatorsevaluatorresponse.d.ts +5 -0
- package/esm/models/routersevaluatorsevaluatorresponse.d.ts.map +1 -1
- package/esm/models/routersevaluatorsevaluatorresponse.js +6 -2
- package/esm/models/routersevaluatorsevaluatorresponse.js.map +1 -1
- package/package.json +1 -1
- package/src/landing-page.ts +1 -1
- package/src/lib/config.ts +2 -2
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +1 -1
- package/src/models/annotationtaskresponse.ts +1 -1
- package/src/models/evaluatorcreate.ts +6 -2
- package/src/models/evaluatordetailresponsecompact.ts +10 -2
- package/src/models/listevaluatorsevaluatorsgetop.ts +2 -0
- package/src/models/routersevaluatorsevaluatorresponse.ts +10 -2
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
* - `llm`: a reply with its conversation history
|
|
20
20
|
* - `llm-general`: a standalone input and output pair
|
|
21
21
|
* - `conversation`: a full conversation
|
|
22
|
+
* - `tool-call`: whether the agent called the right tool, labelled by a person
|
|
22
23
|
*/
|
|
23
24
|
export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
|
|
24
25
|
Tts: "tts",
|
|
@@ -26,6 +27,7 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
|
|
|
26
27
|
Llm: "llm",
|
|
27
28
|
LlmGeneral: "llm-general",
|
|
28
29
|
Conversation: "conversation",
|
|
30
|
+
ToolCall: "tool-call",
|
|
29
31
|
} as const;
|
|
30
32
|
/**
|
|
31
33
|
* What the evaluator judges:
|
|
@@ -37,6 +39,7 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
|
|
|
37
39
|
* - `llm`: a reply with its conversation history
|
|
38
40
|
* - `llm-general`: a standalone input and output pair
|
|
39
41
|
* - `conversation`: a full conversation
|
|
42
|
+
* - `tool-call`: whether the agent called the right tool, labelled by a person
|
|
40
43
|
*/
|
|
41
44
|
export type RoutersEvaluatorsEvaluatorResponseEvaluatorType = ClosedEnum<
|
|
42
45
|
typeof RoutersEvaluatorsEvaluatorResponseEvaluatorType
|
|
@@ -49,9 +52,10 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema = z.enum(
|
|
|
49
52
|
"llm",
|
|
50
53
|
"llm-general",
|
|
51
54
|
"conversation",
|
|
55
|
+
"tool-call",
|
|
52
56
|
],
|
|
53
57
|
).describe(
|
|
54
|
-
"What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
|
|
58
|
+
"What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
|
|
55
59
|
);
|
|
56
60
|
|
|
57
61
|
/**
|
|
@@ -122,6 +126,7 @@ export type RoutersEvaluatorsEvaluatorResponse = {
|
|
|
122
126
|
data_type: RoutersEvaluatorsEvaluatorResponseDataType;
|
|
123
127
|
output_type: RoutersEvaluatorsEvaluatorResponseOutputType;
|
|
124
128
|
is_default: boolean;
|
|
129
|
+
is_protected: boolean;
|
|
125
130
|
slug?: string | null | undefined;
|
|
126
131
|
source_default_slug?: string | null | undefined;
|
|
127
132
|
live_version_id?: string | null | undefined;
|
|
@@ -144,11 +149,14 @@ export const RoutersEvaluatorsEvaluatorResponse$zodSchema: z.ZodType<
|
|
|
144
149
|
),
|
|
145
150
|
evaluator_type: RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema
|
|
146
151
|
.describe(
|
|
147
|
-
"What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
|
|
152
|
+
"What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
|
|
148
153
|
),
|
|
149
154
|
is_default: z.boolean().describe(
|
|
150
155
|
"True when the evaluator is a built-in default or your workspace's editable copy of one. False for an evaluator you created yourself",
|
|
151
156
|
),
|
|
157
|
+
is_protected: z.boolean().describe(
|
|
158
|
+
"True when the evaluator is locked. A locked evaluator cannot be deleted, and only its name, description and rubric can change",
|
|
159
|
+
),
|
|
152
160
|
live_version: EvaluatorLiveVersionSummary$zodSchema.nullable().optional()
|
|
153
161
|
.describe("The version that is currently live"),
|
|
154
162
|
live_version_id: z.string().nullable().optional().describe(
|