@dalmia/calibrate-mcp 0.0.41 → 0.0.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/mcp-server.js +592 -138
  2. package/bin/mcp-server.js.map +14 -14
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/annotationtaskresponse.js +1 -1
  9. package/esm/models/annotationtaskresponse.js.map +1 -1
  10. package/esm/models/evaluatorcreate.d.ts +4 -0
  11. package/esm/models/evaluatorcreate.d.ts.map +1 -1
  12. package/esm/models/evaluatorcreate.js +5 -2
  13. package/esm/models/evaluatorcreate.js.map +1 -1
  14. package/esm/models/evaluatordetailresponsecompact.d.ts +5 -0
  15. package/esm/models/evaluatordetailresponsecompact.d.ts.map +1 -1
  16. package/esm/models/evaluatordetailresponsecompact.js +6 -2
  17. package/esm/models/evaluatordetailresponsecompact.js.map +1 -1
  18. package/esm/models/listevaluatorsevaluatorsgetop.d.ts +2 -0
  19. package/esm/models/listevaluatorsevaluatorsgetop.d.ts.map +1 -1
  20. package/esm/models/listevaluatorsevaluatorsgetop.js +2 -0
  21. package/esm/models/listevaluatorsevaluatorsgetop.js.map +1 -1
  22. package/esm/models/routersevaluatorsevaluatorresponse.d.ts +5 -0
  23. package/esm/models/routersevaluatorsevaluatorresponse.d.ts.map +1 -1
  24. package/esm/models/routersevaluatorsevaluatorresponse.js +6 -2
  25. package/esm/models/routersevaluatorsevaluatorresponse.js.map +1 -1
  26. package/package.json +1 -1
  27. package/src/landing-page.ts +1 -1
  28. package/src/lib/config.ts +2 -2
  29. package/src/mcp-server/mcp-server.ts +1 -1
  30. package/src/mcp-server/server.ts +1 -1
  31. package/src/models/annotationtaskresponse.ts +1 -1
  32. package/src/models/evaluatorcreate.ts +6 -2
  33. package/src/models/evaluatordetailresponsecompact.ts +10 -2
  34. package/src/models/listevaluatorsevaluatorsgetop.ts +2 -0
  35. package/src/models/routersevaluatorsevaluatorresponse.ts +10 -2
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.41/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.42/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.41";
38
+ readonly sdkVersion: "0.0.42";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.41 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.42 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.41",
30
+ sdkVersion: "0.0.42",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.41 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.42 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.41",
21
+ currentVersion: "0.0.42",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -49,7 +49,7 @@ import { tool$tracesCreate } from "./tools/tracesCreate.js";
49
49
  export function createMCPServer(deps) {
50
50
  const server = new McpServer({
51
51
  name: "CalibrateMcp",
52
- version: "0.0.41",
52
+ version: "0.0.42",
53
53
  });
54
54
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
55
55
  security: deps.security,
package/esm/models/annotationtaskresponse.js CHANGED
@@ -31,7 +31,7 @@ export const AnnotationTaskResponse$zodSchema = z.object({
31
31
  evaluators: z.array(z.record(z.string(), z.any())).optional().describe("The evaluators linked to this task, in display order. Fetching one task by ID enriches each with its live version's rubric. Listing tasks returns lightweight metadata (uuid, name, type, position) only"),
32
32
  has_agreement: z.boolean().default(false).describe("Whether the task has at least one comparable human-vs-human or human-vs-evaluator pair, computed over all time"),
33
33
  item_count: z.int().nullable().optional().describe("Number of items in the task"),
34
- items: z.array(z.record(z.string(), z.any())).optional().describe("The task's items, each with its agreement stats. You get these when you fetch one task by ID, not when you list tasks"),
34
+ items: z.array(z.record(z.string(), z.any())).optional().describe("The task's items, each with its agreement stats and an `is_tool_call` flag marking the rows a person labels on the tool call rather than a text reply. You get these when you fetch one task by ID, not when you list tasks"),
35
35
  jobs: z.array(z.record(z.string(), z.any())).optional().describe("The task's labelling jobs. You get these when you fetch one task by ID, not when you list tasks"),
36
36
  name: z.string().describe("Name of the task"),
37
37
  type: AnnotationTaskResponseType$zodSchema.describe("Task type. Determines the shape of each item's payload.\n- `stt`: judge a transcript on its own\n- `llm`: judge one response with its conversation history\n- `llm-general`: judge a standalone `input -> output` pair\n- `conversation`: judge a full conversation"),
package/esm/models/annotationtaskresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"annotationtaskresponse.js","sourceRoot":"","sources":["../../src/models/annotationtaskresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,GAAG,EAAE,KAAK;CACF,CAAC;AAcX,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC,IAAI,CAAC;IACzD,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,KAAK;CACN,CAAC,CAAC,QAAQ,CACT,qQAAqQ,CACtQ,CAAC;AAgBF,MAAM,CAAC,MAAM,gCAAgC,GAEzC,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAC3E,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,wBAAwB,CACzB;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,0MAA0M,CAC3M;IACD,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,gHAAgH,CACjH;IACD,UAAU,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,6BAA6B,CAC9B;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/D,uHAAuH,CACxH;IACD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9D,iGAAiG,CAClG;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC7C,IAAI,EAAE,oCAAoC,CAAC,QAAQ,CACjD,qQAAqQ,CACtQ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;CACpD,CAAC,CAAC"}
1
+ {"version":3,"file":"annotationtaskresponse.js","sourceRoot":"","sources":["../../src/models/annotationtaskresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,GAAG,EAAE,KAAK;CACF,CAAC;AAcX,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC,IAAI,CAAC;IACzD,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,KAAK;CACN,CAAC,CAAC,QAAQ,CACT,qQAAqQ,CACtQ,CAAC;AAgBF,MAAM,CAAC,MAAM,gCAAgC,GAEzC,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAC3E,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,wBAAwB,CACzB;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,0MAA0M,CAC3M;IACD,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,gHAAgH,CACjH;IACD,UAAU,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,6BAA6B,CAC9B;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/D,6NAA6N,CAC9N;IACD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9D,iGAAiG,CAClG;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC7C,IAAI,EAAE,oCAAoC,CAAC,QAAQ,CACjD,qQAAqQ,CACtQ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;CACpD,CAAC,CAAC"}
package/esm/models/evaluatorcreate.d.ts CHANGED
@@ -11,6 +11,7 @@ import { EvaluatorVersionCreate } from "./evaluatorversioncreate.js";
11
11
  * - `llm`: a reply with its conversation history
12
12
  * - `llm-general`: a standalone input and output pair
13
13
  * - `conversation`: a full conversation
14
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
14
15
  */
15
16
  export declare const EvaluatorCreateEvaluatorType: {
16
17
  readonly Tts: "tts";
@@ -18,6 +19,7 @@ export declare const EvaluatorCreateEvaluatorType: {
18
19
  readonly Llm: "llm";
19
20
  readonly LlmGeneral: "llm-general";
20
21
  readonly Conversation: "conversation";
22
+ readonly ToolCall: "tool-call";
21
23
  };
22
24
  /**
23
25
  * What the evaluator judges:
@@ -29,6 +31,7 @@ export declare const EvaluatorCreateEvaluatorType: {
29
31
  * - `llm`: a reply with its conversation history
30
32
  * - `llm-general`: a standalone input and output pair
31
33
  * - `conversation`: a full conversation
34
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
32
35
  */
33
36
  export type EvaluatorCreateEvaluatorType = ClosedEnum<typeof EvaluatorCreateEvaluatorType>;
34
37
  export declare const EvaluatorCreateEvaluatorType$zodSchema: z.ZodEnum<{
@@ -37,6 +40,7 @@ export declare const EvaluatorCreateEvaluatorType$zodSchema: z.ZodEnum<{
37
40
  stt: "stt";
38
41
  llm: "llm";
39
42
  "llm-general": "llm-general";
43
+ "tool-call": "tool-call";
40
44
  }>;
41
45
  /**
42
46
  * The modality the judge reads:
package/esm/models/evaluatorcreate.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluatorcreate.d.ts","sourceRoot":"","sources":["../../src/models/evaluatorcreate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,sBAAsB,EAEvB,MAAM,6BAA6B,CAAC;AAErC;;;;;;;;;;GAUG;AACH,eAAO,MAAM,4BAA4B;;;;;;CAM/B,CAAC;AACX;;;;;;;;;;GAUG;AACH,MAAM,MAAM,4BAA4B,GAAG,UAAU,CACnD,OAAO,4BAA4B,CACpC,CAAC;AAEF,eAAO,MAAM,sCAAsC;;;;;;EAQlD,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,uBAAuB;;;CAG1B,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,uBAAuB,GAAG,UAAU,CAC9C,OAAO,uBAAuB,CAC/B,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;EAGuB,CAAC;AAEtE;;;;;;;GAOG;AACH,eAAO,MAAM,yBAAyB;;;CAG5B,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,yBAAyB,GAAG,UAAU,CAChD,OAAO,yBAAyB,CACjC,CAAC;AAEF,eAAO,MAAM,mCAAmC;;;EAK/C,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,cAAc,CAAC,EAAE,4BAA4B,GAAG,SAAS,CAAC;IAC1D,SAAS,CAAC,EAAE,uBAAuB,GAAG,SAAS,CAAC;IAChD,WAAW,CAAC,EAAE,yBAAyB,GAAG,SAAS,CAAC;IACpD,OAAO,EAAE,sBAAsB,CAAC;CACjC,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAkB/D,CAAC"}
1
+ {"version":3,"file":"evaluatorcreate.d.ts","sourceRoot":"","sources":["../../src/models/evaluatorcreate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,sBAAsB,EAEvB,MAAM,6BAA6B,CAAC;AAErC;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,4BAA4B;;;;;;;CAO/B,CAAC;AACX;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,4BAA4B,GAAG,UAAU,CACnD,OAAO,4BAA4B,CACpC,CAAC;AAEF,eAAO,MAAM,sCAAsC;;;;;;;EASlD,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,uBAAuB;;;CAG1B,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,uBAAuB,GAAG,UAAU,CAC9C,OAAO,uBAAuB,CAC/B,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;EAGuB,CAAC;AAEtE;;;;;;;GAOG;AACH,eAAO,MAAM,yBAAyB;;;CAG5B,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,yBAAyB,GAAG,UAAU,CAChD,OAAO,yBAAyB,CACjC,CAAC;AAEF,eAAO,MAAM,mCAAmC;;;EAK/C,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,cAAc,CAAC,EAAE,4BAA4B,GAAG,SAAS,CAAC;IAC1D,SAAS,CAAC,EAAE,uBAAuB,GAAG,SAAS,CAAC;IAChD,WAAW,CAAC,EAAE,yBAAyB,GAAG,SAAS,CAAC;IACpD,OAAO,EAAE,sBAAsB,CAAC;CACjC,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAkB/D,CAAC"}
package/esm/models/evaluatorcreate.js CHANGED
@@ -13,6 +13,7 @@ import { EvaluatorVersionCreate$zodSchema, } from "./evaluatorversioncreate.js";
13
13
  * - `llm`: a reply with its conversation history
14
14
  * - `llm-general`: a standalone input and output pair
15
15
  * - `conversation`: a full conversation
16
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
16
17
  */
17
18
  export const EvaluatorCreateEvaluatorType = {
18
19
  Tts: "tts",
@@ -20,6 +21,7 @@ export const EvaluatorCreateEvaluatorType = {
20
21
  Llm: "llm",
21
22
  LlmGeneral: "llm-general",
22
23
  Conversation: "conversation",
24
+ ToolCall: "tool-call",
23
25
  };
24
26
  export const EvaluatorCreateEvaluatorType$zodSchema = z.enum([
25
27
  "tts",
@@ -27,7 +29,8 @@ export const EvaluatorCreateEvaluatorType$zodSchema = z.enum([
27
29
  "llm",
28
30
  "llm-general",
29
31
  "conversation",
30
- ]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n");
32
+ "tool-call",
33
+ ]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n");
31
34
  /**
32
35
  * The modality the judge reads:
33
36
  *
@@ -64,7 +67,7 @@ export const EvaluatorCreate$zodSchema = z.object({
64
67
  data_type: EvaluatorCreateDataType$zodSchema.default("text").describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
65
68
  description: z.string().nullable().optional().describe("Description. Omit to leave blank"),
66
69
  evaluator_type: EvaluatorCreateEvaluatorType$zodSchema.default("llm")
67
- .describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
70
+ .describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n"),
68
71
  name: z.string().describe("Evaluator name, unique within your workspace"),
69
72
  output_type: EvaluatorCreateOutputType$zodSchema.default("binary").describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n"),
70
73
  version: EvaluatorVersionCreate$zodSchema.describe("One version of an evaluator: its judge prompt, model, variables, and rubric."),
package/esm/models/evaluatorcreate.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluatorcreate.js","sourceRoot":"","sources":["../../src/models/evaluatorcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,gCAAgC,GACjC,MAAM,6BAA6B,CAAC;AAErC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG;IAC1C,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;CACpB,CAAC;AAgBX,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,CAAC,IAAI,CAAC;IAC3D,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;CACf,CAAC,CAAC,QAAQ,CACT,yNAAyN,CAC1N,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,CAAC,IAAI,CAAC;IACtD,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAC,CAAC,IAAI,CAAC;IACxD,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAWF,MAAM,CAAC,MAAM,yBAAyB,GAA+B,CAAC,CAAC,MAAM,CAAC;IAC5E,SAAS,EAAE,iCAAiC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CACnE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,kCAAkC,CACnC;IACD,cAAc,EAAE,sCAAsC,CAAC,OAAO,CAAC,KAAK,CAAC;SAClE,QAAQ,CACP,yNAAyN,CAC1N;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;IACzE,WAAW,EAAE,mCAAmC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,QAAQ,CACzE,0HAA0H,CAC3H;IACD,OAAO,EAAE,gCAAgC,CAAC,QAAQ,CAChD,8EAA8E,CAC/E;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"evaluatorcreate.js","sourceRoot":"","sources":["../../src/models/evaluatorcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,gCAAgC,GACjC,MAAM,6BAA6B,CAAC;AAErC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG;IAC1C,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,WAAW;CACb,CAAC;AAiBX,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,CAAC,IAAI,CAAC;IAC3D,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,WAAW;CACZ,CAAC,CAAC,QAAQ,CACT,uSAAuS,CACxS,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,CAAC,IAAI,CAAC;IACtD,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAC,CAAC,IAAI,CAAC;IACxD,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAWF,MAAM,CAAC,MAAM,yBAAyB,GAA+B,CAAC,CAAC,MAAM,CAAC;IAC5E,SAAS,EAAE,iCAAiC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CACnE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,kCAAkC,CACnC;IACD,cAAc,EAAE,sCAAsC,CAAC,OAAO,CAAC,KAAK,CAAC;SAClE,QAAQ,CACP,uSAAuS,CACxS;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;IACzE,WAAW,EAAE,mCAAmC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,QAAQ,CACzE,0HAA0H,CAC3H;IACD,OAAO,EAAE,gCAAgC,CAAC,QAAQ,CAChD,8EAA8E,CAC/E;CACF,CAAC,CAAC"}
package/esm/models/evaluatordetailresponsecompact.d.ts CHANGED
@@ -11,6 +11,7 @@ import { EvaluatorVersionCompact } from "./evaluatorversioncompact.js";
11
11
  * - `llm`: a reply with its conversation history
12
12
  * - `llm-general`: a standalone input and output pair
13
13
  * - `conversation`: a full conversation
14
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
14
15
  */
15
16
  export declare const EvaluatorDetailResponseCompactEvaluatorType: {
16
17
  readonly Tts: "tts";
@@ -18,6 +19,7 @@ export declare const EvaluatorDetailResponseCompactEvaluatorType: {
18
19
  readonly Llm: "llm";
19
20
  readonly LlmGeneral: "llm-general";
20
21
  readonly Conversation: "conversation";
22
+ readonly ToolCall: "tool-call";
21
23
  };
22
24
  /**
23
25
  * What the evaluator judges:
@@ -29,6 +31,7 @@ export declare const EvaluatorDetailResponseCompactEvaluatorType: {
29
31
  * - `llm`: a reply with its conversation history
30
32
  * - `llm-general`: a standalone input and output pair
31
33
  * - `conversation`: a full conversation
34
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
32
35
  */
33
36
  export type EvaluatorDetailResponseCompactEvaluatorType = ClosedEnum<typeof EvaluatorDetailResponseCompactEvaluatorType>;
34
37
  export declare const EvaluatorDetailResponseCompactEvaluatorType$zodSchema: z.ZodEnum<{
@@ -37,6 +40,7 @@ export declare const EvaluatorDetailResponseCompactEvaluatorType$zodSchema: z.Zo
37
40
  stt: "stt";
38
41
  llm: "llm";
39
42
  "llm-general": "llm-general";
43
+ "tool-call": "tool-call";
40
44
  }>;
41
45
  /**
42
46
  * The modality the judge reads:
@@ -96,6 +100,7 @@ export type EvaluatorDetailResponseCompact = {
96
100
  data_type: EvaluatorDetailResponseCompactDataType;
97
101
  output_type: EvaluatorDetailResponseCompactOutputType;
98
102
  is_default: boolean;
103
+ is_protected: boolean;
99
104
  slug?: string | null | undefined;
100
105
  source_default_slug?: string | null | undefined;
101
106
  live_version_id?: string | null | undefined;
package/esm/models/evaluatordetailresponsecompact.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluatordetailresponsecompact.d.ts","sourceRoot":"","sources":["../../src/models/evaluatordetailresponsecompact.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,uBAAuB,EAExB,MAAM,8BAA8B,CAAC;AAEtC;;;;;;;;;;GAUG;AACH,eAAO,MAAM,2CAA2C;;;;;;CAM9C,CAAC;AACX;;;;;;;;;;GAUG;AACH,MAAM,MAAM,2CAA2C,GAAG,UAAU,CAClE,OAAO,2CAA2C,CACnD,CAAC;AAEF,eAAO,MAAM,qDAAqD;;;;;;EAQjE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,sCAAsC;;;CAGzC,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,sCAAsC,GAAG,UAAU,CAC7D,OAAO,sCAAsC,CAC9C,CAAC;AAEF,eAAO,MAAM,gDAAgD;;;EAGQ,CAAC;AAEtE;;;;;;;GAOG;AACH,eAAO,MAAM,wCAAwC;;;CAG3C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,wCAAwC,GAAG,UAAU,CAC/D,OAAO,wCAAwC,CAChD,CAAC;AAEF,eAAO,MAAM,kDAAkD;;;EAK9D,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,cAAc,EAAE,2CAA2C,CAAC;IAC5D,SAAS,EAAE,sCAAsC,CAAC;IAClD,WAAW,EAAE,wCAAwC,CAAC;IACtD,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC;CAC1C,CAAC;AAEF,eAAO,MAAM,wCAAwC,EAAE,CAAC,CAAC,OAAO,CAC9D,8BAA8B,CAsC9B,CAAC"}
1
+ {"version":3,"file":"evaluatordetailresponsecompact.d.ts","sourceRoot":"","sources":["../../src/models/evaluatordetailresponsecompact.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,uBAAuB,EAExB,MAAM,8BAA8B,CAAC;AAEtC;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,2CAA2C;;;;;;;CAO9C,CAAC;AACX;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,2CAA2C,GAAG,UAAU,CAClE,OAAO,2CAA2C,CACnD,CAAC;AAEF,eAAO,MAAM,qDAAqD;;;;;;;EASjE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,sCAAsC;;;CAGzC,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,sCAAsC,GAAG,UAAU,CAC7D,OAAO,sCAAsC,CAC9C,CAAC;AAEF,eAAO,MAAM,gDAAgD;;;EAGQ,CAAC;AAEtE;;;;;;;GAOG;AACH,eAAO,MAAM,wCAAwC;;;CAG3C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,wCAAwC,GAAG,UAAU,CAC/D,OAAO,wCAAwC,CAChD,CAAC;AAEF,eAAO,MAAM,kDAAkD;;;EAK9D,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,cAAc,EAAE,2CAA2C,CAAC;IAC5D,SAAS,EAAE,sCAAsC,CAAC;IAClD,WAAW,EAAE,wCAAwC,CAAC;IACtD,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC;CAC1C,CAAC;AAEF,eAAO,MAAM,wCAAwC,EAAE,CAAC,CAAC,OAAO,CAC9D,8BAA8B,CAyC9B,CAAC"}
package/esm/models/evaluatordetailresponsecompact.js CHANGED
@@ -13,6 +13,7 @@ import { EvaluatorVersionCompact$zodSchema, } from "./evaluatorversioncompact.js
13
13
  * - `llm`: a reply with its conversation history
14
14
  * - `llm-general`: a standalone input and output pair
15
15
  * - `conversation`: a full conversation
16
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
16
17
  */
17
18
  export const EvaluatorDetailResponseCompactEvaluatorType = {
18
19
  Tts: "tts",
@@ -20,6 +21,7 @@ export const EvaluatorDetailResponseCompactEvaluatorType = {
20
21
  Llm: "llm",
21
22
  LlmGeneral: "llm-general",
22
23
  Conversation: "conversation",
24
+ ToolCall: "tool-call",
23
25
  };
24
26
  export const EvaluatorDetailResponseCompactEvaluatorType$zodSchema = z.enum([
25
27
  "tts",
@@ -27,7 +29,8 @@ export const EvaluatorDetailResponseCompactEvaluatorType$zodSchema = z.enum([
27
29
  "llm",
28
30
  "llm-general",
29
31
  "conversation",
30
- ]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n");
32
+ "tool-call",
33
+ ]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n");
31
34
  /**
32
35
  * The modality the judge reads:
33
36
  *
@@ -65,8 +68,9 @@ export const EvaluatorDetailResponseCompact$zodSchema = z.object({
65
68
  data_type: EvaluatorDetailResponseCompactDataType$zodSchema.describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
66
69
  description: z.string().nullable().optional().describe("What the evaluator checks"),
67
70
  evaluator_type: EvaluatorDetailResponseCompactEvaluatorType$zodSchema
68
- .describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
71
+ .describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n"),
69
72
  is_default: z.boolean().describe("True when the evaluator is a built-in default or your workspace's editable copy of one. False for an evaluator you created yourself"),
73
+ is_protected: z.boolean().describe("True when the evaluator is locked. A locked evaluator cannot be deleted, and only its name, description and rubric can change"),
70
74
  live_version_id: z.string().nullable().optional().describe("ID of the version that is currently live"),
71
75
  name: z.string().describe("Evaluator name"),
72
76
  output_type: EvaluatorDetailResponseCompactOutputType$zodSchema.describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n"),
package/esm/models/evaluatordetailresponsecompact.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluatordetailresponsecompact.js","sourceRoot":"","sources":["../../src/models/evaluatordetailresponsecompact.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,iCAAiC,GAClC,MAAM,8BAA8B,CAAC;AAEtC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,2CAA2C,GAAG;IACzD,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;CACpB,CAAC;AAgBX,MAAM,CAAC,MAAM,qDAAqD,GAAG,CAAC,CAAC,IAAI,CAAC;IAC1E,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;CACf,CAAC,CAAC,QAAQ,CACT,yNAAyN,CAC1N,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAAG;IACpD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,gDAAgD,GAAG,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG;IACtD,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,kDAAkD,GAAG,CAAC,CAAC,IAAI,CAAC;IACvE,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAkBF,MAAM,CAAC,MAAM,wCAAwC,GAEjD,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,SAAS,EAAE,gDAAgD,CAAC,QAAQ,CAClE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,2BAA2B,CAC5B;IACD,cAAc,EAAE,qDAAqD;SAClE,QAAQ,CACP,yNAAyN,CAC1N;IACH,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAC9B,qIAAqI,CACtI;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,0CAA0C,CAC3C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC3C,WAAW,EAAE,kDAAkD,CAAC,QAAQ,CACtE,0HAA0H,CAC3H;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,8CAA8C,CAC/C;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,qJAAqJ,CACtJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,oDAAoD,CACrD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;IACzC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,QAAQ,CAC3D,oCAAoC,CACrC;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"evaluatordetailresponsecompact.js","sourceRoot":"","sources":["../../src/models/evaluatordetailresponsecompact.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,iCAAiC,GAClC,MAAM,8BAA8B,CAAC;AAEtC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,2CAA2C,GAAG;IACzD,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,WAAW;CACb,CAAC;AAiBX,MAAM,CAAC,MAAM,qDAAqD,GAAG,CAAC,CAAC,IAAI,CAAC;IAC1E,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,WAAW;CACZ,CAAC,CAAC,QAAQ,CACT,uSAAuS,CACxS,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAAG;IACpD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,gDAAgD,GAAG,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG;IACtD,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,kDAAkD,GAAG,CAAC,CAAC,IAAI,CAAC;IACvE,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAmBF,MAAM,CAAC,MAAM,wCAAwC,GAEjD,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,SAAS,EAAE,gDAAgD,CAAC,QAAQ,CAClE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,2BAA2B,CAC5B;IACD,cAAc,EAAE,qDAAqD;SAClE,QAAQ,CACP,uSAAuS,CACxS;IACH,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAC9B,qIAAqI,CACtI;IACD,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAChC,+HAA+H,CAChI;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,0CAA0C,CAC3C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC3C,WAAW,EAAE,kDAAkD,CAAC,QAAQ,CACtE,0HAA0H,CAC3H;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,8CAA8C,CAC/C;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,qJAAqJ,CACtJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,oDAAoD,CACrD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;IACzC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,QAAQ,CAC3D,oCAAoC,CACrC;CACF,CAAC,CAAC"}
package/esm/models/listevaluatorsevaluatorsgetop.d.ts CHANGED
@@ -11,6 +11,7 @@ export declare const ListEvaluatorsEvaluatorsGetEvaluatorType: {
11
11
  readonly Llm: "llm";
12
12
  readonly LlmGeneral: "llm-general";
13
13
  readonly Conversation: "conversation";
14
+ readonly ToolCall: "tool-call";
14
15
  };
15
16
  /**
16
17
  * Filter by what the evaluator judges. Omit for all types
@@ -22,6 +23,7 @@ export declare const ListEvaluatorsEvaluatorsGetEvaluatorType$zodSchema: z.ZodEn
22
23
  stt: "stt";
23
24
  llm: "llm";
24
25
  "llm-general": "llm-general";
26
+ "tool-call": "tool-call";
25
27
  }>;
26
28
  /**
27
29
  * Filter by modality. Omit for all
package/esm/models/listevaluatorsevaluatorsgetop.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"listevaluatorsevaluatorsgetop.d.ts","sourceRoot":"","sources":["../../src/models/listevaluatorsevaluatorsgetop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kCAAkC,EAEnC,MAAM,yCAAyC,CAAC;AAEjD;;GAEG;AACH,eAAO,MAAM,wCAAwC;;;;;;CAM3C,CAAC;AACX;;GAEG;AACH,MAAM,MAAM,wCAAwC,GAAG,UAAU,CAC/D,OAAO,wCAAwC,CAChD,CAAC;AAEF,eAAO,MAAM,kDAAkD;;;;;;EAMO,CAAC;AAEvE;;GAEG;AACH,eAAO,MAAM,mCAAmC;;;CAGtC,CAAC;AACX;;GAEG;AACH,MAAM,MAAM,mCAAmC,GAAG,UAAU,CAC1D,OAAO,mCAAmC,CAC3C,CAAC;AAEF,eAAO,MAAM,6CAA6C;;;EAGX,CAAC;AAEhD,MAAM,MAAM,kCAAkC,GAAG;IAC/C,cAAc,CAAC,EAAE,wCAAwC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7E,SAAS,CAAC,EAAE,mCAAmC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnE,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4CAA4C,EAAE,CAAC,CAAC,OAAO,CAClE,kCAAkC,CAqBlC,CAAC;AAEH,MAAM,MAAM,mCAAmC,GAC3C,kCAAkC,GAClC,mBAAmB,CAAC;AAExB,eAAO,MAAM,6CAA6C,EAAE,CAAC,CAAC,OAAO,CACnE,mCAAmC,CAInC,CAAC"}
1
+ {"version":3,"file":"listevaluatorsevaluatorsgetop.d.ts","sourceRoot":"","sources":["../../src/models/listevaluatorsevaluatorsgetop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kCAAkC,EAEnC,MAAM,yCAAyC,CAAC;AAEjD;;GAEG;AACH,eAAO,MAAM,wCAAwC;;;;;;;CAO3C,CAAC;AACX;;GAEG;AACH,MAAM,MAAM,wCAAwC,GAAG,UAAU,CAC/D,OAAO,wCAAwC,CAChD,CAAC;AAEF,eAAO,MAAM,kDAAkD;;;;;;;EAOO,CAAC;AAEvE;;GAEG;AACH,eAAO,MAAM,mCAAmC;;;CAGtC,CAAC;AACX;;GAEG;AACH,MAAM,MAAM,mCAAmC,GAAG,UAAU,CAC1D,OAAO,mCAAmC,CAC3C,CAAC;AAEF,eAAO,MAAM,6CAA6C;;;EAGX,CAAC;AAEhD,MAAM,MAAM,kCAAkC,GAAG;IAC/C,cAAc,CAAC,EAAE,wCAAwC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7E,SAAS,CAAC,EAAE,mCAAmC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnE,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4CAA4C,EAAE,CAAC,CAAC,OAAO,CAClE,kCAAkC,CAqBlC,CAAC;AAEH,MAAM,MAAM,mCAAmC,GAC3C,kCAAkC,GAClC,mBAAmB,CAAC;AAExB,eAAO,MAAM,6CAA6C,EAAE,CAAC,CAAC,OAAO,CACnE,mCAAmC,CAInC,CAAC"}
package/esm/models/listevaluatorsevaluatorsgetop.js CHANGED
@@ -13,6 +13,7 @@ export const ListEvaluatorsEvaluatorsGetEvaluatorType = {
13
13
  Llm: "llm",
14
14
  LlmGeneral: "llm-general",
15
15
  Conversation: "conversation",
16
+ ToolCall: "tool-call",
16
17
  };
17
18
  export const ListEvaluatorsEvaluatorsGetEvaluatorType$zodSchema = z.enum([
18
19
  "tts",
@@ -20,6 +21,7 @@ export const ListEvaluatorsEvaluatorsGetEvaluatorType$zodSchema = z.enum([
20
21
  "llm",
21
22
  "llm-general",
22
23
  "conversation",
24
+ "tool-call",
23
25
  ]).describe("Filter by what the evaluator judges. Omit for all types");
24
26
  /**
25
27
  * Filter by modality. Omit for all
package/esm/models/listevaluatorsevaluatorsgetop.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"listevaluatorsevaluatorsgetop.js","sourceRoot":"","sources":["../../src/models/listevaluatorsevaluatorsgetop.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,4CAA4C,GAC7C,MAAM,yCAAyC,CAAC;AAEjD;;GAEG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG;IACtD,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;CACpB,CAAC;AAQX,MAAM,CAAC,MAAM,kDAAkD,GAAG,CAAC,CAAC,IAAI,CAAC;IACvE,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;CACf,CAAC,CAAC,QAAQ,CAAC,yDAAyD,CAAC,CAAC;AAEvE;;GAEG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG;IACjD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAQX,MAAM,CAAC,MAAM,6CAA6C,GAAG,CAAC,CAAC,IAAI,CAAC;IAClE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,kCAAkC,CAAC,CAAC;AAYhD,MAAM,CAAC,MAAM,4CAA4C,GAErD,CAAC,CAAC,MAAM,CAAC;IACX,SAAS,EAAE,6CAA6C,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC3E,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,cAAc,EAAE,kDAAkD,CAAC,QAAQ,EAAE;SAC1E,QAAQ,EAAE,CAAC,QAAQ,CAClB,yDAAyD,CAC1D;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAClD,iJAAiJ,CAClJ;IACD,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CACrB,kEAAkE,CACnE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CACjC,kDAAkD,CACnD;IACD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACpB,+DAA+D,CAChE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAMH,MAAM,CAAC,MAAM,6CAA6C,GAEtD,CAAC,CAAC,KAAK,CAAC;IACV,4CAA4C;IAC5C,6BAA6B;CAC9B,CAAC,CAAC"}
1
+ {"version":3,"file":"listevaluatorsevaluatorsgetop.js","sourceRoot":"","sources":["../../src/models/listevaluatorsevaluatorsgetop.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,4CAA4C,GAC7C,MAAM,yCAAyC,CAAC;AAEjD;;GAEG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG;IACtD,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,WAAW;CACb,CAAC;AAQX,MAAM,CAAC,MAAM,kDAAkD,GAAG,CAAC,CAAC,IAAI,CAAC;IACvE,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,WAAW;CACZ,CAAC,CAAC,QAAQ,CAAC,yDAAyD,CAAC,CAAC;AAEvE;;GAEG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG;IACjD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAQX,MAAM,CAAC,MAAM,6CAA6C,GAAG,CAAC,CAAC,IAAI,CAAC;IAClE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,kCAAkC,CAAC,CAAC;AAYhD,MAAM,CAAC,MAAM,4CAA4C,GAErD,CAAC,CAAC,MAAM,CAAC;IACX,SAAS,EAAE,6CAA6C,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC3E,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,cAAc,EAAE,kDAAkD,CAAC,QAAQ,EAAE;SAC1E,QAAQ,EAAE,CAAC,QAAQ,CAClB,yDAAyD,CAC1D;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAClD,iJAAiJ,CAClJ;IACD,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CACrB,kEAAkE,CACnE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CACjC,kDAAkD,CACnD;IACD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACpB,+DAA+D,CAChE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAMH,MAAM,CAAC,MAAM,6CAA6C,GAEtD,CAAC,CAAC,KAAK,CAAC;IACV,4CAA4C;IAC5C,6BAA6B;CAC9B,CAAC,CAAC"}
package/esm/models/routersevaluatorsevaluatorresponse.d.ts CHANGED
@@ -11,6 +11,7 @@ import { EvaluatorLiveVersionSummary } from "./evaluatorliveversionsummary.js";
11
11
  * - `llm`: a reply with its conversation history
12
12
  * - `llm-general`: a standalone input and output pair
13
13
  * - `conversation`: a full conversation
14
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
14
15
  */
15
16
  export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType: {
16
17
  readonly Tts: "tts";
@@ -18,6 +19,7 @@ export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType: {
18
19
  readonly Llm: "llm";
19
20
  readonly LlmGeneral: "llm-general";
20
21
  readonly Conversation: "conversation";
22
+ readonly ToolCall: "tool-call";
21
23
  };
22
24
  /**
23
25
  * What the evaluator judges:
@@ -29,6 +31,7 @@ export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType: {
29
31
  * - `llm`: a reply with its conversation history
30
32
  * - `llm-general`: a standalone input and output pair
31
33
  * - `conversation`: a full conversation
34
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
32
35
  */
33
36
  export type RoutersEvaluatorsEvaluatorResponseEvaluatorType = ClosedEnum<typeof RoutersEvaluatorsEvaluatorResponseEvaluatorType>;
34
37
  export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema: z.ZodEnum<{
@@ -37,6 +40,7 @@ export declare const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema:
37
40
  stt: "stt";
38
41
  llm: "llm";
39
42
  "llm-general": "llm-general";
43
+ "tool-call": "tool-call";
40
44
  }>;
41
45
  /**
42
46
  * The modality the judge reads:
@@ -96,6 +100,7 @@ export type RoutersEvaluatorsEvaluatorResponse = {
96
100
  data_type: RoutersEvaluatorsEvaluatorResponseDataType;
97
101
  output_type: RoutersEvaluatorsEvaluatorResponseOutputType;
98
102
  is_default: boolean;
103
+ is_protected: boolean;
99
104
  slug?: string | null | undefined;
100
105
  source_default_slug?: string | null | undefined;
101
106
  live_version_id?: string | null | undefined;
package/esm/models/routersevaluatorsevaluatorresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"routersevaluatorsevaluatorresponse.d.ts","sourceRoot":"","sources":["../../src/models/routersevaluatorsevaluatorresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,2BAA2B,EAE5B,MAAM,kCAAkC,CAAC;AAE1C;;;;;;;;;;GAUG;AACH,eAAO,MAAM,+CAA+C;;;;;;CAMlD,CAAC;AACX;;;;;;;;;;GAUG;AACH,MAAM,MAAM,+CAA+C,GAAG,UAAU,CACtE,OAAO,+CAA+C,CACvD,CAAC;AAEF,eAAO,MAAM,yDAAyD;;;;;;EAUrE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,0CAA0C;;;CAG7C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,0CAA0C,GAAG,UAAU,CACjE,OAAO,0CAA0C,CAClD,CAAC;AAEF,eAAO,MAAM,oDAAoD;;;EAGI,CAAC;AAEtE;;;;;;;GAOG;AACH,eAAO,MAAM,4CAA4C;;;CAG/C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,4CAA4C,GAAG,UAAU,CACnE,OAAO,4CAA4C,CACpD,CAAC;AAEF,eAAO,MAAM,sDAAsD;;;EAKlE,CAAC;AAEF,MAAM,MAAM,kCAAkC,GAAG;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,cAAc,EAAE,+CAA+C,CAAC;IAChE,SAAS,EAAE,0CAA0C,CAAC;IACtD,WAAW,EAAE,4CAA4C,CAAC;IAC1D,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,2BAA2B,GAAG,IAAI,GAAG,SAAS,CAAC;CAC/D,CAAC;AAEF,eAAO,MAAM,4CAA4C,EAAE,CAAC,CAAC,OAAO,CAClE,kCAAkC,CAqClC,CAAC"}
1
+ {"version":3,"file":"routersevaluatorsevaluatorresponse.d.ts","sourceRoot":"","sources":["../../src/models/routersevaluatorsevaluatorresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,2BAA2B,EAE5B,MAAM,kCAAkC,CAAC;AAE1C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,+CAA+C;;;;;;;CAOlD,CAAC;AACX;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,+CAA+C,GAAG,UAAU,CACtE,OAAO,+CAA+C,CACvD,CAAC;AAEF,eAAO,MAAM,yDAAyD;;;;;;;EAWrE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,0CAA0C;;;CAG7C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,0CAA0C,GAAG,UAAU,CACjE,OAAO,0CAA0C,CAClD,CAAC;AAEF,eAAO,MAAM,oDAAoD;;;EAGI,CAAC;AAEtE;;;;;;;GAOG;AACH,eAAO,MAAM,4CAA4C;;;CAG/C,CAAC;AACX;;;;;;;GAOG;AACH,MAAM,MAAM,4CAA4C,GAAG,UAAU,CACnE,OAAO,4CAA4C,CACpD,CAAC;AAEF,eAAO,MAAM,sDAAsD;;;EAKlE,CAAC;AAEF,MAAM,MAAM,kCAAkC,GAAG;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,cAAc,EAAE,+CAA+C,CAAC;IAChE,SAAS,EAAE,0CAA0C,CAAC;IACtD,WAAW,EAAE,4CAA4C,CAAC;IAC1D,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,2BAA2B,GAAG,IAAI,GAAG,SAAS,CAAC;CAC/D,CAAC;AAEF,eAAO,MAAM,4CAA4C,EAAE,CAAC,CAAC,OAAO,CAClE,kCAAkC,CAwClC,CAAC"}
package/esm/models/routersevaluatorsevaluatorresponse.js CHANGED
@@ -13,6 +13,7 @@ import { EvaluatorLiveVersionSummary$zodSchema, } from "./evaluatorliveversionsu
13
13
  * - `llm`: a reply with its conversation history
14
14
  * - `llm-general`: a standalone input and output pair
15
15
  * - `conversation`: a full conversation
16
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
16
17
  */
17
18
  export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
18
19
  Tts: "tts",
@@ -20,6 +21,7 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
20
21
  Llm: "llm",
21
22
  LlmGeneral: "llm-general",
22
23
  Conversation: "conversation",
24
+ ToolCall: "tool-call",
23
25
  };
24
26
  export const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema = z.enum([
25
27
  "tts",
@@ -27,7 +29,8 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema = z.enum(
27
29
  "llm",
28
30
  "llm-general",
29
31
  "conversation",
30
- ]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n");
32
+ "tool-call",
33
+ ]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n");
31
34
  /**
32
35
  * The modality the judge reads:
33
36
  *
@@ -65,8 +68,9 @@ export const RoutersEvaluatorsEvaluatorResponse$zodSchema = z.object({
65
68
  data_type: RoutersEvaluatorsEvaluatorResponseDataType$zodSchema.describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
66
69
  description: z.string().nullable().optional().describe("What the evaluator checks"),
67
70
  evaluator_type: RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema
68
- .describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
71
+ .describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n"),
69
72
  is_default: z.boolean().describe("True when the evaluator is a built-in default or your workspace's editable copy of one. False for an evaluator you created yourself"),
73
+ is_protected: z.boolean().describe("True when the evaluator is locked. A locked evaluator cannot be deleted, and only its name, description and rubric can change"),
70
74
  live_version: EvaluatorLiveVersionSummary$zodSchema.nullable().optional()
71
75
  .describe("The version that is currently live"),
72
76
  live_version_id: z.string().nullable().optional().describe("ID of the version that is currently live"),
package/esm/models/routersevaluatorsevaluatorresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"routersevaluatorsevaluatorresponse.js","sourceRoot":"","sources":["../../src/models/routersevaluatorsevaluatorresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,qCAAqC,GACtC,MAAM,kCAAkC,CAAC;AAE1C;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,+CAA+C,GAAG;IAC7D,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;CACpB,CAAC;AAgBX,MAAM,CAAC,MAAM,yDAAyD,GAAG,CAAC,CAAC,IAAI,CAC7E;IACE,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;CACf,CACF,CAAC,QAAQ,CACR,yNAAyN,CAC1N,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0CAA0C,GAAG;IACxD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,oDAAoD,GAAG,CAAC,CAAC,IAAI,CAAC;IACzE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,4CAA4C,GAAG;IAC1D,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,sDAAsD,GAAG,CAAC,CAAC,IAAI,CAAC;IAC3E,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAkBF,MAAM,CAAC,MAAM,4CAA4C,GAErD,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,SAAS,EAAE,oDAAoD,CAAC,QAAQ,CACtE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,2BAA2B,CAC5B;IACD,cAAc,EAAE,yDAAyD;SACtE,QAAQ,CACP,yNAAyN,CAC1N;IACH,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAC9B,qIAAqI,CACtI;IACD,YAAY,EAAE,qCAAqC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACtE,QAAQ,CAAC,oCAAoC,CAAC;IACjD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,0CAA0C,CAC3C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC3C,WAAW,EAAE,sDAAsD,CAAC,QAAQ,CAC1E,0HAA0H,CAC3H;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,8CAA8C,CAC/C;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,qJAAqJ,CACtJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,oDAAoD,CACrD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;CAC1C,CAAC,CAAC"}
1
+ {"version":3,"file":"routersevaluatorsevaluatorresponse.js","sourceRoot":"","sources":["../../src/models/routersevaluatorsevaluatorresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,qCAAqC,GACtC,MAAM,kCAAkC,CAAC;AAE1C;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,+CAA+C,GAAG;IAC7D,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,WAAW;CACb,CAAC;AAiBX,MAAM,CAAC,MAAM,yDAAyD,GAAG,CAAC,CAAC,IAAI,CAC7E;IACE,KAAK;IACL,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,WAAW;CACZ,CACF,CAAC,QAAQ,CACR,uSAAuS,CACxS,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0CAA0C,GAAG;IACxD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;CACN,CAAC;AAaX,MAAM,CAAC,MAAM,oDAAoD,GAAG,CAAC,CAAC,IAAI,CAAC;IACzE,MAAM;IACN,OAAO;CACR,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,4CAA4C,GAAG;IAC1D,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;CACR,CAAC;AAaX,MAAM,CAAC,MAAM,sDAAsD,GAAG,CAAC,CAAC,IAAI,CAAC;IAC3E,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0HAA0H,CAC3H,CAAC;AAmBF,MAAM,CAAC,MAAM,4CAA4C,GAErD,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,SAAS,EAAE,oDAAoD,CAAC,QAAQ,CACtE,wDAAwD,CACzD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,2BAA2B,CAC5B;IACD,cAAc,EAAE,yDAAyD;SACtE,QAAQ,CACP,uSAAuS,CACxS;IACH,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAC9B,qIAAqI,CACtI;IACD,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAChC,+HAA+H,CAChI;IACD,YAAY,EAAE,qCAAqC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACtE,QAAQ,CAAC,oCAAoC,CAAC;IACjD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,0CAA0C,CAC3C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC3C,WAAW,EAAE,sDAAsD,CAAC,QAAQ,CAC1E,0HAA0H,CAC3H;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,8CAA8C,CAC/C;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,qJAAqJ,CACtJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,oDAAoD,CACrD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;CAC1C,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.41",
3
+ "version": "0.0.42",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.41/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.42/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.41",
68
+ sdkVersion: "0.0.42",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.41 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.42 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.41",
24
+ currentVersion: "0.0.42",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -71,7 +71,7 @@ export function createMCPServer(deps: {
71
71
  }) {
72
72
  const server = new McpServer({
73
73
  name: "CalibrateMcp",
74
- version: "0.0.41",
74
+ version: "0.0.42",
75
75
  });
76
76
 
77
77
  const getClient = deps.getSDK || (() =>
package/src/models/annotationtaskresponse.ts CHANGED
@@ -75,7 +75,7 @@ export const AnnotationTaskResponse$zodSchema: z.ZodType<
75
75
  "Number of items in the task",
76
76
  ),
77
77
  items: z.array(z.record(z.string(), z.any())).optional().describe(
78
- "The task's items, each with its agreement stats. You get these when you fetch one task by ID, not when you list tasks",
78
+ "The task's items, each with its agreement stats and an `is_tool_call` flag marking the rows a person labels on the tool call rather than a text reply. You get these when you fetch one task by ID, not when you list tasks",
79
79
  ),
80
80
  jobs: z.array(z.record(z.string(), z.any())).optional().describe(
81
81
  "The task's labelling jobs. You get these when you fetch one task by ID, not when you list tasks",
package/src/models/evaluatorcreate.ts CHANGED
@@ -19,6 +19,7 @@ import {
19
19
  * - `llm`: a reply with its conversation history
20
20
  * - `llm-general`: a standalone input and output pair
21
21
  * - `conversation`: a full conversation
22
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
22
23
  */
23
24
  export const EvaluatorCreateEvaluatorType = {
24
25
  Tts: "tts",
@@ -26,6 +27,7 @@ export const EvaluatorCreateEvaluatorType = {
26
27
  Llm: "llm",
27
28
  LlmGeneral: "llm-general",
28
29
  Conversation: "conversation",
30
+ ToolCall: "tool-call",
29
31
  } as const;
30
32
  /**
31
33
  * What the evaluator judges:
@@ -37,6 +39,7 @@ export const EvaluatorCreateEvaluatorType = {
37
39
  * - `llm`: a reply with its conversation history
38
40
  * - `llm-general`: a standalone input and output pair
39
41
  * - `conversation`: a full conversation
42
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
40
43
  */
41
44
  export type EvaluatorCreateEvaluatorType = ClosedEnum<
42
45
  typeof EvaluatorCreateEvaluatorType
@@ -48,8 +51,9 @@ export const EvaluatorCreateEvaluatorType$zodSchema = z.enum([
48
51
  "llm",
49
52
  "llm-general",
50
53
  "conversation",
54
+ "tool-call",
51
55
  ]).describe(
52
- "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
56
+ "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
53
57
  );
54
58
 
55
59
  /**
@@ -130,7 +134,7 @@ export const EvaluatorCreate$zodSchema: z.ZodType<EvaluatorCreate> = z.object({
130
134
  ),
131
135
  evaluator_type: EvaluatorCreateEvaluatorType$zodSchema.default("llm")
132
136
  .describe(
133
- "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
137
+ "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
134
138
  ),
135
139
  name: z.string().describe("Evaluator name, unique within your workspace"),
136
140
  output_type: EvaluatorCreateOutputType$zodSchema.default("binary").describe(
package/src/models/evaluatordetailresponsecompact.ts CHANGED
@@ -19,6 +19,7 @@ import {
19
19
  * - `llm`: a reply with its conversation history
20
20
  * - `llm-general`: a standalone input and output pair
21
21
  * - `conversation`: a full conversation
22
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
22
23
  */
23
24
  export const EvaluatorDetailResponseCompactEvaluatorType = {
24
25
  Tts: "tts",
@@ -26,6 +27,7 @@ export const EvaluatorDetailResponseCompactEvaluatorType = {
26
27
  Llm: "llm",
27
28
  LlmGeneral: "llm-general",
28
29
  Conversation: "conversation",
30
+ ToolCall: "tool-call",
29
31
  } as const;
30
32
  /**
31
33
  * What the evaluator judges:
@@ -37,6 +39,7 @@ export const EvaluatorDetailResponseCompactEvaluatorType = {
37
39
  * - `llm`: a reply with its conversation history
38
40
  * - `llm-general`: a standalone input and output pair
39
41
  * - `conversation`: a full conversation
42
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
40
43
  */
41
44
  export type EvaluatorDetailResponseCompactEvaluatorType = ClosedEnum<
42
45
  typeof EvaluatorDetailResponseCompactEvaluatorType
@@ -48,8 +51,9 @@ export const EvaluatorDetailResponseCompactEvaluatorType$zodSchema = z.enum([
48
51
  "llm",
49
52
  "llm-general",
50
53
  "conversation",
54
+ "tool-call",
51
55
  ]).describe(
52
- "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
56
+ "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
53
57
  );
54
58
 
55
59
  /**
@@ -120,6 +124,7 @@ export type EvaluatorDetailResponseCompact = {
120
124
  data_type: EvaluatorDetailResponseCompactDataType;
121
125
  output_type: EvaluatorDetailResponseCompactOutputType;
122
126
  is_default: boolean;
127
+ is_protected: boolean;
123
128
  slug?: string | null | undefined;
124
129
  source_default_slug?: string | null | undefined;
125
130
  live_version_id?: string | null | undefined;
@@ -142,11 +147,14 @@ export const EvaluatorDetailResponseCompact$zodSchema: z.ZodType<
142
147
  ),
143
148
  evaluator_type: EvaluatorDetailResponseCompactEvaluatorType$zodSchema
144
149
  .describe(
145
- "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
150
+ "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
146
151
  ),
147
152
  is_default: z.boolean().describe(
148
153
  "True when the evaluator is a built-in default or your workspace's editable copy of one. False for an evaluator you created yourself",
149
154
  ),
155
+ is_protected: z.boolean().describe(
156
+ "True when the evaluator is locked. A locked evaluator cannot be deleted, and only its name, description and rubric can change",
157
+ ),
150
158
  live_version_id: z.string().nullable().optional().describe(
151
159
  "ID of the version that is currently live",
152
160
  ),
package/src/models/listevaluatorsevaluatorsgetop.ts CHANGED
@@ -22,6 +22,7 @@ export const ListEvaluatorsEvaluatorsGetEvaluatorType = {
22
22
  Llm: "llm",
23
23
  LlmGeneral: "llm-general",
24
24
  Conversation: "conversation",
25
+ ToolCall: "tool-call",
25
26
  } as const;
26
27
  /**
27
28
  * Filter by what the evaluator judges. Omit for all types
@@ -36,6 +37,7 @@ export const ListEvaluatorsEvaluatorsGetEvaluatorType$zodSchema = z.enum([
36
37
  "llm",
37
38
  "llm-general",
38
39
  "conversation",
40
+ "tool-call",
39
41
  ]).describe("Filter by what the evaluator judges. Omit for all types");
40
42
 
41
43
  /**