@dalmia/calibrate-mcp 0.0.92 → 0.0.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/bin/mcp-server.js +21 -15
  2. package/bin/mcp-server.js.map +12 -12
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/createjobsrequest.d.ts +14 -0
  9. package/esm/models/createjobsrequest.d.ts.map +1 -1
  10. package/esm/models/createjobsrequest.js +11 -0
  11. package/esm/models/createjobsrequest.js.map +1 -1
  12. package/esm/models/evaluatorversioncompact.js +1 -1
  13. package/esm/models/evaluatorversioncompact.js.map +1 -1
  14. package/esm/models/evaluatorversioncreate.js +1 -1
  15. package/esm/models/evaluatorversioncreate.js.map +1 -1
  16. package/esm/models/evaluatorversioncreaterequest.js +1 -1
  17. package/esm/models/evaluatorversioncreaterequest.js.map +1 -1
  18. package/esm/models/outputconfig.js +1 -1
  19. package/esm/models/outputconfig.js.map +1 -1
  20. package/esm/models/outputscaleentry.js +4 -4
  21. package/esm/models/outputscaleentry.js.map +1 -1
  22. package/package.json +1 -1
  23. package/src/landing-page.ts +1 -1
  24. package/src/lib/config.ts +2 -2
  25. package/src/mcp-server/mcp-server.ts +1 -1
  26. package/src/mcp-server/server.ts +1 -1
  27. package/src/models/createjobsrequest.ts +25 -0
  28. package/src/models/evaluatorversioncompact.ts +1 -1
  29. package/src/models/evaluatorversioncreate.ts +1 -1
  30. package/src/models/evaluatorversioncreaterequest.ts +1 -1
  31. package/src/models/outputconfig.ts +1 -1
  32. package/src/models/outputscaleentry.ts +4 -4
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.92/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.94/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.92";
38
+ readonly sdkVersion: "0.0.94";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.92 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.94 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.92",
30
+ sdkVersion: "0.0.94",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.92 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.94 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.92",
21
+ currentVersion: "0.0.94",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -56,7 +56,7 @@ import { tool$tracesList } from "./tools/tracesList.js";
56
56
  export function createMCPServer(deps) {
57
57
  const server = new McpServer({
58
58
  name: "CalibrateMcp",
59
- version: "0.0.92",
59
+ version: "0.0.94",
60
60
  });
61
61
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
62
62
  security: deps.security,
package/esm/models/createjobsrequest.d.ts CHANGED
@@ -17,6 +17,17 @@ export declare const ReasoningMode$zodSchema: z.ZodEnum<{
17
17
  required: "required";
18
18
  hidden: "hidden";
19
19
  }>;
20
+ export declare const EvaluatorReasoningModes: {
21
+ readonly Optional: "optional";
22
+ readonly Required: "required";
23
+ readonly Hidden: "hidden";
24
+ };
25
+ export type EvaluatorReasoningModes = ClosedEnum<typeof EvaluatorReasoningModes>;
26
+ export declare const EvaluatorReasoningModes$zodSchema: z.ZodEnum<{
27
+ optional: "optional";
28
+ required: "required";
29
+ hidden: "hidden";
30
+ }>;
20
31
  export type CreateJobsRequest = {
21
32
  annotator_ids: Array<string>;
22
33
  item_ids?: Array<string> | undefined;
@@ -25,6 +36,9 @@ export type CreateJobsRequest = {
25
36
  evaluator_ids?: Array<string> | null | undefined;
26
37
  comments_enabled?: boolean | undefined;
27
38
  reasoning_mode?: ReasoningMode | undefined;
39
+ evaluator_reasoning_modes?: {
40
+ [k: string]: EvaluatorReasoningModes;
41
+ } | null | undefined;
28
42
  };
29
43
  export declare const CreateJobsRequest$zodSchema: z.ZodType<CreateJobsRequest>;
30
44
  //# sourceMappingURL=createjobsrequest.d.ts.map
package/esm/models/createjobsrequest.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"createjobsrequest.d.ts","sourceRoot":"","sources":["../../src/models/createjobsrequest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;GAEG;AACH,eAAO,MAAM,aAAa;;;;CAIhB,CAAC;AACX;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC;AAE7D,eAAO,MAAM,uBAAuB;;;;EAMnC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;IACrC,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACjC,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9B,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,cAAc,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;CAC5C,CAAC;AAEF,eAAO,MAAM,2BAA2B,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAuBjE,CAAC"}
1
+ {"version":3,"file":"createjobsrequest.d.ts","sourceRoot":"","sources":["../../src/models/createjobsrequest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;GAEG;AACH,eAAO,MAAM,aAAa;;;;CAIhB,CAAC;AACX;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC;AAE7D,eAAO,MAAM,uBAAuB;;;;EAMnC,CAAC;AAEF,eAAO,MAAM,uBAAuB;;;;CAI1B,CAAC;AACX,MAAM,MAAM,uBAAuB,GAAG,UAAU,CAC9C,OAAO,uBAAuB,CAC/B,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;EAI5C,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;IACrC,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACjC,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9B,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,cAAc,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;IAC3C,yBAAyB,CAAC,EACtB;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAA;KAAE,GACxC,IAAI,GACJ,SAAS,CAAC;CACf,CAAC;AAEF,eAAO,MAAM,2BAA2B,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CA6BjE,CAAC"}
package/esm/models/createjobsrequest.js CHANGED
@@ -15,11 +15,22 @@ export const ReasoningMode$zodSchema = z.enum([
15
15
  "required",
16
16
  "hidden",
17
17
  ]).describe("How the labelling form treats the reasoning box on each judgement. `optional` shows it, `required` shows it and asks the annotator to fill it in, `hidden` leaves it out");
18
+ export const EvaluatorReasoningModes = {
19
+ Optional: "optional",
20
+ Required: "required",
21
+ Hidden: "hidden",
22
+ };
23
+ export const EvaluatorReasoningModes$zodSchema = z.enum([
24
+ "optional",
25
+ "required",
26
+ "hidden",
27
+ ]);
18
28
  export const CreateJobsRequest$zodSchema = z
19
29
  .object({
20
30
  annotator_ids: z.array(z.string()).describe("Annotator IDs to assign, creating one labelling job for each annotator. Must be in your workspace"),
21
31
  comments_enabled: z.boolean().default(true).describe("When `true`, the labelling form lets the annotator leave a comment on each item"),
22
32
  evaluator_ids: z.array(z.string()).nullable().optional().describe("Subset of the task's linked evaluators to show in these jobs. Must be a subset of the current links, an empty list gives a 400. Applies to every annotator's job. Omit (`None`) to snapshot every linked evaluator"),
33
+ evaluator_reasoning_modes: z.record(z.string(), EvaluatorReasoningModes$zodSchema).nullable().optional().describe("Map of evaluator ID to how the labelling form treats the reasoning box for that evaluator, with the same values as `reasoning_mode`. Every ID must be an evaluator on these jobs, otherwise you get a 400. An evaluator left out uses `reasoning_mode`"),
23
34
  item_ids: z.array(z.string()).optional().describe("Item IDs to assign. **Required when `select_all=false`**. Ignored when `select_all=true`"),
24
35
  q: z.string().nullable().optional().describe("Case-insensitive substring filter on `payload.name`. Applies only when `select_all=true`"),
25
36
  reasoning_mode: ReasoningMode$zodSchema.default("optional").describe("How the labelling form treats the reasoning box on each judgement. `optional` shows it, `required` shows it and asks the annotator to fill it in, `hidden` leaves it out"),
package/esm/models/createjobsrequest.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"createjobsrequest.js","sourceRoot":"","sources":["../../src/models/createjobsrequest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,QAAQ,EAAE,UAAU;IACpB,QAAQ,EAAE,UAAU;IACpB,MAAM,EAAE,QAAQ;CACR,CAAC;AAMX,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC5C,UAAU;IACV,UAAU;IACV,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0KAA0K,CAC3K,CAAC;AAYF,MAAM,CAAC,MAAM,2BAA2B,GAAiC,CAAC;KACvE,MAAM,CAAC;IACN,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CACzC,mGAAmG,CACpG;IACD,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAClD,iFAAiF,CAClF;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/D,oNAAoN,CACrN;IACD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/C,0FAA0F,CAC3F;IACD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1C,0FAA0F,CAC3F;IACD,cAAc,EAAE,uBAAuB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,QAAQ,CAClE,0KAA0K,CAC3K;IACD,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC7C,sHAAsH,CACvH;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"createjobsrequest.js","sourceRoot":"","sources":["../../src/models/createjobsrequest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,QAAQ,EAAE,UAAU;IACpB,QAAQ,EAAE,UAAU;IACpB,MAAM,EAAE,QAAQ;CACR,CAAC;AAMX,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC5C,UAAU;IACV,UAAU;IACV,QAAQ;CACT,CAAC,CAAC,QAAQ,CACT,0KAA0K,CAC3K,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,QAAQ,EAAE,UAAU;IACpB,QAAQ,EAAE,UAAU;IACpB,MAAM,EAAE,QAAQ;CACR,CAAC;AAKX,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,CAAC,IAAI,CAAC;IACtD,UAAU;IACV,UAAU;IACV,QAAQ;CACT,CAAC,CAAC;AAgBH,MAAM,CAAC,MAAM,2BAA2B,GAAiC,CAAC;KACvE,MAAM,CAAC;IACN,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CACzC,mGAAmG,CACpG;IACD,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAClD,iFAAiF,CAClF;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/D,oNAAoN,CACrN;IACD,yBAAyB,EAAE,CAAC,CAAC,MAAM,CACjC,CAAC,CAAC,MAAM,EAAE,EACV,iCAAiC,CAClC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9B,wPAAwP,CACzP;IACD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/C,0FAA0F,CAC3F;IACD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1C,0FAA0F,CAC3F;IACD,cAAc,EAAE,uBAAuB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,QAAQ,CAClE,0KAA0K,CAC3K;IACD,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC7C,sHAAsH,CACvH;CACF,CAAC,CAAC"}
package/esm/models/evaluatorversioncompact.js CHANGED
@@ -7,7 +7,7 @@ import { VariableSpec$zodSchema } from "./variablespec.js";
7
7
  export const EvaluatorVersionCompact$zodSchema = z.object({
8
8
  created_at: z.string().describe("When the version was created (ISO 8601 UTC)"),
9
9
  judge_model: z.string().describe("The model that runs the judge, named the way its provider does, for example `openai/gpt-4.1` or `anthropic/claude-sonnet-4`"),
10
- output_config: OutputConfig$zodSchema.nullable().optional().describe("The scale points and their labels. Required for a `rating` evaluator. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own"),
10
+ output_config: OutputConfig$zodSchema.nullable().optional().describe("The rows of the rating scale, each a number and its label. Required for a `rating` evaluator. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own"),
11
11
  system_prompt: z.string().nullable().optional().describe("Judge system prompt, with `{{\"{{\"}}variable}}` placeholders unrendered"),
12
12
  uuid: z.string().describe("Version ID"),
13
13
  variables: z.array(VariableSpec$zodSchema).nullable().optional().describe("Declared prompt variables"),
package/esm/models/evaluatorversioncompact.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluatorversioncompact.js","sourceRoot":"","sources":["../../src/models/evaluatorversioncompact.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAYzE,MAAM,CAAC,MAAM,iCAAiC,GAE1C,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,6CAA6C,CAC9C;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC9B,6HAA6H,CAC9H;IACD,aAAa,EAAE,sBAAsB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,0JAA0J,CAC3J;IACD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,0EAA0E,CAC3E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC;IACvC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,2BAA2B,CAC5B;IACD,cAAc,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CAC9B,2GAA2G,CAC5G;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"evaluatorversioncompact.js","sourceRoot":"","sources":["../../src/models/evaluatorversioncompact.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAYzE,MAAM,CAAC,MAAM,iCAAiC,GAE1C,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,6CAA6C,CAC9C;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC9B,6HAA6H,CAC9H;IACD,aAAa,EAAE,sBAAsB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,sTAAsT,CACvT;IACD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,0EAA0E,CAC3E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC;IACvC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,2BAA2B,CAC5B;IACD,cAAc,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CAC9B,2GAA2G,CAC5G;CACF,CAAC,CAAC"}
package/esm/models/evaluatorversioncreate.js CHANGED
@@ -6,7 +6,7 @@ import { OutputConfig$zodSchema } from "./outputconfig.js";
6
6
  import { VariableSpec$zodSchema } from "./variablespec.js";
7
7
  export const EvaluatorVersionCreate$zodSchema = z.object({
8
8
  judge_model: z.string().describe("The model that runs the judge, named the way its provider does, for example `openai/gpt-4.1` or `anthropic/claude-sonnet-4`"),
9
- output_config: OutputConfig$zodSchema.nullable().optional().describe("The scale points and their labels. Required for a `rating` evaluator. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own"),
9
+ output_config: OutputConfig$zodSchema.nullable().optional().describe("The rows of the rating scale, each a number and its label. Required for a `rating` evaluator. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own"),
10
10
  system_prompt: z.string().describe("Judge system prompt. May contain `{{\"{{\"}}variable}}` placeholders"),
11
11
  variables: z.array(VariableSpec$zodSchema).nullable().optional().describe("Declared prompt variables. Omit if the prompt has no `{{\"{{\"}}placeholders}}`"),
12
12
  }).describe("One version of an evaluator: its judge prompt, model, variables, and rubric.");
package/esm/models/evaluatorversioncreate.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluatorversioncreate.js","sourceRoot":"","sources":["../../src/models/evaluatorversioncreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAYzE,MAAM,CAAC,MAAM,gCAAgC,GAEzC,CAAC,CAAC,MAAM,CAAC;IACX,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC9B,6HAA6H,CAC9H;IACD,aAAa,EAAE,sBAAsB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,0JAA0J,CAC3J;IACD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAChC,sEAAsE,CACvE;IACD,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,iFAAiF,CAClF;CACF,CAAC,CAAC,QAAQ,CACT,8EAA8E,CAC/E,CAAC"}
1
+ {"version":3,"file":"evaluatorversioncreate.js","sourceRoot":"","sources":["../../src/models/evaluatorversioncreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAYzE,MAAM,CAAC,MAAM,gCAAgC,GAEzC,CAAC,CAAC,MAAM,CAAC;IACX,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC9B,6HAA6H,CAC9H;IACD,aAAa,EAAE,sBAAsB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,sTAAsT,CACvT;IACD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAChC,sEAAsE,CACvE;IACD,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,iFAAiF,CAClF;CACF,CAAC,CAAC,QAAQ,CACT,8EAA8E,CAC/E,CAAC"}
package/esm/models/evaluatorversioncreaterequest.js CHANGED
@@ -7,7 +7,7 @@ import { VariableSpec$zodSchema } from "./variablespec.js";
7
7
  export const EvaluatorVersionCreateRequest$zodSchema = z.object({
8
8
  judge_model: z.string().describe("The model that runs the judge, named the way its provider does, for example `openai/gpt-4.1` or `anthropic/claude-sonnet-4`"),
9
9
  make_live: z.boolean().default(false).describe("When `true`, immediately point the evaluator's live version at this new version"),
10
- output_config: OutputConfig$zodSchema.nullable().optional().describe("The scale points and their labels. Required for a `rating` evaluator. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own"),
10
+ output_config: OutputConfig$zodSchema.nullable().optional().describe("The rows of the rating scale, each a number and its label. Required for a `rating` evaluator. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own"),
11
11
  system_prompt: z.string().describe("Judge system prompt. May contain `{{\"{{\"}}variable}}` placeholders"),
12
12
  variables: z.array(VariableSpec$zodSchema).nullable().optional().describe("Declared prompt variables. Omit if the prompt has none. After the first version the variable names are fixed. You can change a variable's description or default, but not add, remove, or rename one"),
13
13
  }).describe("Request body for adding a version to an existing evaluator.");
package/esm/models/evaluatorversioncreaterequest.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluatorversioncreaterequest.js","sourceRoot":"","sources":["../../src/models/evaluatorversioncreaterequest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAazE,MAAM,CAAC,MAAM,uCAAuC,GAEhD,CAAC,CAAC,MAAM,CAAC;IACX,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC9B,6HAA6H,CAC9H;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,iFAAiF,CAClF;IACD,aAAa,EAAE,sBAAsB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,0JAA0J,CAC3J;IACD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAChC,sEAAsE,CACvE;IACD,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,sMAAsM,CACvM;CACF,CAAC,CAAC,QAAQ,CAAC,6DAA6D,CAAC,CAAC"}
1
+ {"version":3,"file":"evaluatorversioncreaterequest.js","sourceRoot":"","sources":["../../src/models/evaluatorversioncreaterequest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAgB,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAazE,MAAM,CAAC,MAAM,uCAAuC,GAEhD,CAAC,CAAC,MAAM,CAAC;IACX,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC9B,6HAA6H,CAC9H;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,iFAAiF,CAClF;IACD,aAAa,EAAE,sBAAsB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,sTAAsT,CACvT;IACD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAChC,sEAAsE,CACvE;IACD,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,sMAAsM,CACvM;CACF,CAAC,CAAC,QAAQ,CAAC,6DAA6D,CAAC,CAAC"}
package/esm/models/outputconfig.js CHANGED
@@ -4,6 +4,6 @@
4
4
  import * as z from "zod";
5
5
  import { OutputScaleEntry$zodSchema, } from "./outputscaleentry.js";
6
6
  export const OutputConfig$zodSchema = z.object({
7
- scale: z.array(OutputScaleEntry$zodSchema).nullable().optional().describe("The ordered scale points that make up the rubric, each with its label"),
7
+ scale: z.array(OutputScaleEntry$zodSchema).nullable().optional().describe("The rows of the rating scale, each a number and its label. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number"),
8
8
  });
9
9
  //# sourceMappingURL=outputconfig.js.map
package/esm/models/outputconfig.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"outputconfig.js","sourceRoot":"","sources":["../../src/models/outputconfig.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAM/B,MAAM,CAAC,MAAM,sBAAsB,GAA4B,CAAC,CAAC,MAAM,CAAC;IACtE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,uEAAuE,CACxE;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"outputconfig.js","sourceRoot":"","sources":["../../src/models/outputconfig.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAM/B,MAAM,CAAC,MAAM,sBAAsB,GAA4B,CAAC,CAAC,MAAM,CAAC;IACtE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,+LAA+L,CAChM;CACF,CAAC,CAAC"}
package/esm/models/outputscaleentry.js CHANGED
@@ -3,9 +3,9 @@
3
3
  */
4
4
  import * as z from "zod";
5
5
  export const OutputScaleEntry$zodSchema = z.object({
6
- color: z.string().nullable().optional().describe("Color to show for this level. Omit for the default"),
7
- description: z.string().nullable().optional().describe("Rubric text for this level, added to the judge prompt. Omit to leave this level undescribed"),
8
- name: z.string().describe("Short label for this scale point"),
9
- value: z.any().optional().describe("The value for this scale point. Use a boolean for a `binary` evaluator, a number for a `rating` one"),
6
+ color: z.string().nullable().optional().describe("Color to show for this row. Omit for the default"),
7
+ description: z.string().nullable().optional().describe("Rubric text for this row, added to the judge prompt. Omit to leave this row undescribed"),
8
+ name: z.string().describe("Short label for this row"),
9
+ value: z.any().optional().describe("The value for this row. Use a boolean for a `binary` evaluator, a number for a `rating` one. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number"),
10
10
  }).describe("One entry in an evaluator's output_config.scale. `value` is bool|number|string\ndepending on output_type.");
11
11
  //# sourceMappingURL=outputscaleentry.js.map
package/esm/models/outputscaleentry.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"outputscaleentry.js","sourceRoot":"","sources":["../../src/models/outputscaleentry.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAezB,MAAM,CAAC,MAAM,0BAA0B,GAAgC,CAAC,CAAC,MAAM,CAC7E;IACE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,oDAAoD,CACrD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,6FAA6F,CAC9F;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC7D,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChC,qGAAqG,CACtG;CACF,CACF,CAAC,QAAQ,CACR,2GAA2G,CAC5G,CAAC"}
1
+ {"version":3,"file":"outputscaleentry.js","sourceRoot":"","sources":["../../src/models/outputscaleentry.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAezB,MAAM,CAAC,MAAM,0BAA0B,GAAgC,CAAC,CAAC,MAAM,CAC7E;IACE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yFAAyF,CAC1F;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC;IACrD,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChC,iOAAiO,CAClO;CACF,CACF,CAAC,QAAQ,CACR,2GAA2G,CAC5G,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.92",
3
+ "version": "0.0.94",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.92/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.94/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.92",
68
+ sdkVersion: "0.0.94",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.92 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.94 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.92",
24
+ currentVersion: "0.0.94",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -78,7 +78,7 @@ export function createMCPServer(deps: {
78
78
  }) {
79
79
  const server = new McpServer({
80
80
  name: "CalibrateMcp",
81
- version: "0.0.92",
81
+ version: "0.0.94",
82
82
  });
83
83
 
84
84
  const getClient = deps.getSDK || (() =>
package/src/models/createjobsrequest.ts CHANGED
@@ -26,6 +26,21 @@ export const ReasoningMode$zodSchema = z.enum([
26
26
  "How the labelling form treats the reasoning box on each judgement. `optional` shows it, `required` shows it and asks the annotator to fill it in, `hidden` leaves it out",
27
27
  );
28
28
 
29
+ export const EvaluatorReasoningModes = {
30
+ Optional: "optional",
31
+ Required: "required",
32
+ Hidden: "hidden",
33
+ } as const;
34
+ export type EvaluatorReasoningModes = ClosedEnum<
35
+ typeof EvaluatorReasoningModes
36
+ >;
37
+
38
+ export const EvaluatorReasoningModes$zodSchema = z.enum([
39
+ "optional",
40
+ "required",
41
+ "hidden",
42
+ ]);
43
+
29
44
  export type CreateJobsRequest = {
30
45
  annotator_ids: Array<string>;
31
46
  item_ids?: Array<string> | undefined;
@@ -34,6 +49,10 @@ export type CreateJobsRequest = {
34
49
  evaluator_ids?: Array<string> | null | undefined;
35
50
  comments_enabled?: boolean | undefined;
36
51
  reasoning_mode?: ReasoningMode | undefined;
52
+ evaluator_reasoning_modes?:
53
+ | { [k: string]: EvaluatorReasoningModes }
54
+ | null
55
+ | undefined;
37
56
  };
38
57
 
39
58
  export const CreateJobsRequest$zodSchema: z.ZodType<CreateJobsRequest> = z
@@ -47,6 +66,12 @@ export const CreateJobsRequest$zodSchema: z.ZodType<CreateJobsRequest> = z
47
66
  evaluator_ids: z.array(z.string()).nullable().optional().describe(
48
67
  "Subset of the task's linked evaluators to show in these jobs. Must be a subset of the current links, an empty list gives a 400. Applies to every annotator's job. Omit (`None`) to snapshot every linked evaluator",
49
68
  ),
69
+ evaluator_reasoning_modes: z.record(
70
+ z.string(),
71
+ EvaluatorReasoningModes$zodSchema,
72
+ ).nullable().optional().describe(
73
+ "Map of evaluator ID to how the labelling form treats the reasoning box for that evaluator, with the same values as `reasoning_mode`. Every ID must be an evaluator on these jobs, otherwise you get a 400. An evaluator left out uses `reasoning_mode`",
74
+ ),
50
75
  item_ids: z.array(z.string()).optional().describe(
51
76
  "Item IDs to assign. **Required when `select_all=false`**. Ignored when `select_all=true`",
52
77
  ),
package/src/models/evaluatorversioncompact.ts CHANGED
@@ -26,7 +26,7 @@ export const EvaluatorVersionCompact$zodSchema: z.ZodType<
26
26
  "The model that runs the judge, named the way its provider does, for example `openai/gpt-4.1` or `anthropic/claude-sonnet-4`",
27
27
  ),
28
28
  output_config: OutputConfig$zodSchema.nullable().optional().describe(
29
- "The scale points and their labels. Required for a `rating` evaluator. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own",
29
+ "The rows of the rating scale, each a number and its label. Required for a `rating` evaluator. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own",
30
30
  ),
31
31
  system_prompt: z.string().nullable().optional().describe(
32
32
  "Judge system prompt, with `{{\"{{\"}}variable}}` placeholders unrendered",
package/src/models/evaluatorversioncreate.ts CHANGED
@@ -23,7 +23,7 @@ export const EvaluatorVersionCreate$zodSchema: z.ZodType<
23
23
  "The model that runs the judge, named the way its provider does, for example `openai/gpt-4.1` or `anthropic/claude-sonnet-4`",
24
24
  ),
25
25
  output_config: OutputConfig$zodSchema.nullable().optional().describe(
26
- "The scale points and their labels. Required for a `rating` evaluator. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own",
26
+ "The rows of the rating scale, each a number and its label. Required for a `rating` evaluator. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own",
27
27
  ),
28
28
  system_prompt: z.string().describe(
29
29
  "Judge system prompt. May contain `{{\"{{\"}}variable}}` placeholders",
package/src/models/evaluatorversioncreaterequest.ts CHANGED
@@ -27,7 +27,7 @@ export const EvaluatorVersionCreateRequest$zodSchema: z.ZodType<
27
27
  "When `true`, immediately point the evaluator's live version at this new version",
28
28
  ),
29
29
  output_config: OutputConfig$zodSchema.nullable().optional().describe(
30
- "The scale points and their labels. Required for a `rating` evaluator. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own",
30
+ "The rows of the rating scale, each a number and its label. Required for a `rating` evaluator. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own",
31
31
  ),
32
32
  system_prompt: z.string().describe(
33
33
  "Judge system prompt. May contain `{{\"{{\"}}variable}}` placeholders",
package/src/models/outputconfig.ts CHANGED
@@ -14,6 +14,6 @@ export type OutputConfig = {
14
14
 
15
15
  export const OutputConfig$zodSchema: z.ZodType<OutputConfig> = z.object({
16
16
  scale: z.array(OutputScaleEntry$zodSchema).nullable().optional().describe(
17
- "The ordered scale points that make up the rubric, each with its label",
17
+ "The rows of the rating scale, each a number and its label. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number",
18
18
  ),
19
19
  });
package/src/models/outputscaleentry.ts CHANGED
@@ -20,14 +20,14 @@ export type OutputScaleEntry = {
20
20
  export const OutputScaleEntry$zodSchema: z.ZodType<OutputScaleEntry> = z.object(
21
21
  {
22
22
  color: z.string().nullable().optional().describe(
23
- "Color to show for this level. Omit for the default",
23
+ "Color to show for this row. Omit for the default",
24
24
  ),
25
25
  description: z.string().nullable().optional().describe(
26
- "Rubric text for this level, added to the judge prompt. Omit to leave this level undescribed",
26
+ "Rubric text for this row, added to the judge prompt. Omit to leave this row undescribed",
27
27
  ),
28
- name: z.string().describe("Short label for this scale point"),
28
+ name: z.string().describe("Short label for this row"),
29
29
  value: z.any().optional().describe(
30
- "The value for this scale point. Use a boolean for a `binary` evaluator, a number for a `rating` one",
30
+ "The value for this row. Use a boolean for a `binary` evaluator, a number for a `rating` one. For a rating evaluator, give the label for the worst result the lowest number and the label for the best result the highest number",
31
31
  ),
32
32
  },
33
33
  ).describe(