@dalmia/calibrate-mcp 0.0.69 → 0.0.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/bin/mcp-server.js +52 -15
  2. package/bin/mcp-server.js.map +12 -11
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/chatturn.d.ts +4 -0
  9. package/esm/models/chatturn.d.ts.map +1 -1
  10. package/esm/models/chatturn.js +4 -0
  11. package/esm/models/chatturn.js.map +1 -1
  12. package/esm/models/itempayload.d.ts +19 -3
  13. package/esm/models/itempayload.d.ts.map +1 -1
  14. package/esm/models/itempayload.js +26 -3
  15. package/esm/models/itempayload.js.map +1 -1
  16. package/esm/models/traceingest.d.ts +2 -2
  17. package/esm/models/traceingest.d.ts.map +1 -1
  18. package/esm/models/traceingest.js +1 -1
  19. package/esm/models/traceingest.js.map +1 -1
  20. package/esm/models/turntext.d.ts +13 -0
  21. package/esm/models/turntext.d.ts.map +1 -0
  22. package/esm/models/turntext.js +10 -0
  23. package/esm/models/turntext.js.map +1 -0
  24. package/package.json +1 -1
  25. package/src/landing-page.ts +1 -1
  26. package/src/lib/config.ts +2 -2
  27. package/src/mcp-server/mcp-server.ts +1 -1
  28. package/src/mcp-server/server.ts +1 -1
  29. package/src/models/chatturn.ts +10 -0
  30. package/src/models/itempayload.ts +51 -6
  31. package/src/models/traceingest.ts +2 -2
  32. package/src/models/turntext.ts +28 -0
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.69/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.70/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.69";
38
+ readonly sdkVersion: "0.0.70";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.69 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.70 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.69",
30
+ sdkVersion: "0.0.70",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.69 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.70 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.69",
21
+ currentVersion: "0.0.70",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -51,7 +51,7 @@ import { tool$tracesCreate } from "./tools/tracesCreate.js";
51
51
  export function createMCPServer(deps) {
52
52
  const server = new McpServer({
53
53
  name: "CalibrateMcp",
54
- version: "0.0.69",
54
+ version: "0.0.70",
55
55
  });
56
56
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
57
57
  security: deps.security,
package/esm/models/chatturn.d.ts CHANGED
@@ -5,6 +5,10 @@ import * as z from "zod";
5
5
  export type ChatTurn = {
6
6
  role: string;
7
7
  content?: string | null | undefined;
8
+ label?: string | null | undefined;
9
+ versions?: {
10
+ [k: string]: string;
11
+ } | null | undefined;
8
12
  AdditionalProperties?: {
9
13
  [k: string]: any;
10
14
  };
package/esm/models/chatturn.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"chatturn.d.ts","sourceRoot":"","sources":["../../src/models/chatturn.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,oBAAoB,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC7C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAalD,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAYrD,CAAC"}
1
+ {"version":3,"file":"chatturn.d.ts","sourceRoot":"","sources":["../../src/models/chatturn.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,oBAAoB,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC7C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAmBlD,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAcrD,CAAC"}
package/esm/models/chatturn.js CHANGED
@@ -5,12 +5,16 @@ import * as z from "zod";
5
5
  import { collectExtraKeys as collectExtraKeys$ } from "../lib/schemas.js";
6
6
  export const ChatTurn$zodSchema = collectExtraKeys$(z.object({
7
7
  content: z.string().nullable().optional().describe("Message text. Omit for turns that only carry tool calls"),
8
+ label: z.string().nullable().optional().describe("Display name for the text in `content`, shown to annotators"),
8
9
  role: z.string().describe("Message author role in the conversation history"),
10
+ versions: z.record(z.string(), z.string()).nullable().optional().describe("Other forms of the same text, such as a translation or a human rewrite, keyed by the name annotators see for each. Shown to annotators only"),
9
11
  }).describe("One message in a conversation, in OpenAI chat format").catchall(z.any()), "AdditionalProperties", true);
10
12
  export const ChatTurn$zodSchemaOutbound = z.object({
11
13
  AdditionalProperties: z.record(z.string(), z.any()).optional(),
12
14
  content: z.nullable(z.string()).optional(),
15
+ label: z.nullable(z.string()).optional(),
13
16
  role: z.string(),
17
+ versions: z.nullable(z.record(z.string(), z.string())).optional(),
14
18
  }).describe("One message in a conversation, in OpenAI chat format").transform((v) => {
15
19
  const { AdditionalProperties, ...rest } = v;
16
20
  return {
package/esm/models/chatturn.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"chatturn.js","sourceRoot":"","sources":["../../src/models/chatturn.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,gBAAgB,IAAI,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAW1E,MAAM,CAAC,MAAM,kBAAkB,GAAwB,iBAAiB,CACtE,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,yDAAyD,CAC1D;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,iDAAiD,CAClD;CACF,CAAC,CAAC,QAAQ,CAAC,sDAAsD,CAAC,CAAC,QAAQ,CAC1E,CAAC,CAAC,GAAG,EAAE,CACR,EACD,sBAAsB,EACtB,IAAI,CACL,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAmB,CAAC,CAAC,MAAM,CAAC;IACjE,oBAAoB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9D,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;CACjB,CAAC,CAAC,QAAQ,CAAC,sDAAsD,CAAC,CAAC,SAAS,CAC3E,CAAC,CAAC,EAAE,EAAE;IACJ,MAAM,EAAE,oBAAoB,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO;QACL,GAAG,IAAI;QACP,GAAG,oBAAoB;KACxB,CAAC;AACJ,CAAC,CACF,CAAC"}
1
+ {"version":3,"file":"chatturn.js","sourceRoot":"","sources":["../../src/models/chatturn.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,gBAAgB,IAAI,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAa1E,MAAM,CAAC,MAAM,kBAAkB,GAAwB,iBAAiB,CACtE,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,yDAAyD,CAC1D;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,6DAA6D,CAC9D;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,iDAAiD,CAClD;IACD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,6IAA6I,CAC9I;CACF,CAAC,CAAC,QAAQ,CAAC,sDAAsD,CAAC,CAAC,QAAQ,CAC1E,CAAC,CAAC,GAAG,EAAE,CACR,EACD,sBAAsB,EACtB,IAAI,CACL,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAmB,CAAC,CAAC,MAAM,CAAC;IACjE,oBAAoB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9D,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;CAClE,CAAC,CAAC,QAAQ,CAAC,sDAAsD,CAAC,CAAC,SAAS,CAC3E,CAAC,CAAC,EAAE,EAAE;IACJ,MAAM,EAAE,oBAAoB,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO;QACL,GAAG,IAAI;QACP,GAAG,oBAAoB;KACxB,CAAC;AACJ,CAAC,CACF,CAAC"}
package/esm/models/itempayload.d.ts CHANGED
@@ -1,6 +1,22 @@
1
1
  import * as z from "zod";
2
2
  import { ChatTurn } from "./chatturn.js";
3
3
  import { ToolCall } from "./toolcall.js";
4
+ import { TurnText } from "./turntext.js";
5
+ /**
6
+ * The reply to judge. An empty string when the reply was a tool call. For `llm` tasks
7
+ */
8
+ export type AgentResponseUnion = TurnText | string;
9
+ export declare const AgentResponseUnion$zodSchema: z.ZodType<AgentResponseUnion>;
10
+ /**
11
+ * The standalone input the agent was given. For `llm-general` tasks
12
+ */
13
+ export type ItemPayloadInput = TurnText | string;
14
+ export declare const ItemPayloadInput$zodSchema: z.ZodType<ItemPayloadInput>;
15
+ /**
16
+ * The output to judge. For `llm-general` tasks
17
+ */
18
+ export type Output = TurnText | string;
19
+ export declare const Output$zodSchema: z.ZodType<Output>;
4
20
  /**
5
21
  * What an annotation item holds. Which fields apply depends on the task type
6
22
  */
@@ -12,9 +28,9 @@ export type ItemPayload = {
12
28
  text?: string | null | undefined;
13
29
  audio_path?: string | null | undefined;
14
30
  chat_history?: Array<ChatTurn> | null | undefined;
15
- agent_response?: string | null | undefined;
16
- input?: string | null | undefined;
17
- output?: string | null | undefined;
31
+ agent_response?: TurnText | string | null | undefined;
32
+ input?: TurnText | string | null | undefined;
33
+ output?: TurnText | string | null | undefined;
18
34
  transcript?: Array<ChatTurn> | null | undefined;
19
35
  tool_calls?: Array<ToolCall> | null | undefined;
20
36
  expected_tool_calls?: Array<ToolCall> | null | undefined;
package/esm/models/itempayload.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"itempayload.d.ts","sourceRoot":"","sources":["../../src/models/itempayload.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAsB,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAsB,MAAM,eAAe,CAAC;AAE7D;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,YAAY,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAClD,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC3C,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,UAAU,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,mBAAmB,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACzD,iBAAiB,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,mBAAmB,CAAC,EAChB;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG;YAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAA;KAAE,GACrC,IAAI,GACJ,SAAS,CAAC;CACf,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAkDxD,CAAC"}
1
+ {"version":3,"file":"itempayload.d.ts","sourceRoot":"","sources":["../../src/models/itempayload.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAsB,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAsB,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAsB,MAAM,eAAe,CAAC;AAE7D;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEnD,eAAO,MAAM,4BAA4B,EAAE,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAMpE,CAAC;AAEJ;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEjD,eAAO,MAAM,0BAA0B,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAKlE,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvC,eAAO,MAAM,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAGY,CAAC;AAE5D;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,YAAY,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAClD,cAAc,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,KAAK,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,MAAM,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,UAAU,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,UAAU,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,mBAAmB,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACzD,iBAAiB,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,mBAAmB,CAAC,EAChB;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG;YAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAA;KAAE,GACrC,IAAI,GACJ,SAAS,CAAC;CACf,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CA2DxD,CAAC"}
package/esm/models/itempayload.js CHANGED
@@ -4,9 +4,26 @@
4
4
  import * as z from "zod";
5
5
  import { ChatTurn$zodSchema } from "./chatturn.js";
6
6
  import { ToolCall$zodSchema } from "./toolcall.js";
7
+ import { TurnText$zodSchema } from "./turntext.js";
8
+ export const AgentResponseUnion$zodSchema = z
9
+ .union([
10
+ TurnText$zodSchema,
11
+ z.string(),
12
+ ]).describe("The reply to judge. An empty string when the reply was a tool call. For `llm` tasks");
13
+ export const ItemPayloadInput$zodSchema = z.union([
14
+ TurnText$zodSchema,
15
+ z.string(),
16
+ ]).describe("The standalone input the agent was given. For `llm-general` tasks");
17
+ export const Output$zodSchema = z.union([
18
+ TurnText$zodSchema,
19
+ z.string(),
20
+ ]).describe("The output to judge. For `llm-general` tasks");
7
21
  export const ItemPayload$zodSchema = z.object({
8
22
  actual_tool_calls: z.array(ToolCall$zodSchema).nullable().optional().describe("Tool calls the agent made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks"),
9
- agent_response: z.string().nullable().optional().describe("The reply to judge. An empty string when the reply was a tool call. For `llm` tasks"),
23
+ agent_response: z.union([
24
+ TurnText$zodSchema,
25
+ z.string(),
26
+ ]).nullable().optional().describe("The reply to judge. An empty string when the reply was a tool call. For `llm` tasks"),
10
27
  audio_path: z.string().nullable().optional().describe("Storage key or `s3://` URI of the audio clip. For `tts` tasks"),
11
28
  chat_history: z.array(ChatTurn$zodSchema).nullable().optional().describe("The conversation up to the reply being judged, ending at the user turn. For `llm` tasks"),
12
29
  description: z.string().nullable().optional().describe("A description of the item. For every task type"),
@@ -14,9 +31,15 @@ export const ItemPayload$zodSchema = z.object({
14
31
  .nullable().optional().describe("Values for each evaluator's `{{\"{{\"}}variable}}` placeholders, keyed by evaluator ID. For `llm`, `llm-general` and `conversation` tasks"),
15
32
  expected_tool_calls: z.array(ToolCall$zodSchema).nullable().optional()
16
33
  .describe("Tool calls the agent should have made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks"),
17
- input: z.string().nullable().optional().describe("The standalone input the agent was given. For `llm-general` tasks"),
34
+ input: z.union([
35
+ TurnText$zodSchema,
36
+ z.string(),
37
+ ]).nullable().optional().describe("The standalone input the agent was given. For `llm-general` tasks"),
18
38
  name: z.string().describe("Name of the item, unique within the task. For every task type"),
19
- output: z.string().nullable().optional().describe("The output to judge. For `llm-general` tasks"),
39
+ output: z.union([
40
+ TurnText$zodSchema,
41
+ z.string(),
42
+ ]).nullable().optional().describe("The output to judge. For `llm-general` tasks"),
20
43
  predicted_transcript: z.string().nullable().optional().describe("The transcript to judge. For `stt` tasks"),
21
44
  reference_transcript: z.string().nullable().optional().describe("The correct transcript. For `stt` tasks"),
22
45
  text: z.string().nullable().optional().describe("The text that was spoken. For `tts` tasks"),
package/esm/models/itempayload.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"itempayload.js","sourceRoot":"","sources":["../../src/models/itempayload.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAY,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAY,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA0B7D,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC3E,gIAAgI,CACjI;IACD,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvD,qFAAqF,CACtF;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,+DAA+D,CAChE;IACD,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,yFAAyF,CAC1F;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,gDAAgD,CACjD;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;SACrE,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7B,2IAA2I,CAC5I;IACH,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACnE,QAAQ,CACP,4IAA4I,CAC7I;IACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,mEAAmE,CACpE;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,+DAA+D,CAChE;IACD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/C,8CAA8C,CAC/C;IACD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,0CAA0C,CAC3C;IACD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,yCAAyC,CAC1C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,2CAA2C,CAC5C;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,6KAA6K,CAC9K;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,0DAA0D,CAC3D;CACF,CAAC,CAAC,QAAQ,CACT,4EAA4E,CAC7E,CAAC"}
1
+ {"version":3,"file":"itempayload.js","sourceRoot":"","sources":["../../src/models/itempayload.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAY,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAY,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAY,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAO7D,MAAM,CAAC,MAAM,4BAA4B,GAAkC,CAAC;KACzE,KAAK,CAAC;IACL,kBAAkB;IAClB,CAAC,CAAC,MAAM,EAAE;CACX,CAAC,CAAC,QAAQ,CACT,qFAAqF,CACtF,CAAC;AAOJ,MAAM,CAAC,MAAM,0BAA0B,GAAgC,CAAC,CAAC,KAAK,CAAC;IAC7E,kBAAkB;IAClB,CAAC,CAAC,MAAM,EAAE;CACX,CAAC,CAAC,QAAQ,CACT,mEAAmE,CACpE,CAAC;AAOF,MAAM,CAAC,MAAM,gBAAgB,GAAsB,CAAC,CAAC,KAAK,CAAC;IACzD,kBAAkB;IAClB,CAAC,CAAC,MAAM,EAAE;CACX,CAAC,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAAC;AA0B5D,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC3E,gIAAgI,CACjI;IACD,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC;QACtB,kBAAkB;QAClB,CAAC,CAAC,MAAM,EAAE;KACX,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/B,qFAAqF,CACtF;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,+DAA+D,CAChE;IACD,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,yFAAyF,CAC1F;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,gDAAgD,CACjD;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;SACrE,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7B,2IAA2I,CAC5I;IACH,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACnE,QAAQ,CACP,4IAA4I,CAC7I;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;QACb,kBAAkB;QAClB,CAAC,CAAC,MAAM,EAAE;KACX,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/B,mEAAmE,CACpE;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,+DAA+D,CAChE;IACD,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC;QACd,kBAAkB;QAClB,CAAC,CAAC,MAAM,EAAE;KACX,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/B,8CAA8C,CAC/C;IACD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,0CAA0C,CAC3C;IACD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,yCAAyC,CAC1C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,2CAA2C,CAC5C;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,6KAA6K,CAC9K;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,0DAA0D,CAC3D;CACF,CAAC,CAAC,QAAQ,CACT,4EAA4E,CAC7E,CAAC"}
package/esm/models/traceingest.d.ts CHANGED
@@ -5,8 +5,8 @@ import { TraceTurn } from "./traceturn.js";
5
5
  /**
6
6
  * What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format
7
7
  */
8
- export type Input = string | Array<TraceTurn>;
9
- export declare const Input$zodSchema: z.ZodType<Input>;
8
+ export type TraceIngestInput = string | Array<TraceTurn>;
9
+ export declare const TraceIngestInput$zodSchema: z.ZodType<TraceIngestInput>;
10
10
  export type TraceIngest = {
11
11
  agent_id: string;
12
12
  message_id?: string | null | undefined;
package/esm/models/traceingest.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"traceingest.d.ts","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAuB,MAAM,gBAAgB,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;AAE9C,eAAO,MAAM,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAK5C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAwBvD,CAAC"}
1
+ {"version":3,"file":"traceingest.d.ts","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAuB,MAAM,gBAAgB,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;AAEzD,eAAO,MAAM,0BAA0B,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAKlE,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAwBvD,CAAC"}
package/esm/models/traceingest.js CHANGED
@@ -5,7 +5,7 @@ import * as z from "zod";
5
5
  import { TraceMetadataEntry$zodSchema, } from "./tracemetadataentry.js";
6
6
  import { TraceOutput$zodSchema } from "./traceoutput.js";
7
7
  import { TraceTurn$zodSchema } from "./traceturn.js";
8
- export const Input$zodSchema = z.union([
8
+ export const TraceIngestInput$zodSchema = z.union([
9
9
  z.string(),
10
10
  z.array(TraceTurn$zodSchema),
11
11
  ]).describe("What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format");
package/esm/models/traceingest.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"traceingest.js","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAa,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOhE,MAAM,CAAC,MAAM,eAAe,GAAqB,CAAC,CAAC,KAAK,CAAC;IACvD,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;CAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN,CAAC;AAYF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC3B,4EAA4E,CAC7E;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,yGAAyG,CAC1G;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;QACb,CAAC,CAAC,MAAM,EAAE;QACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;KAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN;IACD,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,sJAAsJ,CACvJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,oGAAoG,CACrG;IACD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,+GAA+G,CAChH;IACH,MAAM,EAAE,qBAAqB;CAC9B,CAAC,CAAC"}
1
+ {"version":3,"file":"traceingest.js","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAa,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOhE,MAAM,CAAC,MAAM,0BAA0B,GAAgC,CAAC,CAAC,KAAK,CAAC;IAC7E,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;CAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN,CAAC;AAYF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC3B,4EAA4E,CAC7E;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,yGAAyG,CAC1G;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;QACb,CAAC,CAAC,MAAM,EAAE;QACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;KAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN;IACD,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,sJAAsJ,CACvJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,oGAAoG,CACrG;IACD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,+GAA+G,CAChH;IACH,MAAM,EAAE,qBAAqB;CAC9B,CAAC,CAAC"}
package/esm/models/turntext.d.ts ADDED
@@ -0,0 +1,13 @@
1
+ import * as z from "zod";
2
+ /**
3
+ * Text with a display name and other forms of it, sent in place of a plain string
4
+ */
5
+ export type TurnText = {
6
+ content: string;
7
+ label?: string | null | undefined;
8
+ versions?: {
9
+ [k: string]: string;
10
+ } | null | undefined;
11
+ };
12
+ export declare const TurnText$zodSchema: z.ZodType<TurnText>;
13
+ //# sourceMappingURL=turntext.d.ts.map
package/esm/models/turntext.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"turntext.d.ts","sourceRoot":"","sources":["../../src/models/turntext.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;CACvD,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAYlD,CAAC"}
package/esm/models/turntext.js ADDED
@@ -0,0 +1,10 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+ import * as z from "zod";
5
+ export const TurnText$zodSchema = z.object({
6
+ content: z.string().describe("The text the LLM judge reads and annotators see by default"),
7
+ label: z.string().nullable().optional().describe("Display name for the text in `content`, shown to annotators"),
8
+ versions: z.record(z.string(), z.string()).nullable().optional().describe("Other forms of the same text, such as a translation or a human rewrite, keyed by the name annotators see for each. Shown to annotators only"),
9
+ }).describe("Text with a display name and other forms of it, sent in place of a plain string");
10
+ //# sourceMappingURL=turntext.js.map
package/esm/models/turntext.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"turntext.js","sourceRoot":"","sources":["../../src/models/turntext.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAWzB,MAAM,CAAC,MAAM,kBAAkB,GAAwB,CAAC,CAAC,MAAM,CAAC;IAC9D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC1B,4DAA4D,CAC7D;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,6DAA6D,CAC9D;IACD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,6IAA6I,CAC9I;CACF,CAAC,CAAC,QAAQ,CACT,iFAAiF,CAClF,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.69",
3
+ "version": "0.0.70",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.69/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.70/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.69",
68
+ sdkVersion: "0.0.70",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.69 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.70 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.69",
24
+ currentVersion: "0.0.70",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -73,7 +73,7 @@ export function createMCPServer(deps: {
73
73
  }) {
74
74
  const server = new McpServer({
75
75
  name: "CalibrateMcp",
76
- version: "0.0.69",
76
+ version: "0.0.70",
77
77
  });
78
78
 
79
79
  const getClient = deps.getSDK || (() =>
package/src/models/chatturn.ts CHANGED
@@ -11,6 +11,8 @@ import { collectExtraKeys as collectExtraKeys$ } from "../lib/schemas.js";
11
11
  export type ChatTurn = {
12
12
  role: string;
13
13
  content?: string | null | undefined;
14
+ label?: string | null | undefined;
15
+ versions?: { [k: string]: string } | null | undefined;
14
16
  AdditionalProperties?: { [k: string]: any };
15
17
  };
16
18
 
@@ -19,9 +21,15 @@ export const ChatTurn$zodSchema: z.ZodType<ChatTurn> = collectExtraKeys$(
19
21
  content: z.string().nullable().optional().describe(
20
22
  "Message text. Omit for turns that only carry tool calls",
21
23
  ),
24
+ label: z.string().nullable().optional().describe(
25
+ "Display name for the text in `content`, shown to annotators",
26
+ ),
22
27
  role: z.string().describe(
23
28
  "Message author role in the conversation history",
24
29
  ),
30
+ versions: z.record(z.string(), z.string()).nullable().optional().describe(
31
+ "Other forms of the same text, such as a translation or a human rewrite, keyed by the name annotators see for each. Shown to annotators only",
32
+ ),
25
33
  }).describe("One message in a conversation, in OpenAI chat format").catchall(
26
34
  z.any(),
27
35
  ),
@@ -32,7 +40,9 @@ export const ChatTurn$zodSchema: z.ZodType<ChatTurn> = collectExtraKeys$(
32
40
  export const ChatTurn$zodSchemaOutbound: z.ZodType<any> = z.object({
33
41
  AdditionalProperties: z.record(z.string(), z.any()).optional(),
34
42
  content: z.nullable(z.string()).optional(),
43
+ label: z.nullable(z.string()).optional(),
35
44
  role: z.string(),
45
+ versions: z.nullable(z.record(z.string(), z.string())).optional(),
36
46
  }).describe("One message in a conversation, in OpenAI chat format").transform(
37
47
  (v) => {
38
48
  const { AdditionalProperties, ...rest } = v;
package/src/models/itempayload.ts CHANGED
@@ -5,6 +5,42 @@
5
5
  import * as z from "zod";
6
6
  import { ChatTurn, ChatTurn$zodSchema } from "./chatturn.js";
7
7
  import { ToolCall, ToolCall$zodSchema } from "./toolcall.js";
8
+ import { TurnText, TurnText$zodSchema } from "./turntext.js";
9
+
10
+ /**
11
+ * The reply to judge. An empty string when the reply was a tool call. For `llm` tasks
12
+ */
13
+ export type AgentResponseUnion = TurnText | string;
14
+
15
+ export const AgentResponseUnion$zodSchema: z.ZodType<AgentResponseUnion> = z
16
+ .union([
17
+ TurnText$zodSchema,
18
+ z.string(),
19
+ ]).describe(
20
+ "The reply to judge. An empty string when the reply was a tool call. For `llm` tasks",
21
+ );
22
+
23
+ /**
24
+ * The standalone input the agent was given. For `llm-general` tasks
25
+ */
26
+ export type ItemPayloadInput = TurnText | string;
27
+
28
+ export const ItemPayloadInput$zodSchema: z.ZodType<ItemPayloadInput> = z.union([
29
+ TurnText$zodSchema,
30
+ z.string(),
31
+ ]).describe(
32
+ "The standalone input the agent was given. For `llm-general` tasks",
33
+ );
34
+
35
+ /**
36
+ * The output to judge. For `llm-general` tasks
37
+ */
38
+ export type Output = TurnText | string;
39
+
40
+ export const Output$zodSchema: z.ZodType<Output> = z.union([
41
+ TurnText$zodSchema,
42
+ z.string(),
43
+ ]).describe("The output to judge. For `llm-general` tasks");
8
44
 
9
45
  /**
10
46
  * What an annotation item holds. Which fields apply depends on the task type
@@ -17,9 +53,9 @@ export type ItemPayload = {
17
53
  text?: string | null | undefined;
18
54
  audio_path?: string | null | undefined;
19
55
  chat_history?: Array<ChatTurn> | null | undefined;
20
- agent_response?: string | null | undefined;
21
- input?: string | null | undefined;
22
- output?: string | null | undefined;
56
+ agent_response?: TurnText | string | null | undefined;
57
+ input?: TurnText | string | null | undefined;
58
+ output?: TurnText | string | null | undefined;
23
59
  transcript?: Array<ChatTurn> | null | undefined;
24
60
  tool_calls?: Array<ToolCall> | null | undefined;
25
61
  expected_tool_calls?: Array<ToolCall> | null | undefined;
@@ -34,7 +70,10 @@ export const ItemPayload$zodSchema: z.ZodType<ItemPayload> = z.object({
34
70
  actual_tool_calls: z.array(ToolCall$zodSchema).nullable().optional().describe(
35
71
  "Tool calls the agent made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks",
36
72
  ),
37
- agent_response: z.string().nullable().optional().describe(
73
+ agent_response: z.union([
74
+ TurnText$zodSchema,
75
+ z.string(),
76
+ ]).nullable().optional().describe(
38
77
  "The reply to judge. An empty string when the reply was a tool call. For `llm` tasks",
39
78
  ),
40
79
  audio_path: z.string().nullable().optional().describe(
@@ -54,13 +93,19 @@ export const ItemPayload$zodSchema: z.ZodType<ItemPayload> = z.object({
54
93
  .describe(
55
94
  "Tool calls the agent should have made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks",
56
95
  ),
57
- input: z.string().nullable().optional().describe(
96
+ input: z.union([
97
+ TurnText$zodSchema,
98
+ z.string(),
99
+ ]).nullable().optional().describe(
58
100
  "The standalone input the agent was given. For `llm-general` tasks",
59
101
  ),
60
102
  name: z.string().describe(
61
103
  "Name of the item, unique within the task. For every task type",
62
104
  ),
63
- output: z.string().nullable().optional().describe(
105
+ output: z.union([
106
+ TurnText$zodSchema,
107
+ z.string(),
108
+ ]).nullable().optional().describe(
64
109
  "The output to judge. For `llm-general` tasks",
65
110
  ),
66
111
  predicted_transcript: z.string().nullable().optional().describe(
package/src/models/traceingest.ts CHANGED
@@ -13,9 +13,9 @@ import { TraceTurn, TraceTurn$zodSchema } from "./traceturn.js";
13
13
  /**
14
14
  * What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format
15
15
  */
16
- export type Input = string | Array<TraceTurn>;
16
+ export type TraceIngestInput = string | Array<TraceTurn>;
17
17
 
18
- export const Input$zodSchema: z.ZodType<Input> = z.union([
18
+ export const TraceIngestInput$zodSchema: z.ZodType<TraceIngestInput> = z.union([
19
19
  z.string(),
20
20
  z.array(TraceTurn$zodSchema),
21
21
  ]).describe(
package/src/models/turntext.ts ADDED
@@ -0,0 +1,28 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+
5
+ import * as z from "zod";
6
+
7
+ /**
8
+ * Text with a display name and other forms of it, sent in place of a plain string
9
+ */
10
+ export type TurnText = {
11
+ content: string;
12
+ label?: string | null | undefined;
13
+ versions?: { [k: string]: string } | null | undefined;
14
+ };
15
+
16
+ export const TurnText$zodSchema: z.ZodType<TurnText> = z.object({
17
+ content: z.string().describe(
18
+ "The text the LLM judge reads and annotators see by default",
19
+ ),
20
+ label: z.string().nullable().optional().describe(
21
+ "Display name for the text in `content`, shown to annotators",
22
+ ),
23
+ versions: z.record(z.string(), z.string()).nullable().optional().describe(
24
+ "Other forms of the same text, such as a translation or a human rewrite, keyed by the name annotators see for each. Shown to annotators only",
25
+ ),
26
+ }).describe(
27
+ "Text with a display name and other forms of it, sent in place of a plain string",
28
+ );