@meetkai/mka1 0.51.34 → 0.51.36

package/src/mcp-server/server.ts

@@ -147,6 +147,7 @@ import { tool$llmEvalsDeleteSchedule } from "./tools/llmEvalsDeleteSchedule.js";
 import { tool$llmEvalsDeleteSuite } from "./tools/llmEvalsDeleteSuite.js";
 import { tool$llmEvalsGetArtifacts } from "./tools/llmEvalsGetArtifacts.js";
 import { tool$llmEvalsGetRun } from "./tools/llmEvalsGetRun.js";
+import { tool$llmEvalsGetSampleAudio } from "./tools/llmEvalsGetSampleAudio.js";
 import { tool$llmEvalsGetSchedule } from "./tools/llmEvalsGetSchedule.js";
 import { tool$llmEvalsGetSuite } from "./tools/llmEvalsGetSuite.js";
 import { tool$llmEvalsGetSuiteVersion } from "./tools/llmEvalsGetSuiteVersion.js";
@@ -155,6 +156,7 @@ import { tool$llmEvalsListRuns } from "./tools/llmEvalsListRuns.js";
 import { tool$llmEvalsListSamples } from "./tools/llmEvalsListSamples.js";
 import { tool$llmEvalsListScheduleRuns } from "./tools/llmEvalsListScheduleRuns.js";
 import { tool$llmEvalsListSchedules } from "./tools/llmEvalsListSchedules.js";
+import { tool$llmEvalsListSuiteLeaderboardRuns } from "./tools/llmEvalsListSuiteLeaderboardRuns.js";
 import { tool$llmEvalsListSuites } from "./tools/llmEvalsListSuites.js";
 import { tool$llmEvalsListSuiteVersions } from "./tools/llmEvalsListSuiteVersions.js";
 import { tool$llmEvalsRerunFailedSamples } from "./tools/llmEvalsRerunFailedSamples.js";
@@ -346,7 +348,7 @@ export function createMCPServer(deps: {
 }) {
   const server = new McpServer({
     name: "SDK",
-    version: "0.51.34",
+    version: "0.51.36",
   });
 
   const client = new SDKCore({
@@ -494,6 +496,7 @@ export function createMCPServer(deps: {
   tool(tool$llmEvalsCreateSuiteVersion);
   tool(tool$llmEvalsListSuiteVersions);
   tool(tool$llmEvalsGetSuiteVersion);
+  tool(tool$llmEvalsListSuiteLeaderboardRuns);
   tool(tool$llmEvalsCreateRun);
   tool(tool$llmEvalsListRuns);
   tool(tool$llmEvalsGetRun);
@@ -502,6 +505,7 @@ export function createMCPServer(deps: {
   tool(tool$llmEvalsRerunFailedSamples);
   tool(tool$llmEvalsRetryFailedRun);
   tool(tool$llmEvalsListSamples);
+  tool(tool$llmEvalsGetSampleAudio);
   tool(tool$llmEvalsGetArtifacts);
   tool(tool$llmEvalsImportHistoricalResults);
   tool(tool$llmEvalsCreateSchedule);

package/src/mcp-server/tools/llmEvalsGetSampleAudio.ts

@@ -0,0 +1,37 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { llmEvalsGetSampleAudio } from "../../funcs/llmEvalsGetSampleAudio.js";
+import * as operations from "../../models/operations/index.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: operations.GetEvalSampleAudioRequest$inboundSchema,
+};
+
+export const tool$llmEvalsGetSampleAudio: ToolDefinition<typeof args> = {
+  name: "llm-evals-get-sample-audio",
+  description: `Get eval sample audio
+
+Returns \`{ object, audio, sample_index, task_id, model }\` for a single transcription sample — the clip reference (a base64 \`data:\` URI or a URL) read from the un-redacted \`dataset_row\` (list responses redact inline \`data:\` blobs), plus the resolved task/model. Prefers the task's configured \`audio_column\`, falling back to a heuristic scan. The clip differs across tasks, so pass \`task_id\` when a run has more than one transcription task; if the index spans multiple tasks and \`task_id\` is omitted, returns 400 rather than guessing. \`model\` is optional (the clip is identical across models) and only extends the index prefix. Returns 404 when the sample or its audio is absent, 413 when the inline clip is too large.`,
+  args,
+  tool: async (client, args, ctx) => {
+    const [result, apiCall] = await llmEvalsGetSampleAudio(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    const value = result.value;
+
+    return formatResult(value, apiCall);
+  },
+};

package/src/mcp-server/tools/llmEvalsListSuiteLeaderboardRuns.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { llmEvalsListSuiteLeaderboardRuns } from "../../funcs/llmEvalsListSuiteLeaderboardRuns.js";
+import * as operations from "../../models/operations/index.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: operations.ListEvalSuiteLeaderboardRunsRequest$inboundSchema,
+};
+
+export const tool$llmEvalsListSuiteLeaderboardRuns: ToolDefinition<
+  typeof args
+> = {
+  name: "llm-evals-list-suite-leaderboard-runs",
+  description: `List leaderboard runs for an eval suite
+
+Returns each ranking group's most recent completed runs for a suite in one request, with the group assignments in \`groups\`. A group is (model, targeted task) for task_ids-restricted runs and (model) for unrestricted runs (task_id null). Build boards from \`groups\`, not from run.models/run.task_ids — a returned run may name models or tasks whose groups it did not rank in. \`truncated: true\` means a server ceiling trimmed the aggregate; whole runs are kept in (best rank, first model, first task, id) order, so every group's newest runs survive before any group's fallbacks and no group is dropped for being old. Resume a truncated board by passing \`next_cursor\` back as \`cursor\` and unioning the responses. See the evals docs for grouping rationale and cost characteristics.`,
+  args,
+  tool: async (client, args, ctx) => {
+    const [result, apiCall] = await llmEvalsListSuiteLeaderboardRuns(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    const value = result.value;
+
+    return formatResult(value, apiCall);
+  },
+};

package/src/models/components/evalleaderboardrungroup.ts

@@ -0,0 +1,86 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type EvalLeaderboardRunGroup = {
+  /**
+   * The group's model.
+   */
+  model: string;
+  /**
+   * The group's targeted task, or null for the per-model covers-everything group that unrestricted runs (no task_ids) rank in — those runs cover every task of their version.
+   */
+  taskId: string | null;
+  /**
+   * 1 is the group's newest completed run. Ranks order runs within a group but are not slot indexes: under truncation a group's surviving ranks may be non-contiguous.
+   */
+  rank: number;
+  /**
+   * The run holding this rank; the full run object is in `data`.
+   */
+  runId: string;
+};
+
+/** @internal */
+export const EvalLeaderboardRunGroup$inboundSchema: z.ZodType<
+  EvalLeaderboardRunGroup,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  model: z.string(),
+  task_id: z.nullable(z.string()),
+  rank: z.number().int(),
+  run_id: z.string(),
+}).transform((v) => {
+  return remap$(v, {
+    "task_id": "taskId",
+    "run_id": "runId",
+  });
+});
+/** @internal */
+export type EvalLeaderboardRunGroup$Outbound = {
+  model: string;
+  task_id: string | null;
+  rank: number;
+  run_id: string;
+};
+
+/** @internal */
+export const EvalLeaderboardRunGroup$outboundSchema: z.ZodType<
+  EvalLeaderboardRunGroup$Outbound,
+  z.ZodTypeDef,
+  EvalLeaderboardRunGroup
+> = z.object({
+  model: z.string(),
+  taskId: z.nullable(z.string()),
+  rank: z.number().int(),
+  runId: z.string(),
+}).transform((v) => {
+  return remap$(v, {
+    taskId: "task_id",
+    runId: "run_id",
+  });
+});
+
+export function evalLeaderboardRunGroupToJSON(
+  evalLeaderboardRunGroup: EvalLeaderboardRunGroup,
+): string {
+  return JSON.stringify(
+    EvalLeaderboardRunGroup$outboundSchema.parse(evalLeaderboardRunGroup),
+  );
+}
+export function evalLeaderboardRunGroupFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalLeaderboardRunGroup, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalLeaderboardRunGroup$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalLeaderboardRunGroup' from JSON`,
+  );
+}

package/src/models/components/evalsampleaudioobject.ts

@@ -0,0 +1,90 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type EvalSampleAudioObject = {
+  object: "eval.sample.audio";
+  /**
+   * Playable clip reference: a base64 `data:` URI or an http(s) URL.
+   */
+  audio: string;
+  /**
+   * Echoes the requested sample index so the client has a complete cache key without tracking it.
+   */
+  sampleIndex: number;
+  /**
+   * The task the clip was resolved from (the clip differs across tasks).
+   */
+  taskId: string;
+  /**
+   * The model row the clip was read from. The clip is identical across models within a task.
+   */
+  model: string;
+};
+
+/** @internal */
+export const EvalSampleAudioObject$inboundSchema: z.ZodType<
+  EvalSampleAudioObject,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  object: z.literal("eval.sample.audio"),
+  audio: z.string(),
+  sample_index: z.number().int(),
+  task_id: z.string(),
+  model: z.string(),
+}).transform((v) => {
+  return remap$(v, {
+    "sample_index": "sampleIndex",
+    "task_id": "taskId",
+  });
+});
+/** @internal */
+export type EvalSampleAudioObject$Outbound = {
+  object: "eval.sample.audio";
+  audio: string;
+  sample_index: number;
+  task_id: string;
+  model: string;
+};
+
+/** @internal */
+export const EvalSampleAudioObject$outboundSchema: z.ZodType<
+  EvalSampleAudioObject$Outbound,
+  z.ZodTypeDef,
+  EvalSampleAudioObject
+> = z.object({
+  object: z.literal("eval.sample.audio"),
+  audio: z.string(),
+  sampleIndex: z.number().int(),
+  taskId: z.string(),
+  model: z.string(),
+}).transform((v) => {
+  return remap$(v, {
+    sampleIndex: "sample_index",
+    taskId: "task_id",
+  });
+});
+
+export function evalSampleAudioObjectToJSON(
+  evalSampleAudioObject: EvalSampleAudioObject,
+): string {
+  return JSON.stringify(
+    EvalSampleAudioObject$outboundSchema.parse(evalSampleAudioObject),
+  );
+}
+export function evalSampleAudioObjectFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalSampleAudioObject, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalSampleAudioObject$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalSampleAudioObject' from JSON`,
+  );
+}

package/src/models/components/index.ts

@@ -198,6 +198,7 @@ export * from "./evaldatasetformat.js";
 export * from "./evalfewshotconfig.js";
 export * from "./evalgenerationconfig.js";
 export * from "./evalhistoricalimportobject.js";
+export * from "./evalleaderboardrungroup.js";
 export * from "./evalmetric.js";
 export * from "./evaloutputextraction.js";
 export * from "./evalpythongrader.js";
@@ -207,6 +208,7 @@ export * from "./evalpythongradermodelaccess.js";
 export * from "./evalpythonpreprocessor.js";
 export * from "./evalrundeletedobject.js";
 export * from "./evalrunstatus.js";
+export * from "./evalsampleaudioobject.js";
 export * from "./evalsamplelist.js";
 export * from "./evalsampleobject.js";
 export * from "./evalsamplestatus.js";

package/src/models/operations/getevalsampleaudio.ts

@@ -0,0 +1,85 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type GetEvalSampleAudioRequest = {
+  runId: string;
+  sampleIndex: number;
+  model?: string | undefined;
+  taskId?: string | undefined;
+  /**
+   * Optional external end-user identifier forwarded by the API gateway.
+   */
+  xOnBehalfOf?: string | undefined;
+};
+
+/** @internal */
+export const GetEvalSampleAudioRequest$inboundSchema: z.ZodType<
+  GetEvalSampleAudioRequest,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  run_id: z.string(),
+  sample_index: z.number().int(),
+  model: z.string().optional(),
+  task_id: z.string().optional(),
+  "X-On-Behalf-Of": z.string().optional(),
+}).transform((v) => {
+  return remap$(v, {
+    "run_id": "runId",
+    "sample_index": "sampleIndex",
+    "task_id": "taskId",
+    "X-On-Behalf-Of": "xOnBehalfOf",
+  });
+});
+/** @internal */
+export type GetEvalSampleAudioRequest$Outbound = {
+  run_id: string;
+  sample_index: number;
+  model?: string | undefined;
+  task_id?: string | undefined;
+  "X-On-Behalf-Of"?: string | undefined;
+};
+
+/** @internal */
+export const GetEvalSampleAudioRequest$outboundSchema: z.ZodType<
+  GetEvalSampleAudioRequest$Outbound,
+  z.ZodTypeDef,
+  GetEvalSampleAudioRequest
+> = z.object({
+  runId: z.string(),
+  sampleIndex: z.number().int(),
+  model: z.string().optional(),
+  taskId: z.string().optional(),
+  xOnBehalfOf: z.string().optional(),
+}).transform((v) => {
+  return remap$(v, {
+    runId: "run_id",
+    sampleIndex: "sample_index",
+    taskId: "task_id",
+    xOnBehalfOf: "X-On-Behalf-Of",
+  });
+});
+
+export function getEvalSampleAudioRequestToJSON(
+  getEvalSampleAudioRequest: GetEvalSampleAudioRequest,
+): string {
+  return JSON.stringify(
+    GetEvalSampleAudioRequest$outboundSchema.parse(getEvalSampleAudioRequest),
+  );
+}
+export function getEvalSampleAudioRequestFromJSON(
+  jsonString: string,
+): SafeParseResult<GetEvalSampleAudioRequest, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => GetEvalSampleAudioRequest$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'GetEvalSampleAudioRequest' from JSON`,
+  );
+}

package/src/models/operations/index.ts

@@ -131,6 +131,7 @@ export * from "./getcurrency.js";
 export * from "./getembeddingsusage.js";
 export * from "./getevalartifacts.js";
 export * from "./getevalrun.js";
+export * from "./getevalsampleaudio.js";
 export * from "./getevalschedule.js";
 export * from "./getevalsuite.js";
 export * from "./getevalsuiteversion.js";
@@ -200,6 +201,7 @@ export * from "./listevalruns.js";
 export * from "./listevalsamples.js";
 export * from "./listevalscheduleruns.js";
 export * from "./listevalschedules.js";
+export * from "./listevalsuiteleaderboardruns.js";
 export * from "./listevalsuites.js";
 export * from "./listevalsuiteversions.js";
 export * from "./listfiles.js";

package/src/models/operations/listevalsamples.ts

@@ -27,6 +27,11 @@ export type ListEvalSamplesRequest = {
   taskId?: string | undefined;
   model?: string | undefined;
   status?: QueryParamStatus | undefined;
+  scoreMetric?: string | undefined;
+  scoreMin?: number | undefined;
+  scoreMax?: number | undefined;
+  includeDatasetRow?: boolean | undefined;
+  sampleIndex?: string | undefined;
   /**
    * Optional external end-user identifier forwarded by the API gateway.
    */
@@ -54,11 +59,21 @@ export const ListEvalSamplesRequest$inboundSchema: z.ZodType<
   task_id: z.string().optional(),
   model: z.string().optional(),
   status: QueryParamStatus$inboundSchema.optional(),
+  score_metric: z.string().optional(),
+  score_min: z.number().optional(),
+  score_max: z.number().optional(),
+  include_dataset_row: z.boolean().default(true),
+  sample_index: z.string().optional(),
   "X-On-Behalf-Of": z.string().optional(),
 }).transform((v) => {
   return remap$(v, {
     "run_id": "runId",
     "task_id": "taskId",
+    "score_metric": "scoreMetric",
+    "score_min": "scoreMin",
+    "score_max": "scoreMax",
+    "include_dataset_row": "includeDatasetRow",
+    "sample_index": "sampleIndex",
     "X-On-Behalf-Of": "xOnBehalfOf",
   });
 });
@@ -70,6 +85,11 @@ export type ListEvalSamplesRequest$Outbound = {
   task_id?: string | undefined;
   model?: string | undefined;
   status?: string | undefined;
+  score_metric?: string | undefined;
+  score_min?: number | undefined;
+  score_max?: number | undefined;
+  include_dataset_row: boolean;
+  sample_index?: string | undefined;
   "X-On-Behalf-Of"?: string | undefined;
 };
 
@@ -85,11 +105,21 @@ export const ListEvalSamplesRequest$outboundSchema: z.ZodType<
   taskId: z.string().optional(),
   model: z.string().optional(),
   status: QueryParamStatus$outboundSchema.optional(),
+  scoreMetric: z.string().optional(),
+  scoreMin: z.number().optional(),
+  scoreMax: z.number().optional(),
+  includeDatasetRow: z.boolean().default(true),
+  sampleIndex: z.string().optional(),
   xOnBehalfOf: z.string().optional(),
 }).transform((v) => {
   return remap$(v, {
     runId: "run_id",
     taskId: "task_id",
+    scoreMetric: "score_metric",
+    scoreMin: "score_min",
+    scoreMax: "score_max",
+    includeDatasetRow: "include_dataset_row",
+    sampleIndex: "sample_index",
     xOnBehalfOf: "X-On-Behalf-Of",
   });
 });