@meetkai/mka1 0.54.23 → 0.54.24

package/src/models/operations/cancelevalrun.ts

@@ -115,6 +115,22 @@ export type CancelEvalRunResponseBody = {
   error: { [k: string]: any } | null;
   artifactFileIds: Array<string>;
   metadata: { [k: string]: string };
+  /**
+   * Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
+   */
+  agentName: string | null;
+  /**
+   * Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
+   */
+  agentVersion: string | null;
+  /**
+   * Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
+   */
+  agentEffort: string | null;
+  /**
+   * Total spend for the run in USD. Null when the harness did not report cost.
+   */
+  costUsd: number | null;
   createdAt: number;
   startedAt: number | null;
   completedAt: number | null;
@@ -503,6 +519,10 @@ export const CancelEvalRunResponseBody$inboundSchema: z.ZodType<
   error: z.nullable(z.record(z.any())),
   artifact_file_ids: z.array(z.string()),
   metadata: z.record(z.string()),
+  agent_name: z.nullable(z.string()),
+  agent_version: z.nullable(z.string()),
+  agent_effort: z.nullable(z.string()),
+  cost_usd: z.nullable(z.number()),
   created_at: z.number().int(),
   started_at: z.nullable(z.number().int()),
   completed_at: z.nullable(z.number().int()),
@@ -520,6 +540,10 @@ export const CancelEvalRunResponseBody$inboundSchema: z.ZodType<
     "embedding_model": "embeddingModel",
     "request_counts": "requestCounts",
     "artifact_file_ids": "artifactFileIds",
+    "agent_name": "agentName",
+    "agent_version": "agentVersion",
+    "agent_effort": "agentEffort",
+    "cost_usd": "costUsd",
     "created_at": "createdAt",
     "started_at": "startedAt",
     "completed_at": "completedAt",
@@ -547,6 +571,10 @@ export type CancelEvalRunResponseBody$Outbound = {
   error: { [k: string]: any } | null;
   artifact_file_ids: Array<string>;
   metadata: { [k: string]: string };
+  agent_name: string | null;
+  agent_version: string | null;
+  agent_effort: string | null;
+  cost_usd: number | null;
   created_at: number;
   started_at: number | null;
   completed_at: number | null;
@@ -578,6 +606,10 @@ export const CancelEvalRunResponseBody$outboundSchema: z.ZodType<
   error: z.nullable(z.record(z.any())),
   artifactFileIds: z.array(z.string()),
   metadata: z.record(z.string()),
+  agentName: z.nullable(z.string()),
+  agentVersion: z.nullable(z.string()),
+  agentEffort: z.nullable(z.string()),
+  costUsd: z.nullable(z.number()),
   createdAt: z.number().int(),
   startedAt: z.nullable(z.number().int()),
   completedAt: z.nullable(z.number().int()),
@@ -595,6 +627,10 @@ export const CancelEvalRunResponseBody$outboundSchema: z.ZodType<
     embeddingModel: "embedding_model",
     requestCounts: "request_counts",
     artifactFileIds: "artifact_file_ids",
+    agentName: "agent_name",
+    agentVersion: "agent_version",
+    agentEffort: "agent_effort",
+    costUsd: "cost_usd",
     createdAt: "created_at",
     startedAt: "started_at",
     completedAt: "completed_at",

package/src/models/operations/createevalrun.ts

@@ -111,6 +111,22 @@ export type CreateEvalRunResponseBody = {
   error: { [k: string]: any } | null;
   artifactFileIds: Array<string>;
   metadata: { [k: string]: string };
+  /**
+   * Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
+   */
+  agentName: string | null;
+  /**
+   * Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
+   */
+  agentVersion: string | null;
+  /**
+   * Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
+   */
+  agentEffort: string | null;
+  /**
+   * Total spend for the run in USD. Null when the harness did not report cost.
+   */
+  costUsd: number | null;
   createdAt: number;
   startedAt: number | null;
   completedAt: number | null;
@@ -473,6 +489,10 @@ export const CreateEvalRunResponseBody$inboundSchema: z.ZodType<
   error: z.nullable(z.record(z.any())),
   artifact_file_ids: z.array(z.string()),
   metadata: z.record(z.string()),
+  agent_name: z.nullable(z.string()),
+  agent_version: z.nullable(z.string()),
+  agent_effort: z.nullable(z.string()),
+  cost_usd: z.nullable(z.number()),
   created_at: z.number().int(),
   started_at: z.nullable(z.number().int()),
   completed_at: z.nullable(z.number().int()),
@@ -490,6 +510,10 @@ export const CreateEvalRunResponseBody$inboundSchema: z.ZodType<
     "embedding_model": "embeddingModel",
     "request_counts": "requestCounts",
     "artifact_file_ids": "artifactFileIds",
+    "agent_name": "agentName",
+    "agent_version": "agentVersion",
+    "agent_effort": "agentEffort",
+    "cost_usd": "costUsd",
     "created_at": "createdAt",
     "started_at": "startedAt",
     "completed_at": "completedAt",
@@ -517,6 +541,10 @@ export type CreateEvalRunResponseBody$Outbound = {
   error: { [k: string]: any } | null;
   artifact_file_ids: Array<string>;
   metadata: { [k: string]: string };
+  agent_name: string | null;
+  agent_version: string | null;
+  agent_effort: string | null;
+  cost_usd: number | null;
   created_at: number;
   started_at: number | null;
   completed_at: number | null;
@@ -548,6 +576,10 @@ export const CreateEvalRunResponseBody$outboundSchema: z.ZodType<
   error: z.nullable(z.record(z.any())),
   artifactFileIds: z.array(z.string()),
   metadata: z.record(z.string()),
+  agentName: z.nullable(z.string()),
+  agentVersion: z.nullable(z.string()),
+  agentEffort: z.nullable(z.string()),
+  costUsd: z.nullable(z.number()),
   createdAt: z.number().int(),
   startedAt: z.nullable(z.number().int()),
   completedAt: z.nullable(z.number().int()),
@@ -565,6 +597,10 @@ export const CreateEvalRunResponseBody$outboundSchema: z.ZodType<
     embeddingModel: "embedding_model",
     requestCounts: "request_counts",
     artifactFileIds: "artifact_file_ids",
+    agentName: "agent_name",
+    agentVersion: "agent_version",
+    agentEffort: "agent_effort",
+    costUsd: "cost_usd",
     createdAt: "created_at",
     startedAt: "started_at",
     completedAt: "completed_at",

package/src/models/operations/getevalrun.ts

@@ -111,6 +111,22 @@ export type GetEvalRunResponseBody = {
   error: { [k: string]: any } | null;
   artifactFileIds: Array<string>;
   metadata: { [k: string]: string };
+  /**
+   * Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
+   */
+  agentName: string | null;
+  /**
+   * Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
+   */
+  agentVersion: string | null;
+  /**
+   * Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
+   */
+  agentEffort: string | null;
+  /**
+   * Total spend for the run in USD. Null when the harness did not report cost.
+   */
+  costUsd: number | null;
   createdAt: number;
   startedAt: number | null;
   completedAt: number | null;
@@ -495,6 +511,10 @@ export const GetEvalRunResponseBody$inboundSchema: z.ZodType<
   error: z.nullable(z.record(z.any())),
   artifact_file_ids: z.array(z.string()),
   metadata: z.record(z.string()),
+  agent_name: z.nullable(z.string()),
+  agent_version: z.nullable(z.string()),
+  agent_effort: z.nullable(z.string()),
+  cost_usd: z.nullable(z.number()),
   created_at: z.number().int(),
   started_at: z.nullable(z.number().int()),
   completed_at: z.nullable(z.number().int()),
@@ -512,6 +532,10 @@ export const GetEvalRunResponseBody$inboundSchema: z.ZodType<
     "embedding_model": "embeddingModel",
     "request_counts": "requestCounts",
     "artifact_file_ids": "artifactFileIds",
+    "agent_name": "agentName",
+    "agent_version": "agentVersion",
+    "agent_effort": "agentEffort",
+    "cost_usd": "costUsd",
     "created_at": "createdAt",
     "started_at": "startedAt",
     "completed_at": "completedAt",
@@ -539,6 +563,10 @@ export type GetEvalRunResponseBody$Outbound = {
   error: { [k: string]: any } | null;
   artifact_file_ids: Array<string>;
   metadata: { [k: string]: string };
+  agent_name: string | null;
+  agent_version: string | null;
+  agent_effort: string | null;
+  cost_usd: number | null;
   created_at: number;
   started_at: number | null;
   completed_at: number | null;
@@ -570,6 +598,10 @@ export const GetEvalRunResponseBody$outboundSchema: z.ZodType<
   error: z.nullable(z.record(z.any())),
   artifactFileIds: z.array(z.string()),
   metadata: z.record(z.string()),
+  agentName: z.nullable(z.string()),
+  agentVersion: z.nullable(z.string()),
+  agentEffort: z.nullable(z.string()),
+  costUsd: z.nullable(z.number()),
   createdAt: z.number().int(),
   startedAt: z.nullable(z.number().int()),
   completedAt: z.nullable(z.number().int()),
@@ -587,6 +619,10 @@ export const GetEvalRunResponseBody$outboundSchema: z.ZodType<
     embeddingModel: "embedding_model",
     requestCounts: "request_counts",
     artifactFileIds: "artifact_file_ids",
+    agentName: "agent_name",
+    agentVersion: "agent_version",
+    agentEffort: "agent_effort",
+    costUsd: "cost_usd",
     createdAt: "created_at",
     startedAt: "started_at",
     completedAt: "completed_at",

package/src/models/operations/getevalrunoverview.ts

@@ -0,0 +1,73 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type GetEvalRunOverviewRequest = {
+  runId: string;
+  metric?: string | undefined;
+  /**
+   * Optional external end-user identifier forwarded by the API gateway.
+   */
+  xOnBehalfOf?: string | undefined;
+};
+
+/** @internal */
+export const GetEvalRunOverviewRequest$inboundSchema: z.ZodType<
+  GetEvalRunOverviewRequest,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  run_id: z.string(),
+  metric: z.string().default("reward"),
+  "X-On-Behalf-Of": z.string().optional(),
+}).transform((v) => {
+  return remap$(v, {
+    "run_id": "runId",
+    "X-On-Behalf-Of": "xOnBehalfOf",
+  });
+});
+/** @internal */
+export type GetEvalRunOverviewRequest$Outbound = {
+  run_id: string;
+  metric: string;
+  "X-On-Behalf-Of"?: string | undefined;
+};
+
+/** @internal */
+export const GetEvalRunOverviewRequest$outboundSchema: z.ZodType<
+  GetEvalRunOverviewRequest$Outbound,
+  z.ZodTypeDef,
+  GetEvalRunOverviewRequest
+> = z.object({
+  runId: z.string(),
+  metric: z.string().default("reward"),
+  xOnBehalfOf: z.string().optional(),
+}).transform((v) => {
+  return remap$(v, {
+    runId: "run_id",
+    xOnBehalfOf: "X-On-Behalf-Of",
+  });
+});
+
+export function getEvalRunOverviewRequestToJSON(
+  getEvalRunOverviewRequest: GetEvalRunOverviewRequest,
+): string {
+  return JSON.stringify(
+    GetEvalRunOverviewRequest$outboundSchema.parse(getEvalRunOverviewRequest),
+  );
+}
+export function getEvalRunOverviewRequestFromJSON(
+  jsonString: string,
+): SafeParseResult<GetEvalRunOverviewRequest, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => GetEvalRunOverviewRequest$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'GetEvalRunOverviewRequest' from JSON`,
+  );
+}

package/src/models/operations/getevalsampletrajectory.ts

@@ -0,0 +1,199 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type GetEvalSampleTrajectoryRequest = {
+  runId: string;
+  sampleIndex: number;
+  taskId?: string | undefined;
+  model?: string | undefined;
+  /**
+   * Optional external end-user identifier forwarded by the API gateway.
+   */
+  xOnBehalfOf?: string | undefined;
+};
+
+/**
+ * OK
+ */
+export type GetEvalSampleTrajectoryResponseBody = {
+  object: "eval.sample.trajectory";
+  runId: string;
+  sampleIndex: number;
+  taskId: string;
+  model: string;
+  /**
+   * Interchange format of `steps`, e.g. `ATIF-v1.7`. Read it before assuming a step shape.
+   */
+  schemaVersion: string;
+  stepCount: number;
+  sizeBytes: number;
+  /**
+   * Steps exactly as the harness wrote them — reasoning_content, tool_calls[{ function_name, arguments }], observation.results[{ source_call_id, content }]. Passed through unvalidated so a harness upgrade does not need a gateway release.
+   */
+  steps: Array<any | null>;
+  /**
+   * Bounded tails of the trial's log files, keyed by their path in the trial dir (`trial.log`, `verifier/test-stdout.txt`, ...). Null for trials recorded before logs were captured.
+   */
+  logs: { [k: string]: string } | null;
+};
+
+/** @internal */
+export const GetEvalSampleTrajectoryRequest$inboundSchema: z.ZodType<
+  GetEvalSampleTrajectoryRequest,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  run_id: z.string(),
+  sample_index: z.number().int(),
+  task_id: z.string().optional(),
+  model: z.string().optional(),
+  "X-On-Behalf-Of": z.string().optional(),
+}).transform((v) => {
+  return remap$(v, {
+    "run_id": "runId",
+    "sample_index": "sampleIndex",
+    "task_id": "taskId",
+    "X-On-Behalf-Of": "xOnBehalfOf",
+  });
+});
+/** @internal */
+export type GetEvalSampleTrajectoryRequest$Outbound = {
+  run_id: string;
+  sample_index: number;
+  task_id?: string | undefined;
+  model?: string | undefined;
+  "X-On-Behalf-Of"?: string | undefined;
+};
+
+/** @internal */
+export const GetEvalSampleTrajectoryRequest$outboundSchema: z.ZodType<
+  GetEvalSampleTrajectoryRequest$Outbound,
+  z.ZodTypeDef,
+  GetEvalSampleTrajectoryRequest
+> = z.object({
+  runId: z.string(),
+  sampleIndex: z.number().int(),
+  taskId: z.string().optional(),
+  model: z.string().optional(),
+  xOnBehalfOf: z.string().optional(),
+}).transform((v) => {
+  return remap$(v, {
+    runId: "run_id",
+    sampleIndex: "sample_index",
+    taskId: "task_id",
+    xOnBehalfOf: "X-On-Behalf-Of",
+  });
+});
+
+export function getEvalSampleTrajectoryRequestToJSON(
+  getEvalSampleTrajectoryRequest: GetEvalSampleTrajectoryRequest,
+): string {
+  return JSON.stringify(
+    GetEvalSampleTrajectoryRequest$outboundSchema.parse(
+      getEvalSampleTrajectoryRequest,
+    ),
+  );
+}
+export function getEvalSampleTrajectoryRequestFromJSON(
+  jsonString: string,
+): SafeParseResult<GetEvalSampleTrajectoryRequest, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => GetEvalSampleTrajectoryRequest$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'GetEvalSampleTrajectoryRequest' from JSON`,
+  );
+}
+
+/** @internal */
+export const GetEvalSampleTrajectoryResponseBody$inboundSchema: z.ZodType<
+  GetEvalSampleTrajectoryResponseBody,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  object: z.literal("eval.sample.trajectory"),
+  run_id: z.string(),
+  sample_index: z.number().int(),
+  task_id: z.string(),
+  model: z.string(),
+  schema_version: z.string(),
+  step_count: z.number().int(),
+  size_bytes: z.number().int(),
+  steps: z.array(z.nullable(z.any())),
+  logs: z.nullable(z.record(z.string())),
+}).transform((v) => {
+  return remap$(v, {
+    "run_id": "runId",
+    "sample_index": "sampleIndex",
+    "task_id": "taskId",
+    "schema_version": "schemaVersion",
+    "step_count": "stepCount",
+    "size_bytes": "sizeBytes",
+  });
+});
+/** @internal */
+export type GetEvalSampleTrajectoryResponseBody$Outbound = {
+  object: "eval.sample.trajectory";
+  run_id: string;
+  sample_index: number;
+  task_id: string;
+  model: string;
+  schema_version: string;
+  step_count: number;
+  size_bytes: number;
+  steps: Array<any | null>;
+  logs: { [k: string]: string } | null;
+};
+
+/** @internal */
+export const GetEvalSampleTrajectoryResponseBody$outboundSchema: z.ZodType<
+  GetEvalSampleTrajectoryResponseBody$Outbound,
+  z.ZodTypeDef,
+  GetEvalSampleTrajectoryResponseBody
+> = z.object({
+  object: z.literal("eval.sample.trajectory"),
+  runId: z.string(),
+  sampleIndex: z.number().int(),
+  taskId: z.string(),
+  model: z.string(),
+  schemaVersion: z.string(),
+  stepCount: z.number().int(),
+  sizeBytes: z.number().int(),
+  steps: z.array(z.nullable(z.any())),
+  logs: z.nullable(z.record(z.string())),
+}).transform((v) => {
+  return remap$(v, {
+    runId: "run_id",
+    sampleIndex: "sample_index",
+    taskId: "task_id",
+    schemaVersion: "schema_version",
+    stepCount: "step_count",
+    sizeBytes: "size_bytes",
+  });
+});
+
+export function getEvalSampleTrajectoryResponseBodyToJSON(
+  getEvalSampleTrajectoryResponseBody: GetEvalSampleTrajectoryResponseBody,
+): string {
+  return JSON.stringify(
+    GetEvalSampleTrajectoryResponseBody$outboundSchema.parse(
+      getEvalSampleTrajectoryResponseBody,
+    ),
+  );
+}
+export function getEvalSampleTrajectoryResponseBodyFromJSON(
+  jsonString: string,
+): SafeParseResult<GetEvalSampleTrajectoryResponseBody, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) =>
+      GetEvalSampleTrajectoryResponseBody$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'GetEvalSampleTrajectoryResponseBody' from JSON`,
+  );
+}

package/src/models/operations/getevalsuiteleaderboard.ts

@@ -0,0 +1,93 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type GetEvalSuiteLeaderboardRequest = {
+  suiteId: string;
+  suiteVersion?: number | undefined;
+  metric?: string | undefined;
+  agent?: string | undefined;
+  model?: string | undefined;
+  limit?: number | undefined;
+  /**
+   * Optional external end-user identifier forwarded by the API gateway.
+   */
+  xOnBehalfOf?: string | undefined;
+};
+
+/** @internal */
+export const GetEvalSuiteLeaderboardRequest$inboundSchema: z.ZodType<
+  GetEvalSuiteLeaderboardRequest,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  suite_id: z.string(),
+  suite_version: z.number().int().optional(),
+  metric: z.string().default("reward"),
+  agent: z.string().optional(),
+  model: z.string().optional(),
+  limit: z.number().int().default(100),
+  "X-On-Behalf-Of": z.string().optional(),
+}).transform((v) => {
+  return remap$(v, {
+    "suite_id": "suiteId",
+    "suite_version": "suiteVersion",
+    "X-On-Behalf-Of": "xOnBehalfOf",
+  });
+});
+/** @internal */
+export type GetEvalSuiteLeaderboardRequest$Outbound = {
+  suite_id: string;
+  suite_version?: number | undefined;
+  metric: string;
+  agent?: string | undefined;
+  model?: string | undefined;
+  limit: number;
+  "X-On-Behalf-Of"?: string | undefined;
+};
+
+/** @internal */
+export const GetEvalSuiteLeaderboardRequest$outboundSchema: z.ZodType<
+  GetEvalSuiteLeaderboardRequest$Outbound,
+  z.ZodTypeDef,
+  GetEvalSuiteLeaderboardRequest
+> = z.object({
+  suiteId: z.string(),
+  suiteVersion: z.number().int().optional(),
+  metric: z.string().default("reward"),
+  agent: z.string().optional(),
+  model: z.string().optional(),
+  limit: z.number().int().default(100),
+  xOnBehalfOf: z.string().optional(),
+}).transform((v) => {
+  return remap$(v, {
+    suiteId: "suite_id",
+    suiteVersion: "suite_version",
+    xOnBehalfOf: "X-On-Behalf-Of",
+  });
+});
+
+export function getEvalSuiteLeaderboardRequestToJSON(
+  getEvalSuiteLeaderboardRequest: GetEvalSuiteLeaderboardRequest,
+): string {
+  return JSON.stringify(
+    GetEvalSuiteLeaderboardRequest$outboundSchema.parse(
+      getEvalSuiteLeaderboardRequest,
+    ),
+  );
+}
+export function getEvalSuiteLeaderboardRequestFromJSON(
+  jsonString: string,
+): SafeParseResult<GetEvalSuiteLeaderboardRequest, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => GetEvalSuiteLeaderboardRequest$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'GetEvalSuiteLeaderboardRequest' from JSON`,
+  );
+}