@meetkai/mka1 0.54.22 → 0.54.24

package/src/models/components/evalrunoverview.ts

@@ -0,0 +1,240 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+import {
+  EvalRunStatus,
+  EvalRunStatus$inboundSchema,
+  EvalRunStatus$outboundSchema,
+} from "./evalrunstatus.js";
+
+export type Trials = {
+  total: number;
+  completed: number;
+  /**
+   * Trials in the `failed` status — no score produced.
+   */
+  failed: number;
+  /**
+   * Trials with a non-null `error`. Overlaps `failed` rather than partitioning it: a harness trial can carry an agent timeout and still be completed and scored.
+   */
+  errored: number;
+  /**
+   * Trials that were rerun through rerun-failed, counted from the pre-rerun snapshot. 0 when the run was never rerun.
+   */
+  retried: number;
+};
+
+/**
+ * Summed over the run's trials. Each field is null when no trial reported it.
+ */
+export type Tokens = {
+  input: number | null;
+  output: number | null;
+  /**
+   * Share of `input` served from the provider's prompt cache.
+   */
+  cached: number | null;
+};
+
+export type EvalRunOverview = {
+  object: "eval.run.overview";
+  runId: string;
+  suiteId: string;
+  status: EvalRunStatus;
+  agentName: string | null;
+  agentVersion: string | null;
+  agentEffort: string | null;
+  models: Array<string>;
+  metric: string;
+  /**
+   * Mean of `metric` over trials that carry it numerically. Null when none do.
+   */
+  avgReward: number | null;
+  trials: Trials;
+  costUsd: number | null;
+  /**
+   * Summed over the run's trials. Each field is null when no trial reported it.
+   */
+  tokens: Tokens;
+};
+
+/** @internal */
+export const Trials$inboundSchema: z.ZodType<Trials, z.ZodTypeDef, unknown> = z
+  .object({
+    total: z.number().int(),
+    completed: z.number().int(),
+    failed: z.number().int(),
+    errored: z.number().int(),
+    retried: z.number().int(),
+  });
+/** @internal */
+export type Trials$Outbound = {
+  total: number;
+  completed: number;
+  failed: number;
+  errored: number;
+  retried: number;
+};
+
+/** @internal */
+export const Trials$outboundSchema: z.ZodType<
+  Trials$Outbound,
+  z.ZodTypeDef,
+  Trials
+> = z.object({
+  total: z.number().int(),
+  completed: z.number().int(),
+  failed: z.number().int(),
+  errored: z.number().int(),
+  retried: z.number().int(),
+});
+
+export function trialsToJSON(trials: Trials): string {
+  return JSON.stringify(Trials$outboundSchema.parse(trials));
+}
+export function trialsFromJSON(
+  jsonString: string,
+): SafeParseResult<Trials, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => Trials$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'Trials' from JSON`,
+  );
+}
+
+/** @internal */
+export const Tokens$inboundSchema: z.ZodType<Tokens, z.ZodTypeDef, unknown> = z
+  .object({
+    input: z.nullable(z.number().int()),
+    output: z.nullable(z.number().int()),
+    cached: z.nullable(z.number().int()),
+  });
+/** @internal */
+export type Tokens$Outbound = {
+  input: number | null;
+  output: number | null;
+  cached: number | null;
+};
+
+/** @internal */
+export const Tokens$outboundSchema: z.ZodType<
+  Tokens$Outbound,
+  z.ZodTypeDef,
+  Tokens
+> = z.object({
+  input: z.nullable(z.number().int()),
+  output: z.nullable(z.number().int()),
+  cached: z.nullable(z.number().int()),
+});
+
+export function tokensToJSON(tokens: Tokens): string {
+  return JSON.stringify(Tokens$outboundSchema.parse(tokens));
+}
+export function tokensFromJSON(
+  jsonString: string,
+): SafeParseResult<Tokens, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => Tokens$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'Tokens' from JSON`,
+  );
+}
+
+/** @internal */
+export const EvalRunOverview$inboundSchema: z.ZodType<
+  EvalRunOverview,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  object: z.literal("eval.run.overview"),
+  run_id: z.string(),
+  suite_id: z.string(),
+  status: EvalRunStatus$inboundSchema,
+  agent_name: z.nullable(z.string()),
+  agent_version: z.nullable(z.string()),
+  agent_effort: z.nullable(z.string()),
+  models: z.array(z.string()),
+  metric: z.string(),
+  avg_reward: z.nullable(z.number()),
+  trials: z.lazy(() => Trials$inboundSchema),
+  cost_usd: z.nullable(z.number()),
+  tokens: z.lazy(() => Tokens$inboundSchema),
+}).transform((v) => {
+  return remap$(v, {
+    "run_id": "runId",
+    "suite_id": "suiteId",
+    "agent_name": "agentName",
+    "agent_version": "agentVersion",
+    "agent_effort": "agentEffort",
+    "avg_reward": "avgReward",
+    "cost_usd": "costUsd",
+  });
+});
+/** @internal */
+export type EvalRunOverview$Outbound = {
+  object: "eval.run.overview";
+  run_id: string;
+  suite_id: string;
+  status: string;
+  agent_name: string | null;
+  agent_version: string | null;
+  agent_effort: string | null;
+  models: Array<string>;
+  metric: string;
+  avg_reward: number | null;
+  trials: Trials$Outbound;
+  cost_usd: number | null;
+  tokens: Tokens$Outbound;
+};
+
+/** @internal */
+export const EvalRunOverview$outboundSchema: z.ZodType<
+  EvalRunOverview$Outbound,
+  z.ZodTypeDef,
+  EvalRunOverview
+> = z.object({
+  object: z.literal("eval.run.overview"),
+  runId: z.string(),
+  suiteId: z.string(),
+  status: EvalRunStatus$outboundSchema,
+  agentName: z.nullable(z.string()),
+  agentVersion: z.nullable(z.string()),
+  agentEffort: z.nullable(z.string()),
+  models: z.array(z.string()),
+  metric: z.string(),
+  avgReward: z.nullable(z.number()),
+  trials: z.lazy(() => Trials$outboundSchema),
+  costUsd: z.nullable(z.number()),
+  tokens: z.lazy(() => Tokens$outboundSchema),
+}).transform((v) => {
+  return remap$(v, {
+    runId: "run_id",
+    suiteId: "suite_id",
+    agentName: "agent_name",
+    agentVersion: "agent_version",
+    agentEffort: "agent_effort",
+    avgReward: "avg_reward",
+    costUsd: "cost_usd",
+  });
+});
+
+export function evalRunOverviewToJSON(
+  evalRunOverview: EvalRunOverview,
+): string {
+  return JSON.stringify(EvalRunOverview$outboundSchema.parse(evalRunOverview));
+}
+export function evalRunOverviewFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalRunOverview, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalRunOverview$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalRunOverview' from JSON`,
+  );
+}

package/src/models/components/evalsampleobject.ts

@@ -33,7 +33,30 @@ export type EvalSampleObject = {
   extractedOutput: string | null;
   scores: { [k: string]: any };
   judge: { [k: string]: any } | null;
+  /**
+   * The exception that ended this sample, if any. For a harness trial this also carries an agent timeout — which is still a *scored* trial, so do not read a non-null `error` here as an unscored one; check `status` and `scores`.
+   */
   error: { [k: string]: any } | null;
+  /**
+   * Spend for this one trial in USD. Null outside harness runs, and when the harness reported no cost.
+   */
+  costUsd: number | null;
+  /**
+   * Prompt tokens for the whole trial session. Includes `cached_tokens`.
+   */
+  inputTokens: number | null;
+  /**
+   * Completion tokens for the whole trial session.
+   */
+  outputTokens: number | null;
+  /**
+   * The share of `input_tokens` served from the provider's prompt cache. Reported separately because it is what makes two agents at the same token count cost an order of magnitude apart.
+   */
+  cachedTokens: number | null;
+  /**
+   * Per-phase wall clock for a harness trial: `environment_setup`, `agent_setup`, `agent_execution`, `verifier`, each `{ started_at, finished_at }` as ISO 8601 timestamps exactly as the harness recorded them. A phase the harness did not time is absent rather than null. Drives the trial timing bar. Null outside harness runs.
+   */
+  phaseTimings: { [k: string]: any } | null;
   createdAt: number;
   startedAt: number | null;
   completedAt: number | null;
@@ -62,6 +85,11 @@ export const EvalSampleObject$inboundSchema: z.ZodType<
   scores: z.record(z.any()),
   judge: z.nullable(z.record(z.any())),
   error: z.nullable(z.record(z.any())),
+  cost_usd: z.nullable(z.number()),
+  input_tokens: z.nullable(z.number().int()),
+  output_tokens: z.nullable(z.number().int()),
+  cached_tokens: z.nullable(z.number().int()),
+  phase_timings: z.nullable(z.record(z.any())),
   created_at: z.number().int(),
   started_at: z.nullable(z.number().int()),
   completed_at: z.nullable(z.number().int()),
@@ -74,6 +102,11 @@ export const EvalSampleObject$inboundSchema: z.ZodType<
     "response_id": "responseId",
     "output_text": "outputText",
     "extracted_output": "extractedOutput",
+    "cost_usd": "costUsd",
+    "input_tokens": "inputTokens",
+    "output_tokens": "outputTokens",
+    "cached_tokens": "cachedTokens",
+    "phase_timings": "phaseTimings",
     "created_at": "createdAt",
     "started_at": "startedAt",
     "completed_at": "completedAt",
@@ -98,6 +131,11 @@ export type EvalSampleObject$Outbound = {
   scores: { [k: string]: any };
   judge: { [k: string]: any } | null;
   error: { [k: string]: any } | null;
+  cost_usd: number | null;
+  input_tokens: number | null;
+  output_tokens: number | null;
+  cached_tokens: number | null;
+  phase_timings: { [k: string]: any } | null;
   created_at: number;
   started_at: number | null;
   completed_at: number | null;
@@ -126,6 +164,11 @@ export const EvalSampleObject$outboundSchema: z.ZodType<
   scores: z.record(z.any()),
   judge: z.nullable(z.record(z.any())),
   error: z.nullable(z.record(z.any())),
+  costUsd: z.nullable(z.number()),
+  inputTokens: z.nullable(z.number().int()),
+  outputTokens: z.nullable(z.number().int()),
+  cachedTokens: z.nullable(z.number().int()),
+  phaseTimings: z.nullable(z.record(z.any())),
   createdAt: z.number().int(),
   startedAt: z.nullable(z.number().int()),
   completedAt: z.nullable(z.number().int()),
@@ -138,6 +181,11 @@ export const EvalSampleObject$outboundSchema: z.ZodType<
     responseId: "response_id",
     outputText: "output_text",
     extractedOutput: "extracted_output",
+    costUsd: "cost_usd",
+    inputTokens: "input_tokens",
+    outputTokens: "output_tokens",
+    cachedTokens: "cached_tokens",
+    phaseTimings: "phase_timings",
     createdAt: "created_at",
     startedAt: "started_at",
     completedAt: "completed_at",

package/src/models/components/evalsampletrajectoryuploadedobject.ts

@@ -0,0 +1,98 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type EvalSampleTrajectoryUploadedObject = {
+  object: "eval.sample.trajectory.uploaded";
+  runId: string;
+  sampleIndex: number;
+  taskId: string;
+  model: string;
+  stepCount: number;
+  /**
+   * Measured from the stored payload, so it describes what was actually written.
+   */
+  sizeBytes: number;
+};
+
+/** @internal */
+export const EvalSampleTrajectoryUploadedObject$inboundSchema: z.ZodType<
+  EvalSampleTrajectoryUploadedObject,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  object: z.literal("eval.sample.trajectory.uploaded"),
+  run_id: z.string(),
+  sample_index: z.number().int(),
+  task_id: z.string(),
+  model: z.string(),
+  step_count: z.number().int(),
+  size_bytes: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    "run_id": "runId",
+    "sample_index": "sampleIndex",
+    "task_id": "taskId",
+    "step_count": "stepCount",
+    "size_bytes": "sizeBytes",
+  });
+});
+/** @internal */
+export type EvalSampleTrajectoryUploadedObject$Outbound = {
+  object: "eval.sample.trajectory.uploaded";
+  run_id: string;
+  sample_index: number;
+  task_id: string;
+  model: string;
+  step_count: number;
+  size_bytes: number;
+};
+
+/** @internal */
+export const EvalSampleTrajectoryUploadedObject$outboundSchema: z.ZodType<
+  EvalSampleTrajectoryUploadedObject$Outbound,
+  z.ZodTypeDef,
+  EvalSampleTrajectoryUploadedObject
+> = z.object({
+  object: z.literal("eval.sample.trajectory.uploaded"),
+  runId: z.string(),
+  sampleIndex: z.number().int(),
+  taskId: z.string(),
+  model: z.string(),
+  stepCount: z.number().int(),
+  sizeBytes: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    runId: "run_id",
+    sampleIndex: "sample_index",
+    taskId: "task_id",
+    stepCount: "step_count",
+    sizeBytes: "size_bytes",
+  });
+});
+
+export function evalSampleTrajectoryUploadedObjectToJSON(
+  evalSampleTrajectoryUploadedObject: EvalSampleTrajectoryUploadedObject,
+): string {
+  return JSON.stringify(
+    EvalSampleTrajectoryUploadedObject$outboundSchema.parse(
+      evalSampleTrajectoryUploadedObject,
+    ),
+  );
+}
+export function evalSampleTrajectoryUploadedObjectFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalSampleTrajectoryUploadedObject, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) =>
+      EvalSampleTrajectoryUploadedObject$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalSampleTrajectoryUploadedObject' from JSON`,
+  );
+}

package/src/models/components/evalsuiteleaderboard.ts

@@ -0,0 +1,93 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+import {
+  EvalLeaderboardRow,
+  EvalLeaderboardRow$inboundSchema,
+  EvalLeaderboardRow$Outbound,
+  EvalLeaderboardRow$outboundSchema,
+} from "./evalleaderboardrow.js";
+
+export type EvalSuiteLeaderboard = {
+  object: "eval.leaderboard";
+  suiteId: string;
+  /**
+   * The version this board was restricted to, or null when it spans every version.
+   */
+  suiteVersion: number | null;
+  /**
+   * The `scores` key `accuracy` averages. Echoed back so a board rendered from a non-default metric is self-describing.
+   */
+  metric: string;
+  /**
+   * Rows ordered by `accuracy` descending, nulls last, then by agent, model, and effort.
+   */
+  data: Array<EvalLeaderboardRow>;
+};
+
+/** @internal */
+export const EvalSuiteLeaderboard$inboundSchema: z.ZodType<
+  EvalSuiteLeaderboard,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  object: z.literal("eval.leaderboard"),
+  suite_id: z.string(),
+  suite_version: z.nullable(z.number().int()),
+  metric: z.string(),
+  data: z.array(EvalLeaderboardRow$inboundSchema),
+}).transform((v) => {
+  return remap$(v, {
+    "suite_id": "suiteId",
+    "suite_version": "suiteVersion",
+  });
+});
+/** @internal */
+export type EvalSuiteLeaderboard$Outbound = {
+  object: "eval.leaderboard";
+  suite_id: string;
+  suite_version: number | null;
+  metric: string;
+  data: Array<EvalLeaderboardRow$Outbound>;
+};
+
+/** @internal */
+export const EvalSuiteLeaderboard$outboundSchema: z.ZodType<
+  EvalSuiteLeaderboard$Outbound,
+  z.ZodTypeDef,
+  EvalSuiteLeaderboard
+> = z.object({
+  object: z.literal("eval.leaderboard"),
+  suiteId: z.string(),
+  suiteVersion: z.nullable(z.number().int()),
+  metric: z.string(),
+  data: z.array(EvalLeaderboardRow$outboundSchema),
+}).transform((v) => {
+  return remap$(v, {
+    suiteId: "suite_id",
+    suiteVersion: "suite_version",
+  });
+});
+
+export function evalSuiteLeaderboardToJSON(
+  evalSuiteLeaderboard: EvalSuiteLeaderboard,
+): string {
+  return JSON.stringify(
+    EvalSuiteLeaderboard$outboundSchema.parse(evalSuiteLeaderboard),
+  );
+}
+export function evalSuiteLeaderboardFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalSuiteLeaderboard, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalSuiteLeaderboard$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalSuiteLeaderboard' from JSON`,
+  );
+}