@meetkai/mka1 0.54.22 → 0.54.24

package/src/mcp-server/tools/llmEvalsUploadSampleTrajectory.ts

@@ -0,0 +1,38 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { llmEvalsUploadSampleTrajectory } from "../../funcs/llmEvalsUploadSampleTrajectory.js";
+import * as operations from "../../models/operations/index.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: operations.UploadEvalSampleTrajectoryRequest$inboundSchema,
+};
+
+export const tool$llmEvalsUploadSampleTrajectory: ToolDefinition<typeof args> =
+  {
+    name: "llm-evals-upload-sample-trajectory",
+    description: `Upload an eval sample trajectory
+
+Stores one coding-agent trial's session, in whatever interchange format the harness produced (record it in \`schema_version\`, e.g. \`ATIF-v1.7\`). \`steps\` is stored verbatim and never validated, so a harness format change needs no gateway release. Separate from the run import because a trajectory averages ~92 KB and a whole run's worth would not fit in one request. Upserts, so re-sending after a timeout replaces the trajectory rather than failing. Sample indices are unique per (task, model), not per run, so pass \`task_id\` and/or \`model\` when the run covers more than one — an ambiguous index returns 400 rather than writing to the wrong trial. Rejects a payload whose \`steps\` and \`logs\` together exceed 16 MiB with 413.`,
+    args,
+    tool: async (client, args, ctx) => {
+      const [result, apiCall] = await llmEvalsUploadSampleTrajectory(
+        client,
+        args.request,
+        { fetchOptions: { signal: ctx.signal } },
+      ).$inspect();
+
+      if (!result.ok) {
+        return {
+          content: [{ type: "text", text: result.error.message }],
+          isError: true,
+        };
+      }
+
+      const value = result.value;
+
+      return formatResult(value, apiCall);
+    },
+  };

package/src/mcp-server/tools/llmResponsesCreate.ts

@@ -0,0 +1,37 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { llmResponsesCreate } from "../../funcs/llmResponsesCreate.js";
+import * as operations from "../../models/operations/index.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: operations.CreateResponseRequest$inboundSchema,
+};
+
+export const tool$llmResponsesCreate: ToolDefinition<typeof args> = {
+  name: "llm-responses-create",
+  description: `Create an agent-powered response with tool support
+
+Creates a new AI agent response using advanced language models with autonomous tool usage capabilities.`,
+  args,
+  tool: async (client, args, ctx) => {
+    const [result, apiCall] = await llmResponsesCreate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    const value = result.value;
+
+    return formatResult(value, apiCall);
+  },
+};

package/src/models/components/appendagenttrialsrequest.ts

@@ -0,0 +1,224 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { ClosedEnum } from "../../types/enums.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+/**
+ * `failed` means no score was produced at all. An agent timeout is NOT that — the harness still runs the verifier and scores the trial, so it stays `completed` and carries its exception in `error`.
+ */
+export const AppendAgentTrialsRequestStatus = {
+  Completed: "completed",
+  Failed: "failed",
+} as const;
+/**
+ * `failed` means no score was produced at all. An agent timeout is NOT that — the harness still runs the verifier and scores the trial, so it stays `completed` and carries its exception in `error`.
+ */
+export type AppendAgentTrialsRequestStatus = ClosedEnum<
+  typeof AppendAgentTrialsRequestStatus
+>;
+
+export type AppendAgentTrialsRequestTrials = {
+  /**
+   * Must already exist in the target suite version's manifest; unknown ids are rejected rather than invented.
+   */
+  taskId: string;
+  sampleIndex: number;
+  /**
+   * `failed` means no score was produced at all. An agent timeout is NOT that — the harness still runs the verifier and scores the trial, so it stays `completed` and carries its exception in `error`.
+   */
+  status?: AppendAgentTrialsRequestStatus | undefined;
+  /**
+   * The task instruction.
+   */
+  prompt?: string | undefined;
+  target?: string | null | undefined;
+  /**
+   * What decided the score — the verifier's output. The agent's session goes to the trajectory route.
+   */
+  outputText?: string | null | undefined;
+  /**
+   * Metric values for this trial, e.g. `{ "reward": 1, "resolved": 1 }`. The board averages `reward` by default.
+   */
+  scores?: { [k: string]: number } | undefined;
+  /**
+   * The exception this trial recorded, if any. Does not by itself make the trial a failure.
+   */
+  error?: { [k: string]: any } | null | undefined;
+  costUsd?: number | null | undefined;
+  inputTokens?: number | null | undefined;
+  outputTokens?: number | null | undefined;
+  /**
+   * The share of `input_tokens` served from the provider's prompt cache.
+   */
+  cachedTokens?: number | null | undefined;
+  phaseTimings?: { [k: string]: any } | null | undefined;
+};
+
+export type AppendAgentTrialsRequest = {
+  /**
+   * Each trial pins its own `sample_index` (unique per task and model): re-sending a batch replaces the same rows instead of duplicating them, so a retried batch is safe.
+   */
+  trials?: Array<AppendAgentTrialsRequestTrials> | undefined;
+  /**
+   * Marks the run completed once these trials land — the last call of a streaming job sends this, usually with an empty `trials`.
+   */
+  finalize?: boolean | undefined;
+};
+
+/** @internal */
+export const AppendAgentTrialsRequestStatus$inboundSchema: z.ZodNativeEnum<
+  typeof AppendAgentTrialsRequestStatus
+> = z.nativeEnum(AppendAgentTrialsRequestStatus);
+/** @internal */
+export const AppendAgentTrialsRequestStatus$outboundSchema: z.ZodNativeEnum<
+  typeof AppendAgentTrialsRequestStatus
+> = AppendAgentTrialsRequestStatus$inboundSchema;
+
+/** @internal */
+export const AppendAgentTrialsRequestTrials$inboundSchema: z.ZodType<
+  AppendAgentTrialsRequestTrials,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  task_id: z.string(),
+  sample_index: z.number().int(),
+  status: AppendAgentTrialsRequestStatus$inboundSchema.default("completed"),
+  prompt: z.string().default(""),
+  target: z.nullable(z.string()).optional(),
+  output_text: z.nullable(z.string()).optional(),
+  scores: z.record(z.number()).optional(),
+  error: z.nullable(z.record(z.any())).optional(),
+  cost_usd: z.nullable(z.number()).optional(),
+  input_tokens: z.nullable(z.number().int()).optional(),
+  output_tokens: z.nullable(z.number().int()).optional(),
+  cached_tokens: z.nullable(z.number().int()).optional(),
+  phase_timings: z.nullable(z.record(z.any())).optional(),
+}).transform((v) => {
+  return remap$(v, {
+    "task_id": "taskId",
+    "sample_index": "sampleIndex",
+    "output_text": "outputText",
+    "cost_usd": "costUsd",
+    "input_tokens": "inputTokens",
+    "output_tokens": "outputTokens",
+    "cached_tokens": "cachedTokens",
+    "phase_timings": "phaseTimings",
+  });
+});
+/** @internal */
+export type AppendAgentTrialsRequestTrials$Outbound = {
+  task_id: string;
+  sample_index: number;
+  status: string;
+  prompt: string;
+  target?: string | null | undefined;
+  output_text?: string | null | undefined;
+  scores?: { [k: string]: number } | undefined;
+  error?: { [k: string]: any } | null | undefined;
+  cost_usd?: number | null | undefined;
+  input_tokens?: number | null | undefined;
+  output_tokens?: number | null | undefined;
+  cached_tokens?: number | null | undefined;
+  phase_timings?: { [k: string]: any } | null | undefined;
+};
+
+/** @internal */
+export const AppendAgentTrialsRequestTrials$outboundSchema: z.ZodType<
+  AppendAgentTrialsRequestTrials$Outbound,
+  z.ZodTypeDef,
+  AppendAgentTrialsRequestTrials
+> = z.object({
+  taskId: z.string(),
+  sampleIndex: z.number().int(),
+  status: AppendAgentTrialsRequestStatus$outboundSchema.default("completed"),
+  prompt: z.string().default(""),
+  target: z.nullable(z.string()).optional(),
+  outputText: z.nullable(z.string()).optional(),
+  scores: z.record(z.number()).optional(),
+  error: z.nullable(z.record(z.any())).optional(),
+  costUsd: z.nullable(z.number()).optional(),
+  inputTokens: z.nullable(z.number().int()).optional(),
+  outputTokens: z.nullable(z.number().int()).optional(),
+  cachedTokens: z.nullable(z.number().int()).optional(),
+  phaseTimings: z.nullable(z.record(z.any())).optional(),
+}).transform((v) => {
+  return remap$(v, {
+    taskId: "task_id",
+    sampleIndex: "sample_index",
+    outputText: "output_text",
+    costUsd: "cost_usd",
+    inputTokens: "input_tokens",
+    outputTokens: "output_tokens",
+    cachedTokens: "cached_tokens",
+    phaseTimings: "phase_timings",
+  });
+});
+
+export function appendAgentTrialsRequestTrialsToJSON(
+  appendAgentTrialsRequestTrials: AppendAgentTrialsRequestTrials,
+): string {
+  return JSON.stringify(
+    AppendAgentTrialsRequestTrials$outboundSchema.parse(
+      appendAgentTrialsRequestTrials,
+    ),
+  );
+}
+export function appendAgentTrialsRequestTrialsFromJSON(
+  jsonString: string,
+): SafeParseResult<AppendAgentTrialsRequestTrials, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => AppendAgentTrialsRequestTrials$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'AppendAgentTrialsRequestTrials' from JSON`,
+  );
+}
+
+/** @internal */
+export const AppendAgentTrialsRequest$inboundSchema: z.ZodType<
+  AppendAgentTrialsRequest,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  trials: z.array(z.lazy(() => AppendAgentTrialsRequestTrials$inboundSchema))
+    .optional(),
+  finalize: z.boolean().default(false),
+});
+/** @internal */
+export type AppendAgentTrialsRequest$Outbound = {
+  trials?: Array<AppendAgentTrialsRequestTrials$Outbound> | undefined;
+  finalize: boolean;
+};
+
+/** @internal */
+export const AppendAgentTrialsRequest$outboundSchema: z.ZodType<
+  AppendAgentTrialsRequest$Outbound,
+  z.ZodTypeDef,
+  AppendAgentTrialsRequest
+> = z.object({
+  trials: z.array(z.lazy(() => AppendAgentTrialsRequestTrials$outboundSchema))
+    .optional(),
+  finalize: z.boolean().default(false),
+});
+
+export function appendAgentTrialsRequestToJSON(
+  appendAgentTrialsRequest: AppendAgentTrialsRequest,
+): string {
+  return JSON.stringify(
+    AppendAgentTrialsRequest$outboundSchema.parse(appendAgentTrialsRequest),
+  );
+}
+export function appendAgentTrialsRequestFromJSON(
+  jsonString: string,
+): SafeParseResult<AppendAgentTrialsRequest, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => AppendAgentTrialsRequest$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'AppendAgentTrialsRequest' from JSON`,
+  );
+}

package/src/models/components/evalleaderboardrow.ts

@@ -0,0 +1,200 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type AccuracyCi = {
+  low: number;
+  high: number;
+  margin: number;
+};
+
+export type EvalLeaderboardRow = {
+  agentName: string;
+  /**
+   * Version from the row's newest run. Not part of the row identity, so older runs in this row may have reported a different one.
+   */
+  agentVersion: string | null;
+  agentEffort: string | null;
+  model: string;
+  /**
+   * Mean of `metric` over the row's scored trials. Null when no trial carried a numeric value for that metric — distinct from 0, which means every trial scored zero.
+   */
+  accuracy: number | null;
+  /**
+   * 95% Wald interval, p ± 1.96·sqrt(p(1-p)/n), clamped to [0, 1]. Derived per request rather than stored, so it cannot drift when a failed trial is rerun. It assumes `metric` is a 0/1 outcome (the harness `reward`); on a continuous metric the mean is still right but this interval is not, so read it only for pass/fail metrics. Null when `accuracy` is.
+   */
+  accuracyCi: AccuracyCi | null;
+  /**
+   * Every sample in the row's runs, whatever its status.
+   */
+  trials: number;
+  /**
+   * Trials carrying a numeric `metric` — the denominator behind `accuracy` and `accuracy_ci`.
+   */
+  scoredTrials: number;
+  /**
+   * Trials that ended in the `failed` status, i.e. produced no score at all. An agent timeout is NOT one of these: the harness still verifies and scores that trial.
+   */
+  failedTrials: number;
+  /**
+   * Summed trial spend, falling back to the runs' own totals when trials carry no cost. Null when neither reported any.
+   */
+  costUsd: number | null;
+  runCount: number;
+  /**
+   * The runs behind this row, largest (most trials) first, ties newest first — element 0 is the run to open when the reader drills into this row, so a full benchmark job outranks a newer one-task smoke run.
+   */
+  runIds: Array<string>;
+  /**
+   * Creation time of the newest run in the row, unix seconds.
+   */
+  lastRunAt: number;
+};
+
+/** @internal */
+export const AccuracyCi$inboundSchema: z.ZodType<
+  AccuracyCi,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  low: z.number(),
+  high: z.number(),
+  margin: z.number(),
+});
+/** @internal */
+export type AccuracyCi$Outbound = {
+  low: number;
+  high: number;
+  margin: number;
+};
+
+/** @internal */
+export const AccuracyCi$outboundSchema: z.ZodType<
+  AccuracyCi$Outbound,
+  z.ZodTypeDef,
+  AccuracyCi
+> = z.object({
+  low: z.number(),
+  high: z.number(),
+  margin: z.number(),
+});
+
+export function accuracyCiToJSON(accuracyCi: AccuracyCi): string {
+  return JSON.stringify(AccuracyCi$outboundSchema.parse(accuracyCi));
+}
+export function accuracyCiFromJSON(
+  jsonString: string,
+): SafeParseResult<AccuracyCi, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => AccuracyCi$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'AccuracyCi' from JSON`,
+  );
+}
+
+/** @internal */
+export const EvalLeaderboardRow$inboundSchema: z.ZodType<
+  EvalLeaderboardRow,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  agent_name: z.string(),
+  agent_version: z.nullable(z.string()),
+  agent_effort: z.nullable(z.string()),
+  model: z.string(),
+  accuracy: z.nullable(z.number()),
+  accuracy_ci: z.nullable(z.lazy(() => AccuracyCi$inboundSchema)),
+  trials: z.number().int(),
+  scored_trials: z.number().int(),
+  failed_trials: z.number().int(),
+  cost_usd: z.nullable(z.number()),
+  run_count: z.number().int(),
+  run_ids: z.array(z.string()),
+  last_run_at: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    "agent_name": "agentName",
+    "agent_version": "agentVersion",
+    "agent_effort": "agentEffort",
+    "accuracy_ci": "accuracyCi",
+    "scored_trials": "scoredTrials",
+    "failed_trials": "failedTrials",
+    "cost_usd": "costUsd",
+    "run_count": "runCount",
+    "run_ids": "runIds",
+    "last_run_at": "lastRunAt",
+  });
+});
+/** @internal */
+export type EvalLeaderboardRow$Outbound = {
+  agent_name: string;
+  agent_version: string | null;
+  agent_effort: string | null;
+  model: string;
+  accuracy: number | null;
+  accuracy_ci: AccuracyCi$Outbound | null;
+  trials: number;
+  scored_trials: number;
+  failed_trials: number;
+  cost_usd: number | null;
+  run_count: number;
+  run_ids: Array<string>;
+  last_run_at: number;
+};
+
+/** @internal */
+export const EvalLeaderboardRow$outboundSchema: z.ZodType<
+  EvalLeaderboardRow$Outbound,
+  z.ZodTypeDef,
+  EvalLeaderboardRow
+> = z.object({
+  agentName: z.string(),
+  agentVersion: z.nullable(z.string()),
+  agentEffort: z.nullable(z.string()),
+  model: z.string(),
+  accuracy: z.nullable(z.number()),
+  accuracyCi: z.nullable(z.lazy(() => AccuracyCi$outboundSchema)),
+  trials: z.number().int(),
+  scoredTrials: z.number().int(),
+  failedTrials: z.number().int(),
+  costUsd: z.nullable(z.number()),
+  runCount: z.number().int(),
+  runIds: z.array(z.string()),
+  lastRunAt: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    agentName: "agent_name",
+    agentVersion: "agent_version",
+    agentEffort: "agent_effort",
+    accuracyCi: "accuracy_ci",
+    scoredTrials: "scored_trials",
+    failedTrials: "failed_trials",
+    costUsd: "cost_usd",
+    runCount: "run_count",
+    runIds: "run_ids",
+    lastRunAt: "last_run_at",
+  });
+});
+
+export function evalLeaderboardRowToJSON(
+  evalLeaderboardRow: EvalLeaderboardRow,
+): string {
+  return JSON.stringify(
+    EvalLeaderboardRow$outboundSchema.parse(evalLeaderboardRow),
+  );
+}
+export function evalLeaderboardRowFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalLeaderboardRow, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalLeaderboardRow$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalLeaderboardRow' from JSON`,
+  );
+}