@meetkai/mka1 0.54.23 → 0.54.24

package/src/funcs/llmEvalsGetRunOverview.ts

@@ -0,0 +1,192 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { SDKCore } from "../core.js";
+import { encodeFormQuery, encodeSimple } from "../lib/encodings.js";
+import { matchStatusCode } from "../lib/http.js";
+import * as M from "../lib/matchers.js";
+import { compactMap } from "../lib/primitives.js";
+import { safeParse } from "../lib/schemas.js";
+import { RequestOptions } from "../lib/sdks.js";
+import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js";
+import { pathToFunc } from "../lib/url.js";
+import * as components from "../models/components/index.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { ResponseValidationError } from "../models/errors/responsevalidationerror.js";
+import { SDKError } from "../models/errors/sdkerror.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import * as operations from "../models/operations/index.js";
+import { APICall, APIPromise } from "../types/async.js";
+import { Result } from "../types/fp.js";
+
+/**
+ * Get an eval run overview
+ *
+ * @remarks
+ * Aggregate header for one run's trials view: trial counts, average reward, cost, and the input/output/cached token split. Its own route rather than fields on the run object because it scans the run's samples — folding it into `GET /evals/runs/{run_id}` would charge that scan to every run list and to the status polling that runs for the whole length of an eval. `trials.errored` overlaps `trials.failed` instead of partitioning it: a coding-agent trial can carry an agent timeout in `error` and still be completed and scored. `trials.retried` counts trials rerun through rerun-failed, read from the pre-rerun snapshot, since a rerun overwrites the sample in place.
+ *
+ * If set, this operation will use {@link Security.bearerAuth} from the global security.
+ */
+export function llmEvalsGetRunOverview(
+  client: SDKCore,
+  request: operations.GetEvalRunOverviewRequest,
+  options?: RequestOptions,
+): APIPromise<
+  Result<
+    components.EvalRunOverview,
+    | SDKError
+    | ResponseValidationError
+    | ConnectionError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | InvalidRequestError
+    | UnexpectedClientError
+    | SDKValidationError
+  >
+> {
+  return new APIPromise($do(
+    client,
+    request,
+    options,
+  ));
+}
+
+async function $do(
+  client: SDKCore,
+  request: operations.GetEvalRunOverviewRequest,
+  options?: RequestOptions,
+): Promise<
+  [
+    Result<
+      components.EvalRunOverview,
+      | SDKError
+      | ResponseValidationError
+      | ConnectionError
+      | RequestAbortedError
+      | RequestTimeoutError
+      | InvalidRequestError
+      | UnexpectedClientError
+      | SDKValidationError
+    >,
+    APICall,
+  ]
+> {
+  const parsed = safeParse(
+    request,
+    (value) => operations.GetEvalRunOverviewRequest$outboundSchema.parse(value),
+    "Input validation failed",
+  );
+  if (!parsed.ok) {
+    return [parsed, { status: "invalid" }];
+  }
+  const payload = parsed.value;
+  const body = null;
+
+  const pathParams = {
+    run_id: encodeSimple("run_id", payload.run_id, {
+      explode: false,
+      charEncoding: "percent",
+    }),
+  };
+  const path = pathToFunc("/api/v1/llm/evals/runs/{run_id}/overview")(
+    pathParams,
+  );
+
+  const query = encodeFormQuery({
+    "metric": payload.metric,
+  });
+
+  const headers = new Headers(compactMap({
+    Accept: "application/json",
+    "X-On-Behalf-Of": encodeSimple(
+      "X-On-Behalf-Of",
+      payload["X-On-Behalf-Of"],
+      { explode: false, charEncoding: "none" },
+    ),
+  }));
+
+  const secConfig = await extractSecurity(client._options.bearerAuth);
+  const securityInput = secConfig == null ? {} : { bearerAuth: secConfig };
+  const requestSecurity = resolveGlobalSecurity(securityInput, [0]);
+
+  const context = {
+    options: client._options,
+    baseURL: options?.serverURL ?? client._baseURL ?? "",
+    operationID: "getEvalRunOverview",
+    oAuth2Scopes: null,
+
+    resolvedSecurity: requestSecurity,
+
+    securitySource: client._options.bearerAuth,
+    retryConfig: options?.retries
+      || client._options.retryConfig
+      || {
+        strategy: "backoff",
+        backoff: {
+          initialInterval: 500,
+          maxInterval: 5000,
+          exponent: 1.5,
+          maxElapsedTime: 15000,
+        },
+        retryConnectionErrors: true,
+      }
+      || { strategy: "none" },
+    retryCodes: options?.retryCodes || ["429", "500", "502", "503", "504"],
+  };
+
+  const requestRes = client._createRequest(context, {
+    security: requestSecurity,
+    method: "GET",
+    baseURL: options?.serverURL,
+    path: path,
+    headers: headers,
+    query: query,
+    body: body,
+    userAgent: client._options.userAgent,
+    timeoutMs: options?.timeoutMs || client._options.timeoutMs || -1,
+  }, options);
+  if (!requestRes.ok) {
+    return [requestRes, { status: "invalid" }];
+  }
+  const req = requestRes.value;
+
+  const doResult = await client._do(req, {
+    context,
+    isErrorStatusCode: (statusCode: number) =>
+      matchStatusCode({ status: statusCode } as Response, ["4XX", "5XX"]),
+    retryConfig: context.retryConfig,
+    retryCodes: context.retryCodes,
+  });
+  if (!doResult.ok) {
+    return [doResult, { status: "request-error", request: req }];
+  }
+  const response = doResult.value;
+
+  const [result] = await M.match<
+    components.EvalRunOverview,
+    | SDKError
+    | ResponseValidationError
+    | ConnectionError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | InvalidRequestError
+    | UnexpectedClientError
+    | SDKValidationError
+  >(
+    M.json(200, components.EvalRunOverview$inboundSchema),
+    M.fail("4XX"),
+    M.fail("5XX"),
+  )(response, req);
+  if (!result.ok) {
+    return [result, { status: "complete", request: req, response }];
+  }
+
+  return [result, { status: "complete", request: req, response }];
+}

package/src/funcs/llmEvalsGetSampleTrajectory.ts

@@ -0,0 +1,197 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { SDKCore } from "../core.js";
+import { encodeFormQuery, encodeSimple } from "../lib/encodings.js";
+import { matchStatusCode } from "../lib/http.js";
+import * as M from "../lib/matchers.js";
+import { compactMap } from "../lib/primitives.js";
+import { safeParse } from "../lib/schemas.js";
+import { RequestOptions } from "../lib/sdks.js";
+import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js";
+import { pathToFunc } from "../lib/url.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { ResponseValidationError } from "../models/errors/responsevalidationerror.js";
+import { SDKError } from "../models/errors/sdkerror.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import * as operations from "../models/operations/index.js";
+import { APICall, APIPromise } from "../types/async.js";
+import { Result } from "../types/fp.js";
+
+/**
+ * Get an eval sample trajectory
+ *
+ * @remarks
+ * Returns one coding-agent trial's session — the steps the agent took, verbatim in the harness's interchange format (see `schema_version`, e.g. `ATIF-v1.7`). Deliberately not part of the samples list: a trajectory averages ~92 KB and that list pages up to 500 rows. Sample indices are unique per (task, model), not per run, so pass `task_id` and/or `model` when the run covers more than one; an index that matches several trials returns 400 rather than guessing, and never falls back to another trial's session. Returns 404 when the sample does not exist or recorded no trajectory.
+ *
+ * If set, this operation will use {@link Security.bearerAuth} from the global security.
+ */
+export function llmEvalsGetSampleTrajectory(
+  client: SDKCore,
+  request: operations.GetEvalSampleTrajectoryRequest,
+  options?: RequestOptions,
+): APIPromise<
+  Result<
+    operations.GetEvalSampleTrajectoryResponseBody,
+    | SDKError
+    | ResponseValidationError
+    | ConnectionError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | InvalidRequestError
+    | UnexpectedClientError
+    | SDKValidationError
+  >
+> {
+  return new APIPromise($do(
+    client,
+    request,
+    options,
+  ));
+}
+
+async function $do(
+  client: SDKCore,
+  request: operations.GetEvalSampleTrajectoryRequest,
+  options?: RequestOptions,
+): Promise<
+  [
+    Result<
+      operations.GetEvalSampleTrajectoryResponseBody,
+      | SDKError
+      | ResponseValidationError
+      | ConnectionError
+      | RequestAbortedError
+      | RequestTimeoutError
+      | InvalidRequestError
+      | UnexpectedClientError
+      | SDKValidationError
+    >,
+    APICall,
+  ]
+> {
+  const parsed = safeParse(
+    request,
+    (value) =>
+      operations.GetEvalSampleTrajectoryRequest$outboundSchema.parse(value),
+    "Input validation failed",
+  );
+  if (!parsed.ok) {
+    return [parsed, { status: "invalid" }];
+  }
+  const payload = parsed.value;
+  const body = null;
+
+  const pathParams = {
+    run_id: encodeSimple("run_id", payload.run_id, {
+      explode: false,
+      charEncoding: "percent",
+    }),
+    sample_index: encodeSimple("sample_index", payload.sample_index, {
+      explode: false,
+      charEncoding: "percent",
+    }),
+  };
+  const path = pathToFunc(
+    "/api/v1/llm/evals/runs/{run_id}/samples/{sample_index}/trajectory",
+  )(pathParams);
+
+  const query = encodeFormQuery({
+    "model": payload.model,
+    "task_id": payload.task_id,
+  });
+
+  const headers = new Headers(compactMap({
+    Accept: "application/json",
+    "X-On-Behalf-Of": encodeSimple(
+      "X-On-Behalf-Of",
+      payload["X-On-Behalf-Of"],
+      { explode: false, charEncoding: "none" },
+    ),
+  }));
+
+  const secConfig = await extractSecurity(client._options.bearerAuth);
+  const securityInput = secConfig == null ? {} : { bearerAuth: secConfig };
+  const requestSecurity = resolveGlobalSecurity(securityInput, [0]);
+
+  const context = {
+    options: client._options,
+    baseURL: options?.serverURL ?? client._baseURL ?? "",
+    operationID: "getEvalSampleTrajectory",
+    oAuth2Scopes: null,
+
+    resolvedSecurity: requestSecurity,
+
+    securitySource: client._options.bearerAuth,
+    retryConfig: options?.retries
+      || client._options.retryConfig
+      || {
+        strategy: "backoff",
+        backoff: {
+          initialInterval: 500,
+          maxInterval: 5000,
+          exponent: 1.5,
+          maxElapsedTime: 15000,
+        },
+        retryConnectionErrors: true,
+      }
+      || { strategy: "none" },
+    retryCodes: options?.retryCodes || ["429", "500", "502", "503", "504"],
+  };
+
+  const requestRes = client._createRequest(context, {
+    security: requestSecurity,
+    method: "GET",
+    baseURL: options?.serverURL,
+    path: path,
+    headers: headers,
+    query: query,
+    body: body,
+    userAgent: client._options.userAgent,
+    timeoutMs: options?.timeoutMs || client._options.timeoutMs || -1,
+  }, options);
+  if (!requestRes.ok) {
+    return [requestRes, { status: "invalid" }];
+  }
+  const req = requestRes.value;
+
+  const doResult = await client._do(req, {
+    context,
+    isErrorStatusCode: (statusCode: number) =>
+      matchStatusCode({ status: statusCode } as Response, ["4XX", "5XX"]),
+    retryConfig: context.retryConfig,
+    retryCodes: context.retryCodes,
+  });
+  if (!doResult.ok) {
+    return [doResult, { status: "request-error", request: req }];
+  }
+  const response = doResult.value;
+
+  const [result] = await M.match<
+    operations.GetEvalSampleTrajectoryResponseBody,
+    | SDKError
+    | ResponseValidationError
+    | ConnectionError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | InvalidRequestError
+    | UnexpectedClientError
+    | SDKValidationError
+  >(
+    M.json(200, operations.GetEvalSampleTrajectoryResponseBody$inboundSchema),
+    M.fail("4XX"),
+    M.fail("5XX"),
+  )(response, req);
+  if (!result.ok) {
+    return [result, { status: "complete", request: req, response }];
+  }
+
+  return [result, { status: "complete", request: req, response }];
+}

package/src/funcs/llmEvalsGetSuiteLeaderboard.ts

@@ -0,0 +1,197 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { SDKCore } from "../core.js";
+import { encodeFormQuery, encodeSimple } from "../lib/encodings.js";
+import { matchStatusCode } from "../lib/http.js";
+import * as M from "../lib/matchers.js";
+import { compactMap } from "../lib/primitives.js";
+import { safeParse } from "../lib/schemas.js";
+import { RequestOptions } from "../lib/sdks.js";
+import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js";
+import { pathToFunc } from "../lib/url.js";
+import * as components from "../models/components/index.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { ResponseValidationError } from "../models/errors/responsevalidationerror.js";
+import { SDKError } from "../models/errors/sdkerror.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import * as operations from "../models/operations/index.js";
+import { APICall, APIPromise } from "../types/async.js";
+import { Result } from "../types/fp.js";
+
+/**
+ * Get the coding-agent leaderboard for an eval suite
+ *
+ * @remarks
+ * Ranks the suite's completed coding-agent runs, one row per (agent, model, effort). Only runs that recorded an `agent_name` participate — a QA or transcription run of the same suite has no agent to attribute a score to and is not a board row. `accuracy` is the mean of `metric` (default `reward`) over the row's scored trials, and `accuracy_ci` is the 95% Wald interval around it; both are derived per request rather than stored, so rerunning a failed trial moves them instead of leaving a stale copy behind. `agent_version` is reported but is not part of the row identity, so an agent upgrade does not split a row in two. Rows are ordered by accuracy descending, nulls last. Pass `suite_version` when the suite has more than one, otherwise the board mixes task sets.
+ *
+ * If set, this operation will use {@link Security.bearerAuth} from the global security.
+ */
+export function llmEvalsGetSuiteLeaderboard(
+  client: SDKCore,
+  request: operations.GetEvalSuiteLeaderboardRequest,
+  options?: RequestOptions,
+): APIPromise<
+  Result<
+    components.EvalSuiteLeaderboard,
+    | SDKError
+    | ResponseValidationError
+    | ConnectionError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | InvalidRequestError
+    | UnexpectedClientError
+    | SDKValidationError
+  >
+> {
+  return new APIPromise($do(
+    client,
+    request,
+    options,
+  ));
+}
+
+async function $do(
+  client: SDKCore,
+  request: operations.GetEvalSuiteLeaderboardRequest,
+  options?: RequestOptions,
+): Promise<
+  [
+    Result<
+      components.EvalSuiteLeaderboard,
+      | SDKError
+      | ResponseValidationError
+      | ConnectionError
+      | RequestAbortedError
+      | RequestTimeoutError
+      | InvalidRequestError
+      | UnexpectedClientError
+      | SDKValidationError
+    >,
+    APICall,
+  ]
+> {
+  const parsed = safeParse(
+    request,
+    (value) =>
+      operations.GetEvalSuiteLeaderboardRequest$outboundSchema.parse(value),
+    "Input validation failed",
+  );
+  if (!parsed.ok) {
+    return [parsed, { status: "invalid" }];
+  }
+  const payload = parsed.value;
+  const body = null;
+
+  const pathParams = {
+    suite_id: encodeSimple("suite_id", payload.suite_id, {
+      explode: false,
+      charEncoding: "percent",
+    }),
+  };
+  const path = pathToFunc("/api/v1/llm/evals/suites/{suite_id}/leaderboard")(
+    pathParams,
+  );
+
+  const query = encodeFormQuery({
+    "agent": payload.agent,
+    "limit": payload.limit,
+    "metric": payload.metric,
+    "model": payload.model,
+    "suite_version": payload.suite_version,
+  });
+
+  const headers = new Headers(compactMap({
+    Accept: "application/json",
+    "X-On-Behalf-Of": encodeSimple(
+      "X-On-Behalf-Of",
+      payload["X-On-Behalf-Of"],
+      { explode: false, charEncoding: "none" },
+    ),
+  }));
+
+  const secConfig = await extractSecurity(client._options.bearerAuth);
+  const securityInput = secConfig == null ? {} : { bearerAuth: secConfig };
+  const requestSecurity = resolveGlobalSecurity(securityInput, [0]);
+
+  const context = {
+    options: client._options,
+    baseURL: options?.serverURL ?? client._baseURL ?? "",
+    operationID: "getEvalSuiteLeaderboard",
+    oAuth2Scopes: null,
+
+    resolvedSecurity: requestSecurity,
+
+    securitySource: client._options.bearerAuth,
+    retryConfig: options?.retries
+      || client._options.retryConfig
+      || {
+        strategy: "backoff",
+        backoff: {
+          initialInterval: 500,
+          maxInterval: 5000,
+          exponent: 1.5,
+          maxElapsedTime: 15000,
+        },
+        retryConnectionErrors: true,
+      }
+      || { strategy: "none" },
+    retryCodes: options?.retryCodes || ["429", "500", "502", "503", "504"],
+  };
+
+  const requestRes = client._createRequest(context, {
+    security: requestSecurity,
+    method: "GET",
+    baseURL: options?.serverURL,
+    path: path,
+    headers: headers,
+    query: query,
+    body: body,
+    userAgent: client._options.userAgent,
+    timeoutMs: options?.timeoutMs || client._options.timeoutMs || -1,
+  }, options);
+  if (!requestRes.ok) {
+    return [requestRes, { status: "invalid" }];
+  }
+  const req = requestRes.value;
+
+  const doResult = await client._do(req, {
+    context,
+    isErrorStatusCode: (statusCode: number) =>
+      matchStatusCode({ status: statusCode } as Response, ["4XX", "5XX"]),
+    retryConfig: context.retryConfig,
+    retryCodes: context.retryCodes,
+  });
+  if (!doResult.ok) {
+    return [doResult, { status: "request-error", request: req }];
+  }
+  const response = doResult.value;
+
+  const [result] = await M.match<
+    components.EvalSuiteLeaderboard,
+    | SDKError
+    | ResponseValidationError
+    | ConnectionError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | InvalidRequestError
+    | UnexpectedClientError
+    | SDKValidationError
+  >(
+    M.json(200, components.EvalSuiteLeaderboard$inboundSchema),
+    M.fail("4XX"),
+    M.fail("5XX"),
+  )(response, req);
+  if (!result.ok) {
+    return [result, { status: "complete", request: req, response }];
+  }
+
+  return [result, { status: "complete", request: req, response }];
+}