@meetkai/mka1 0.54.72 → 0.54.73

package/src/funcs/llmEvalsMeasureSuiteTokens.ts

@@ -0,0 +1,191 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { SDKCore } from "../core.js";
+import { encodeJSON, encodeSimple } from "../lib/encodings.js";
+import { matchStatusCode } from "../lib/http.js";
+import * as M from "../lib/matchers.js";
+import { compactMap } from "../lib/primitives.js";
+import { safeParse } from "../lib/schemas.js";
+import { RequestOptions } from "../lib/sdks.js";
+import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js";
+import { pathToFunc } from "../lib/url.js";
+import * as components from "../models/components/index.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { ResponseValidationError } from "../models/errors/responsevalidationerror.js";
+import { SDKError } from "../models/errors/sdkerror.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import * as operations from "../models/operations/index.js";
+import { APICall, APIPromise } from "../types/async.js";
+import { Result } from "../types/fp.js";
+
+/**
+ * Count tokens from stored text for a suite's unmeasured samples
+ *
+ * @remarks
+ * Backfill for rows written before the gateway recorded usage. For samples of the suite's completed runs that have answer or reasoning text but no token figure, counts prompt, reasoning and answer tokens from the stored text (tiktoken o200k_base, one ruler for every model), marks them `tokens_estimated`, and refreshes each run's per-task `tokens` and run totals so the leaderboard's efficiency figures appear. Processes `limit` runs per call, oldest run id first; call again until `remaining_runs` is 0. Samples the provider or the import already measured are left alone, and a sample with no text stays unmeasured. New rows get the same figures at generation or import time, so this is a no-op on a suite with nothing to measure.
+ *
+ * If set, this operation will use {@link Security.bearerAuth} from the global security.
+ */
+export function llmEvalsMeasureSuiteTokens(
+  client: SDKCore,
+  request: operations.MeasureEvalSuiteTokensRequest,
+  options?: RequestOptions,
+): APIPromise<
+  Result<
+    components.EvalSuiteTokenMeasurement,
+    | SDKError
+    | ResponseValidationError
+    | ConnectionError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | InvalidRequestError
+    | UnexpectedClientError
+    | SDKValidationError
+  >
+> {
+  return new APIPromise($do(
+    client,
+    request,
+    options,
+  ));
+}
+
+async function $do(
+  client: SDKCore,
+  request: operations.MeasureEvalSuiteTokensRequest,
+  options?: RequestOptions,
+): Promise<
+  [
+    Result<
+      components.EvalSuiteTokenMeasurement,
+      | SDKError
+      | ResponseValidationError
+      | ConnectionError
+      | RequestAbortedError
+      | RequestTimeoutError
+      | InvalidRequestError
+      | UnexpectedClientError
+      | SDKValidationError
+    >,
+    APICall,
+  ]
+> {
+  const parsed = safeParse(
+    request,
+    (value) =>
+      operations.MeasureEvalSuiteTokensRequest$outboundSchema.parse(value),
+    "Input validation failed",
+  );
+  if (!parsed.ok) {
+    return [parsed, { status: "invalid" }];
+  }
+  const payload = parsed.value;
+  const body = encodeJSON("body", payload.MeasureEvalSuiteTokensBody, {
+    explode: true,
+  });
+
+  const pathParams = {
+    suite_id: encodeSimple("suite_id", payload.suite_id, {
+      explode: false,
+      charEncoding: "percent",
+    }),
+  };
+  const path = pathToFunc("/api/v1/llm/evals/suites/{suite_id}/tokens/measure")(
+    pathParams,
+  );
+
+  const headers = new Headers(compactMap({
+    "Content-Type": "application/json",
+    Accept: "application/json",
+    "X-On-Behalf-Of": encodeSimple(
+      "X-On-Behalf-Of",
+      payload["X-On-Behalf-Of"],
+      { explode: false, charEncoding: "none" },
+    ),
+  }));
+
+  const secConfig = await extractSecurity(client._options.bearerAuth);
+  const securityInput = secConfig == null ? {} : { bearerAuth: secConfig };
+  const requestSecurity = resolveGlobalSecurity(securityInput, [0]);
+
+  const context = {
+    options: client._options,
+    baseURL: options?.serverURL ?? client._baseURL ?? "",
+    operationID: "measureEvalSuiteTokens",
+    oAuth2Scopes: null,
+
+    resolvedSecurity: requestSecurity,
+
+    securitySource: client._options.bearerAuth,
+    retryConfig: options?.retries
+      || client._options.retryConfig
+      || {
+        strategy: "backoff",
+        backoff: {
+          initialInterval: 500,
+          maxInterval: 5000,
+          exponent: 1.5,
+          maxElapsedTime: 15000,
+        },
+        retryConnectionErrors: true,
+      }
+      || { strategy: "none" },
+    retryCodes: options?.retryCodes || ["429", "500", "502", "503", "504"],
+  };
+
+  const requestRes = client._createRequest(context, {
+    security: requestSecurity,
+    method: "POST",
+    baseURL: options?.serverURL,
+    path: path,
+    headers: headers,
+    body: body,
+    userAgent: client._options.userAgent,
+    timeoutMs: options?.timeoutMs || client._options.timeoutMs || -1,
+  }, options);
+  if (!requestRes.ok) {
+    return [requestRes, { status: "invalid" }];
+  }
+  const req = requestRes.value;
+
+  const doResult = await client._do(req, {
+    context,
+    isErrorStatusCode: (statusCode: number) =>
+      matchStatusCode({ status: statusCode } as Response, ["4XX", "5XX"]),
+    retryConfig: context.retryConfig,
+    retryCodes: context.retryCodes,
+  });
+  if (!doResult.ok) {
+    return [doResult, { status: "request-error", request: req }];
+  }
+  const response = doResult.value;
+
+  const [result] = await M.match<
+    components.EvalSuiteTokenMeasurement,
+    | SDKError
+    | ResponseValidationError
+    | ConnectionError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | InvalidRequestError
+    | UnexpectedClientError
+    | SDKValidationError
+  >(
+    M.json(200, components.EvalSuiteTokenMeasurement$inboundSchema),
+    M.fail("4XX"),
+    M.fail("5XX"),
+  )(response, req);
+  if (!result.ok) {
+    return [result, { status: "complete", request: req, response }];
+  }
+
+  return [result, { status: "complete", request: req, response }];
+}

package/src/lib/config.ts

@@ -65,7 +65,7 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
 export const SDK_METADATA = {
   language: "typescript",
   openapiDocVersion: "1.1.0",
-  sdkVersion: "0.54.72",
+  sdkVersion: "0.54.73",
   genVersion: "2.943.0",
-  userAgent: "speakeasy-sdk/typescript 0.54.72 2.943.0 1.1.0 @meetkai/mka1",
+  userAgent: "speakeasy-sdk/typescript 0.54.73 2.943.0 1.1.0 @meetkai/mka1",
 } as const;

package/src/mcp-server/mcp-server.ts

@@ -19,7 +19,7 @@ const routes = buildRouteMap({
 export const app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.54.72",
+    currentVersion: "0.54.73",
   },
 });
 

package/src/mcp-server/server.ts

@@ -247,6 +247,7 @@ import { tool$llmEvalsListSchedules } from "./tools/llmEvalsListSchedules.js";
 import { tool$llmEvalsListSuiteLeaderboardRuns } from "./tools/llmEvalsListSuiteLeaderboardRuns.js";
 import { tool$llmEvalsListSuites } from "./tools/llmEvalsListSuites.js";
 import { tool$llmEvalsListSuiteVersions } from "./tools/llmEvalsListSuiteVersions.js";
+import { tool$llmEvalsMeasureSuiteTokens } from "./tools/llmEvalsMeasureSuiteTokens.js";
 import { tool$llmEvalsRerunFailedSamples } from "./tools/llmEvalsRerunFailedSamples.js";
 import { tool$llmEvalsRetryFailedRun } from "./tools/llmEvalsRetryFailedRun.js";
 import { tool$llmEvalsTriggerSchedule } from "./tools/llmEvalsTriggerSchedule.js";
@@ -473,7 +474,7 @@ export function createMCPServer(deps: {
 }) {
   const server = new McpServer({
     name: "SDK",
-    version: "0.54.72",
+    version: "0.54.73",
   });
 
   const client = new SDKCore({
@@ -683,6 +684,7 @@ export function createMCPServer(deps: {
   tool(tool$llmEvalsGetSuiteVersion);
   tool(tool$llmEvalsListSuiteLeaderboardRuns);
   tool(tool$llmEvalsGetSuiteLeaderboard);
+  tool(tool$llmEvalsMeasureSuiteTokens);
   tool(tool$llmEvalsCreateRun);
   tool(tool$llmEvalsListRuns);
   tool(tool$llmEvalsGetRun);

package/src/mcp-server/tools/llmEvalsMeasureSuiteTokens.ts

@@ -0,0 +1,37 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { llmEvalsMeasureSuiteTokens } from "../../funcs/llmEvalsMeasureSuiteTokens.js";
+import * as operations from "../../models/operations/index.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: operations.MeasureEvalSuiteTokensRequest$inboundSchema,
+};
+
+export const tool$llmEvalsMeasureSuiteTokens: ToolDefinition<typeof args> = {
+  name: "llm-evals-measure-suite-tokens",
+  description: `Count tokens from stored text for a suite's unmeasured samples
+
+Backfill for rows written before the gateway recorded usage. For samples of the suite's completed runs that have answer or reasoning text but no token figure, counts prompt, reasoning and answer tokens from the stored text (tiktoken o200k_base, one ruler for every model), marks them \`tokens_estimated\`, and refreshes each run's per-task \`tokens\` and run totals so the leaderboard's efficiency figures appear. Processes \`limit\` runs per call, oldest run id first; call again until \`remaining_runs\` is 0. Samples the provider or the import already measured are left alone, and a sample with no text stays unmeasured. New rows get the same figures at generation or import time, so this is a no-op on a suite with nothing to measure.`,
+  args,
+  tool: async (client, args, ctx) => {
+    const [result, apiCall] = await llmEvalsMeasureSuiteTokens(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    const value = result.value;
+
+    return formatResult(value, apiCall);
+  },
+};

package/src/models/components/evalsampleobject.ts

@@ -57,6 +57,10 @@ export type EvalSampleObject = {
    * The share of `output_tokens` the provider attributed to reasoning. Null when it reported no split.
    */
   reasoningTokens: number | null;
+  /**
+   * True when the token figures were counted from the stored text by the gateway (tiktoken o200k_base, one ruler for every model) because neither the provider nor the imported row reported usage. Comparable verbosity, not a billing count.
+   */
+  tokensEstimated: boolean;
   /**
    * Per-phase wall clock for a harness trial: `environment_setup`, `agent_setup`, `agent_execution`, `verifier`, each `{ started_at, finished_at }` as ISO 8601 timestamps exactly as the harness recorded them. A phase the harness did not time is absent rather than null. Drives the trial timing bar. Null outside harness runs.
    */
@@ -94,6 +98,7 @@ export const EvalSampleObject$inboundSchema: z.ZodType<
   output_tokens: z.nullable(z.number().int()),
   cached_tokens: z.nullable(z.number().int()),
   reasoning_tokens: z.nullable(z.number().int()),
+  tokens_estimated: z.boolean(),
   phase_timings: z.nullable(z.record(z.any())),
   created_at: z.number().int(),
   started_at: z.nullable(z.number().int()),
@@ -112,6 +117,7 @@ export const EvalSampleObject$inboundSchema: z.ZodType<
     "output_tokens": "outputTokens",
     "cached_tokens": "cachedTokens",
     "reasoning_tokens": "reasoningTokens",
+    "tokens_estimated": "tokensEstimated",
     "phase_timings": "phaseTimings",
     "created_at": "createdAt",
     "started_at": "startedAt",
@@ -142,6 +148,7 @@ export type EvalSampleObject$Outbound = {
   output_tokens: number | null;
   cached_tokens: number | null;
   reasoning_tokens: number | null;
+  tokens_estimated: boolean;
   phase_timings: { [k: string]: any } | null;
   created_at: number;
   started_at: number | null;
@@ -176,6 +183,7 @@ export const EvalSampleObject$outboundSchema: z.ZodType<
   outputTokens: z.nullable(z.number().int()),
   cachedTokens: z.nullable(z.number().int()),
   reasoningTokens: z.nullable(z.number().int()),
+  tokensEstimated: z.boolean(),
   phaseTimings: z.nullable(z.record(z.any())),
   createdAt: z.number().int(),
   startedAt: z.nullable(z.number().int()),
@@ -194,6 +202,7 @@ export const EvalSampleObject$outboundSchema: z.ZodType<
     outputTokens: "output_tokens",
     cachedTokens: "cached_tokens",
     reasoningTokens: "reasoning_tokens",
+    tokensEstimated: "tokens_estimated",
     phaseTimings: "phase_timings",
     createdAt: "created_at",
     startedAt: "started_at",

package/src/models/components/evalsuitetokenmeasurement.ts

@@ -0,0 +1,91 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type EvalSuiteTokenMeasurement = {
+  object: "eval.suite.token_measurement";
+  suiteId: string;
+  /**
+   * Runs whose samples were counted in this call.
+   */
+  measuredRuns: number;
+  /**
+   * Samples that gained token figures in this call.
+   */
+  measuredSamples: number;
+  /**
+   * Runs of the suite that still have samples with text but no token figure, after this call.
+   */
+  remainingRuns: number;
+};
+
+/** @internal */
+export const EvalSuiteTokenMeasurement$inboundSchema: z.ZodType<
+  EvalSuiteTokenMeasurement,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  object: z.literal("eval.suite.token_measurement"),
+  suite_id: z.string(),
+  measured_runs: z.number().int(),
+  measured_samples: z.number().int(),
+  remaining_runs: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    "suite_id": "suiteId",
+    "measured_runs": "measuredRuns",
+    "measured_samples": "measuredSamples",
+    "remaining_runs": "remainingRuns",
+  });
+});
+/** @internal */
+export type EvalSuiteTokenMeasurement$Outbound = {
+  object: "eval.suite.token_measurement";
+  suite_id: string;
+  measured_runs: number;
+  measured_samples: number;
+  remaining_runs: number;
+};
+
+/** @internal */
+export const EvalSuiteTokenMeasurement$outboundSchema: z.ZodType<
+  EvalSuiteTokenMeasurement$Outbound,
+  z.ZodTypeDef,
+  EvalSuiteTokenMeasurement
+> = z.object({
+  object: z.literal("eval.suite.token_measurement"),
+  suiteId: z.string(),
+  measuredRuns: z.number().int(),
+  measuredSamples: z.number().int(),
+  remainingRuns: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    suiteId: "suite_id",
+    measuredRuns: "measured_runs",
+    measuredSamples: "measured_samples",
+    remainingRuns: "remaining_runs",
+  });
+});
+
+export function evalSuiteTokenMeasurementToJSON(
+  evalSuiteTokenMeasurement: EvalSuiteTokenMeasurement,
+): string {
+  return JSON.stringify(
+    EvalSuiteTokenMeasurement$outboundSchema.parse(evalSuiteTokenMeasurement),
+  );
+}
+export function evalSuiteTokenMeasurementFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalSuiteTokenMeasurement, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalSuiteTokenMeasurement$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalSuiteTokenMeasurement' from JSON`,
+  );
+}

package/src/models/components/index.ts

@@ -291,6 +291,7 @@ export * from "./evalsuiteleaderboard.js";
 export * from "./evalsuitelist.js";
 export * from "./evalsuitemanifest.js";
 export * from "./evalsuiteobject.js";
+export * from "./evalsuitetokenmeasurement.js";
 export * from "./evaltask.js";
 export * from "./evaltasktype.js";
 export * from "./event.js";
@@ -455,6 +456,7 @@ export * from "./mcptoolcallwithid.js";
 export * from "./mcptoolchoice.js";
 export * from "./mcptooldefinition.js";
 export * from "./mcptoolfilter.js";
+export * from "./measureevalsuitetokensbody.js";
 export * from "./memoryentry.js";
 export * from "./memoryentrydeleted.js";
 export * from "./memoryentrylist.js";

package/src/models/components/measureevalsuitetokensbody.ts

@@ -0,0 +1,54 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type MeasureEvalSuiteTokensBody = {
+  /**
+   * How many runs to measure in this call. Call again until `remaining_runs` is 0.
+   */
+  limit?: number | undefined;
+};
+
+/** @internal */
+export const MeasureEvalSuiteTokensBody$inboundSchema: z.ZodType<
+  MeasureEvalSuiteTokensBody,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  limit: z.number().int().default(5),
+});
+/** @internal */
+export type MeasureEvalSuiteTokensBody$Outbound = {
+  limit: number;
+};
+
+/** @internal */
+export const MeasureEvalSuiteTokensBody$outboundSchema: z.ZodType<
+  MeasureEvalSuiteTokensBody$Outbound,
+  z.ZodTypeDef,
+  MeasureEvalSuiteTokensBody
+> = z.object({
+  limit: z.number().int().default(5),
+});
+
+export function measureEvalSuiteTokensBodyToJSON(
+  measureEvalSuiteTokensBody: MeasureEvalSuiteTokensBody,
+): string {
+  return JSON.stringify(
+    MeasureEvalSuiteTokensBody$outboundSchema.parse(measureEvalSuiteTokensBody),
+  );
+}
+export function measureEvalSuiteTokensBodyFromJSON(
+  jsonString: string,
+): SafeParseResult<MeasureEvalSuiteTokensBody, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => MeasureEvalSuiteTokensBody$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'MeasureEvalSuiteTokensBody' from JSON`,
+  );
+}

package/src/models/operations/index.ts

@@ -338,6 +338,7 @@ export * from "./livekittoken.js";
 export * from "./llmcheck.js";
 export * from "./llmgrant.js";
 export * from "./llmrevoke.js";
+export * from "./measureevalsuitetokens.js";
 export * from "./modifyvectorstore.js";
 export * from "./patchrepo.js";
 export * from "./pauseagentschedule.js";

package/src/models/operations/measureevalsuitetokens.ts

@@ -0,0 +1,80 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import * as components from "../components/index.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type MeasureEvalSuiteTokensRequest = {
+  suiteId: string;
+  /**
+   * Optional external end-user identifier forwarded by the API gateway.
+   */
+  xOnBehalfOf?: string | undefined;
+  measureEvalSuiteTokensBody: components.MeasureEvalSuiteTokensBody;
+};
+
+/** @internal */
+export const MeasureEvalSuiteTokensRequest$inboundSchema: z.ZodType<
+  MeasureEvalSuiteTokensRequest,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  suite_id: z.string(),
+  "X-On-Behalf-Of": z.string().optional(),
+  MeasureEvalSuiteTokensBody:
+    components.MeasureEvalSuiteTokensBody$inboundSchema,
+}).transform((v) => {
+  return remap$(v, {
+    "suite_id": "suiteId",
+    "X-On-Behalf-Of": "xOnBehalfOf",
+    "MeasureEvalSuiteTokensBody": "measureEvalSuiteTokensBody",
+  });
+});
+/** @internal */
+export type MeasureEvalSuiteTokensRequest$Outbound = {
+  suite_id: string;
+  "X-On-Behalf-Of"?: string | undefined;
+  MeasureEvalSuiteTokensBody: components.MeasureEvalSuiteTokensBody$Outbound;
+};
+
+/** @internal */
+export const MeasureEvalSuiteTokensRequest$outboundSchema: z.ZodType<
+  MeasureEvalSuiteTokensRequest$Outbound,
+  z.ZodTypeDef,
+  MeasureEvalSuiteTokensRequest
+> = z.object({
+  suiteId: z.string(),
+  xOnBehalfOf: z.string().optional(),
+  measureEvalSuiteTokensBody:
+    components.MeasureEvalSuiteTokensBody$outboundSchema,
+}).transform((v) => {
+  return remap$(v, {
+    suiteId: "suite_id",
+    xOnBehalfOf: "X-On-Behalf-Of",
+    measureEvalSuiteTokensBody: "MeasureEvalSuiteTokensBody",
+  });
+});
+
+export function measureEvalSuiteTokensRequestToJSON(
+  measureEvalSuiteTokensRequest: MeasureEvalSuiteTokensRequest,
+): string {
+  return JSON.stringify(
+    MeasureEvalSuiteTokensRequest$outboundSchema.parse(
+      measureEvalSuiteTokensRequest,
+    ),
+  );
+}
+export function measureEvalSuiteTokensRequestFromJSON(
+  jsonString: string,
+): SafeParseResult<MeasureEvalSuiteTokensRequest, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => MeasureEvalSuiteTokensRequest$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'MeasureEvalSuiteTokensRequest' from JSON`,
+  );
+}

package/src/react-query/index.ts

@@ -245,6 +245,7 @@ export * from "./llmEvalsListSchedules.js";
 export * from "./llmEvalsListSuiteLeaderboardRuns.js";
 export * from "./llmEvalsListSuites.js";
 export * from "./llmEvalsListSuiteVersions.js";
+export * from "./llmEvalsMeasureSuiteTokens.js";
 export * from "./llmEvalsRerunFailedSamples.js";
 export * from "./llmEvalsRetryFailedRun.js";
 export * from "./llmEvalsTriggerSchedule.js";