@meetkai/mka1 0.54.72 → 0.54.74

package/src/models/operations/measureevalsuitetokens.ts

@@ -0,0 +1,80 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import * as components from "../components/index.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type MeasureEvalSuiteTokensRequest = {
+  suiteId: string;
+  /**
+   * Optional external end-user identifier forwarded by the API gateway.
+   */
+  xOnBehalfOf?: string | undefined;
+  measureEvalSuiteTokensBody: components.MeasureEvalSuiteTokensBody;
+};
+
+/** @internal */
+export const MeasureEvalSuiteTokensRequest$inboundSchema: z.ZodType<
+  MeasureEvalSuiteTokensRequest,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  suite_id: z.string(),
+  "X-On-Behalf-Of": z.string().optional(),
+  MeasureEvalSuiteTokensBody:
+    components.MeasureEvalSuiteTokensBody$inboundSchema,
+}).transform((v) => {
+  return remap$(v, {
+    "suite_id": "suiteId",
+    "X-On-Behalf-Of": "xOnBehalfOf",
+    "MeasureEvalSuiteTokensBody": "measureEvalSuiteTokensBody",
+  });
+});
+/** @internal */
+export type MeasureEvalSuiteTokensRequest$Outbound = {
+  suite_id: string;
+  "X-On-Behalf-Of"?: string | undefined;
+  MeasureEvalSuiteTokensBody: components.MeasureEvalSuiteTokensBody$Outbound;
+};
+
+/** @internal */
+export const MeasureEvalSuiteTokensRequest$outboundSchema: z.ZodType<
+  MeasureEvalSuiteTokensRequest$Outbound,
+  z.ZodTypeDef,
+  MeasureEvalSuiteTokensRequest
+> = z.object({
+  suiteId: z.string(),
+  xOnBehalfOf: z.string().optional(),
+  measureEvalSuiteTokensBody:
+    components.MeasureEvalSuiteTokensBody$outboundSchema,
+}).transform((v) => {
+  return remap$(v, {
+    suiteId: "suite_id",
+    xOnBehalfOf: "X-On-Behalf-Of",
+    measureEvalSuiteTokensBody: "MeasureEvalSuiteTokensBody",
+  });
+});
+
+export function measureEvalSuiteTokensRequestToJSON(
+  measureEvalSuiteTokensRequest: MeasureEvalSuiteTokensRequest,
+): string {
+  return JSON.stringify(
+    MeasureEvalSuiteTokensRequest$outboundSchema.parse(
+      measureEvalSuiteTokensRequest,
+    ),
+  );
+}
+export function measureEvalSuiteTokensRequestFromJSON(
+  jsonString: string,
+): SafeParseResult<MeasureEvalSuiteTokensRequest, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => MeasureEvalSuiteTokensRequest$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'MeasureEvalSuiteTokensRequest' from JSON`,
+  );
+}

package/src/react-query/index.ts

@@ -245,6 +245,7 @@ export * from "./llmEvalsListSchedules.js";
 export * from "./llmEvalsListSuiteLeaderboardRuns.js";
 export * from "./llmEvalsListSuites.js";
 export * from "./llmEvalsListSuiteVersions.js";
+export * from "./llmEvalsMeasureSuiteTokens.js";
 export * from "./llmEvalsRerunFailedSamples.js";
 export * from "./llmEvalsRetryFailedRun.js";
 export * from "./llmEvalsTriggerSchedule.js";

package/src/react-query/llmEvalsMeasureSuiteTokens.ts

@@ -0,0 +1,110 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import {
+  MutationKey,
+  useMutation,
+  UseMutationResult,
+} from "@tanstack/react-query";
+import { SDKCore } from "../core.js";
+import { llmEvalsMeasureSuiteTokens } from "../funcs/llmEvalsMeasureSuiteTokens.js";
+import { combineSignals } from "../lib/primitives.js";
+import { RequestOptions } from "../lib/sdks.js";
+import * as components from "../models/components/index.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { ResponseValidationError } from "../models/errors/responsevalidationerror.js";
+import { SDKError } from "../models/errors/sdkerror.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import * as operations from "../models/operations/index.js";
+import { unwrapAsync } from "../types/fp.js";
+import { useSDKContext } from "./_context.js";
+import { MutationHookOptions } from "./_types.js";
+
+export type LlmEvalsMeasureSuiteTokensMutationVariables = {
+  request: operations.MeasureEvalSuiteTokensRequest;
+  options?: RequestOptions;
+};
+
+export type LlmEvalsMeasureSuiteTokensMutationData =
+  components.EvalSuiteTokenMeasurement;
+
+export type LlmEvalsMeasureSuiteTokensMutationError =
+  | SDKError
+  | ResponseValidationError
+  | ConnectionError
+  | RequestAbortedError
+  | RequestTimeoutError
+  | InvalidRequestError
+  | UnexpectedClientError
+  | SDKValidationError;
+
+/**
+ * Count tokens from stored text for a suite's unmeasured samples
+ *
+ * @remarks
+ * Backfill for rows written before the gateway recorded usage. For samples of the suite's completed runs that have answer or reasoning text but no token figure, counts prompt, reasoning and answer tokens from the stored text (tiktoken o200k_base, one ruler for every model), marks them `tokens_estimated`, and refreshes each run's per-task `tokens` and run totals so the leaderboard's efficiency figures appear. Processes `limit` runs per call, oldest run id first; call again until `remaining_runs` is 0. Samples the provider or the import already measured are left alone, and a sample with no text stays unmeasured. New rows get the same figures at generation or import time, so this is a no-op on a suite with nothing to measure.
+ */
+export function useLlmEvalsMeasureSuiteTokensMutation(
+  options?: MutationHookOptions<
+    LlmEvalsMeasureSuiteTokensMutationData,
+    LlmEvalsMeasureSuiteTokensMutationError,
+    LlmEvalsMeasureSuiteTokensMutationVariables
+  >,
+): UseMutationResult<
+  LlmEvalsMeasureSuiteTokensMutationData,
+  LlmEvalsMeasureSuiteTokensMutationError,
+  LlmEvalsMeasureSuiteTokensMutationVariables
+> {
+  const client = useSDKContext();
+  return useMutation({
+    ...buildLlmEvalsMeasureSuiteTokensMutation(client, options),
+    ...options,
+  });
+}
+
+export function mutationKeyLlmEvalsMeasureSuiteTokens(): MutationKey {
+  return ["@meetkai/mka1", "evals", "measureSuiteTokens"];
+}
+
+export function buildLlmEvalsMeasureSuiteTokensMutation(
+  client$: SDKCore,
+  hookOptions?: RequestOptions,
+): {
+  mutationKey: MutationKey;
+  mutationFn: (
+    variables: LlmEvalsMeasureSuiteTokensMutationVariables,
+  ) => Promise<LlmEvalsMeasureSuiteTokensMutationData>;
+} {
+  return {
+    mutationKey: mutationKeyLlmEvalsMeasureSuiteTokens(),
+    mutationFn: function llmEvalsMeasureSuiteTokensMutationFn({
+      request,
+      options,
+    }): Promise<LlmEvalsMeasureSuiteTokensMutationData> {
+      const mergedOptions = {
+        ...hookOptions,
+        ...options,
+        fetchOptions: {
+          ...hookOptions?.fetchOptions,
+          ...options?.fetchOptions,
+          signal: combineSignals(
+            hookOptions?.fetchOptions?.signal,
+            options?.fetchOptions?.signal,
+          ),
+        },
+      };
+      return unwrapAsync(llmEvalsMeasureSuiteTokens(
+        client$,
+        request,
+        mergedOptions,
+      ));
+    },
+  };
+}

package/src/sdk/evals.ts

@@ -29,6 +29,7 @@ import { llmEvalsListSchedules } from "../funcs/llmEvalsListSchedules.js";
 import { llmEvalsListSuiteLeaderboardRuns } from "../funcs/llmEvalsListSuiteLeaderboardRuns.js";
 import { llmEvalsListSuites } from "../funcs/llmEvalsListSuites.js";
 import { llmEvalsListSuiteVersions } from "../funcs/llmEvalsListSuiteVersions.js";
+import { llmEvalsMeasureSuiteTokens } from "../funcs/llmEvalsMeasureSuiteTokens.js";
 import { llmEvalsRerunFailedSamples } from "../funcs/llmEvalsRerunFailedSamples.js";
 import { llmEvalsRetryFailedRun } from "../funcs/llmEvalsRetryFailedRun.js";
 import { llmEvalsTriggerSchedule } from "../funcs/llmEvalsTriggerSchedule.js";
@@ -193,6 +194,23 @@ export class Evals extends ClientSDK {
     ));
   }
 
+  /**
+   * Count tokens from stored text for a suite's unmeasured samples
+   *
+   * @remarks
+   * Backfill for rows written before the gateway recorded usage. For samples of the suite's completed runs that have answer or reasoning text but no token figure, counts prompt, reasoning and answer tokens from the stored text (tiktoken o200k_base, one ruler for every model), marks them `tokens_estimated`, and refreshes each run's per-task `tokens` and run totals so the leaderboard's efficiency figures appear. Processes `limit` runs per call, oldest run id first; call again until `remaining_runs` is 0. Samples the provider or the import already measured are left alone, and a sample with no text stays unmeasured. New rows get the same figures at generation or import time, so this is a no-op on a suite with nothing to measure.
+   */
+  async measureSuiteTokens(
+    request: operations.MeasureEvalSuiteTokensRequest,
+    options?: RequestOptions,
+  ): Promise<components.EvalSuiteTokenMeasurement> {
+    return unwrapAsync(llmEvalsMeasureSuiteTokens(
+      this,
+      request,
+      options,
+    ));
+  }
+
   /**
    * Create an eval run
    *