@meetkai/mka1 0.54.26 → 0.54.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meetkai/mka1",
-  "version": "0.54.26",
+  "version": "0.54.27",
   "author": "Daniel Wilder <dauncy@gmail.com>",
   "homepage": "https://docs.mka1.com",
   "publishConfig": {

package/src/lib/config.ts

@@ -65,7 +65,7 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
 export const SDK_METADATA = {
   language: "typescript",
   openapiDocVersion: "1.1.0",
-  sdkVersion: "0.54.26",
+  sdkVersion: "0.54.27",
   genVersion: "2.935.1",
-  userAgent: "speakeasy-sdk/typescript 0.54.26 2.935.1 1.1.0 @meetkai/mka1",
+  userAgent: "speakeasy-sdk/typescript 0.54.27 2.935.1 1.1.0 @meetkai/mka1",
 } as const;

package/src/mcp-server/mcp-server.ts

@@ -19,7 +19,7 @@ const routes = buildRouteMap({
 export const app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.54.26",
+    currentVersion: "0.54.27",
   },
 });
 

package/src/mcp-server/server.ts

@@ -412,7 +412,7 @@ export function createMCPServer(deps: {
 }) {
   const server = new McpServer({
     name: "SDK",
-    version: "0.54.26",
+    version: "0.54.27",
   });
 
   const client = new SDKCore({

package/src/models/components/evalembeddingconfig.ts

@@ -0,0 +1,272 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { ClosedEnum } from "../../types/enums.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+import {
+  EvalDataset,
+  EvalDataset$inboundSchema,
+  EvalDataset$Outbound,
+  EvalDataset$outboundSchema,
+} from "./evaldataset.js";
+
+/**
+ * What the vectors are scored as. Decides the dataset layout the task expects and the metrics it reports.
+ */
+export const Kind = {
+  Sts: "sts",
+  Classification: "classification",
+  MultilabelClassification: "multilabel_classification",
+  Retrieval: "retrieval",
+  Reranking: "reranking",
+} as const;
+/**
+ * What the vectors are scored as. Decides the dataset layout the task expects and the metrics it reports.
+ */
+export type Kind = ClosedEnum<typeof Kind>;
+
+/**
+ * Configuration of an `embedding` task: which columns hold the texts and labels, the auxiliary datasets, and the scoring protocol.
+ */
+export type EvalEmbeddingConfig = {
+  /**
+   * What the vectors are scored as. Decides the dataset layout the task expects and the metrics it reports.
+   */
+  kind: Kind;
+  /**
+   * Row column with the text to embed (classification, multilabel, queries and corpus documents).
+   */
+  textColumn?: string | undefined;
+  /**
+   * STS: first sentence of the pair.
+   */
+  text1Column?: string | undefined;
+  /**
+   * STS: second sentence of the pair.
+   */
+  text2Column?: string | undefined;
+  /**
+   * STS: gold similarity score. Retrieval/reranking: relevance score column of the qrels rows.
+   */
+  scoreColumn?: string | undefined;
+  /**
+   * Classification: label column. Multilabel: a list of labels per row.
+   */
+  labelColumn?: string | undefined;
+  /**
+   * Retrieval/reranking: id column of query and corpus rows.
+   */
+  idColumn?: string | undefined;
+  /**
+   * Retrieval/reranking: optional corpus title column, prepended to the document text the way MTEB does.
+   */
+  titleColumn?: string | undefined;
+  /**
+   * Retrieval/reranking: query id column of the qrels and top-ranked rows.
+   */
+  queryIdColumn?: string | undefined;
+  /**
+   * Retrieval/reranking: document id column of the qrels rows.
+   */
+  corpusIdColumn?: string | undefined;
+  /**
+   * Reranking: list of candidate document ids per query in the top-ranked rows.
+   */
+  corpusIdsColumn?: string | undefined;
+  /**
+   * Dataset backing an eval task.
+   */
+  trainDataset?: EvalDataset | undefined;
+  /**
+   * Dataset backing an eval task.
+   */
+  corpusDataset?: EvalDataset | undefined;
+  /**
+   * Dataset backing an eval task.
+   */
+  qrelsDataset?: EvalDataset | undefined;
+  /**
+   * Dataset backing an eval task.
+   */
+  topRankedDataset?: EvalDataset | undefined;
+  /**
+   * Classification/multilabel: training rows sampled per label in each experiment (MTEB: 8).
+   */
+  samplesPerLabel?: number | undefined;
+  /**
+   * Classification/multilabel: fit-and-score repetitions averaged into the reported metrics (MTEB: 10).
+   */
+  nExperiments?: number | undefined;
+  /**
+   * Seed for the training-row sampling.
+   */
+  seed?: number | undefined;
+  /**
+   * Retrieval/reranking: cutoffs for nDCG, MAP, recall, precision and MRR.
+   */
+  kValues?: Array<number> | undefined;
+  /**
+   * Each text is cut to this many characters before it is embedded. Providers reject over-long inputs with a 4xx, which fails the sample terminally and leaves a hole in the corpus; the default 60000 sits under the gateway's own 64,000-character ceiling for its embedding models. Raise it only for a model that accepts more.
+   */
+  maxChars?: number | undefined;
+  /**
+   * Fail preparation when the task would embed more rows than this (eval + train + corpus + queries). Ceiling 50000; vectors are stored per sample and scored in memory.
+   */
+  maxTexts?: number | undefined;
+};
+
+/** @internal */
+export const Kind$inboundSchema: z.ZodNativeEnum<typeof Kind> = z.nativeEnum(
+  Kind,
+);
+/** @internal */
+export const Kind$outboundSchema: z.ZodNativeEnum<typeof Kind> =
+  Kind$inboundSchema;
+
+/** @internal */
+export const EvalEmbeddingConfig$inboundSchema: z.ZodType<
+  EvalEmbeddingConfig,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  kind: Kind$inboundSchema,
+  text_column: z.string().default("text"),
+  text1_column: z.string().default("sentence1"),
+  text2_column: z.string().default("sentence2"),
+  score_column: z.string().default("score"),
+  label_column: z.string().default("label"),
+  id_column: z.string().default("_id"),
+  title_column: z.string().default("title"),
+  query_id_column: z.string().default("query-id"),
+  corpus_id_column: z.string().default("corpus-id"),
+  corpus_ids_column: z.string().default("corpus-ids"),
+  train_dataset: EvalDataset$inboundSchema.optional(),
+  corpus_dataset: EvalDataset$inboundSchema.optional(),
+  qrels_dataset: EvalDataset$inboundSchema.optional(),
+  top_ranked_dataset: EvalDataset$inboundSchema.optional(),
+  samples_per_label: z.number().int().default(8),
+  n_experiments: z.number().int().default(10),
+  seed: z.number().int().default(42),
+  k_values: z.array(z.number().int()).optional(),
+  max_chars: z.number().int().default(60000),
+  max_texts: z.number().int().default(50000),
+}).transform((v) => {
+  return remap$(v, {
+    "text_column": "textColumn",
+    "text1_column": "text1Column",
+    "text2_column": "text2Column",
+    "score_column": "scoreColumn",
+    "label_column": "labelColumn",
+    "id_column": "idColumn",
+    "title_column": "titleColumn",
+    "query_id_column": "queryIdColumn",
+    "corpus_id_column": "corpusIdColumn",
+    "corpus_ids_column": "corpusIdsColumn",
+    "train_dataset": "trainDataset",
+    "corpus_dataset": "corpusDataset",
+    "qrels_dataset": "qrelsDataset",
+    "top_ranked_dataset": "topRankedDataset",
+    "samples_per_label": "samplesPerLabel",
+    "n_experiments": "nExperiments",
+    "k_values": "kValues",
+    "max_chars": "maxChars",
+    "max_texts": "maxTexts",
+  });
+});
+/** @internal */
+export type EvalEmbeddingConfig$Outbound = {
+  kind: string;
+  text_column: string;
+  text1_column: string;
+  text2_column: string;
+  score_column: string;
+  label_column: string;
+  id_column: string;
+  title_column: string;
+  query_id_column: string;
+  corpus_id_column: string;
+  corpus_ids_column: string;
+  train_dataset?: EvalDataset$Outbound | undefined;
+  corpus_dataset?: EvalDataset$Outbound | undefined;
+  qrels_dataset?: EvalDataset$Outbound | undefined;
+  top_ranked_dataset?: EvalDataset$Outbound | undefined;
+  samples_per_label: number;
+  n_experiments: number;
+  seed: number;
+  k_values?: Array<number> | undefined;
+  max_chars: number;
+  max_texts: number;
+};
+
+/** @internal */
+export const EvalEmbeddingConfig$outboundSchema: z.ZodType<
+  EvalEmbeddingConfig$Outbound,
+  z.ZodTypeDef,
+  EvalEmbeddingConfig
+> = z.object({
+  kind: Kind$outboundSchema,
+  textColumn: z.string().default("text"),
+  text1Column: z.string().default("sentence1"),
+  text2Column: z.string().default("sentence2"),
+  scoreColumn: z.string().default("score"),
+  labelColumn: z.string().default("label"),
+  idColumn: z.string().default("_id"),
+  titleColumn: z.string().default("title"),
+  queryIdColumn: z.string().default("query-id"),
+  corpusIdColumn: z.string().default("corpus-id"),
+  corpusIdsColumn: z.string().default("corpus-ids"),
+  trainDataset: EvalDataset$outboundSchema.optional(),
+  corpusDataset: EvalDataset$outboundSchema.optional(),
+  qrelsDataset: EvalDataset$outboundSchema.optional(),
+  topRankedDataset: EvalDataset$outboundSchema.optional(),
+  samplesPerLabel: z.number().int().default(8),
+  nExperiments: z.number().int().default(10),
+  seed: z.number().int().default(42),
+  kValues: z.array(z.number().int()).optional(),
+  maxChars: z.number().int().default(60000),
+  maxTexts: z.number().int().default(50000),
+}).transform((v) => {
+  return remap$(v, {
+    textColumn: "text_column",
+    text1Column: "text1_column",
+    text2Column: "text2_column",
+    scoreColumn: "score_column",
+    labelColumn: "label_column",
+    idColumn: "id_column",
+    titleColumn: "title_column",
+    queryIdColumn: "query_id_column",
+    corpusIdColumn: "corpus_id_column",
+    corpusIdsColumn: "corpus_ids_column",
+    trainDataset: "train_dataset",
+    corpusDataset: "corpus_dataset",
+    qrelsDataset: "qrels_dataset",
+    topRankedDataset: "top_ranked_dataset",
+    samplesPerLabel: "samples_per_label",
+    nExperiments: "n_experiments",
+    kValues: "k_values",
+    maxChars: "max_chars",
+    maxTexts: "max_texts",
+  });
+});
+
+export function evalEmbeddingConfigToJSON(
+  evalEmbeddingConfig: EvalEmbeddingConfig,
+): string {
+  return JSON.stringify(
+    EvalEmbeddingConfig$outboundSchema.parse(evalEmbeddingConfig),
+  );
+}
+export function evalEmbeddingConfigFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalEmbeddingConfig, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalEmbeddingConfig$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalEmbeddingConfig' from JSON`,
+  );
+}

package/src/models/components/evaltask.ts

@@ -13,6 +13,12 @@ import {
   EvalDataset$Outbound,
   EvalDataset$outboundSchema,
 } from "./evaldataset.js";
+import {
+  EvalEmbeddingConfig,
+  EvalEmbeddingConfig$inboundSchema,
+  EvalEmbeddingConfig$Outbound,
+  EvalEmbeddingConfig$outboundSchema,
+} from "./evalembeddingconfig.js";
 import {
   EvalFewshotConfig,
   EvalFewshotConfig$inboundSchema,
@@ -65,7 +71,11 @@ export type EvalTask = {
   choices?: Array<string> | undefined;
   outputExtraction?: EvalOutputExtraction | undefined;
   metrics?: Array<EvalMetric> | undefined;
-  grader: EvalPythonGrader;
+  grader?: EvalPythonGrader | undefined;
+  /**
+   * Configuration of an `embedding` task: which columns hold the texts and labels, the auxiliary datasets, and the scoring protocol.
+   */
+  embedding?: EvalEmbeddingConfig | undefined;
   preprocess?: EvalPythonPreprocessor | undefined;
   fewshot?: EvalFewshotConfig | undefined;
   /**
@@ -94,7 +104,8 @@ export const EvalTask$inboundSchema: z.ZodType<
   choices: z.array(z.string()).optional(),
   output_extraction: EvalOutputExtraction$inboundSchema.optional(),
   metrics: z.array(EvalMetric$inboundSchema).optional(),
-  grader: EvalPythonGrader$inboundSchema,
+  grader: EvalPythonGrader$inboundSchema.optional(),
+  embedding: EvalEmbeddingConfig$inboundSchema.optional(),
   preprocess: EvalPythonPreprocessor$inboundSchema.optional(),
   fewshot: EvalFewshotConfig$inboundSchema.optional(),
   num_fewshot: z.number().int().optional(),
@@ -119,7 +130,8 @@ export type EvalTask$Outbound = {
   choices?: Array<string> | undefined;
   output_extraction?: EvalOutputExtraction$Outbound | undefined;
   metrics?: Array<EvalMetric$Outbound> | undefined;
-  grader: EvalPythonGrader$Outbound;
+  grader?: EvalPythonGrader$Outbound | undefined;
+  embedding?: EvalEmbeddingConfig$Outbound | undefined;
   preprocess?: EvalPythonPreprocessor$Outbound | undefined;
   fewshot?: EvalFewshotConfig$Outbound | undefined;
   num_fewshot?: number | undefined;
@@ -142,7 +154,8 @@ export const EvalTask$outboundSchema: z.ZodType<
   choices: z.array(z.string()).optional(),
   outputExtraction: EvalOutputExtraction$outboundSchema.optional(),
   metrics: z.array(EvalMetric$outboundSchema).optional(),
-  grader: EvalPythonGrader$outboundSchema,
+  grader: EvalPythonGrader$outboundSchema.optional(),
+  embedding: EvalEmbeddingConfig$outboundSchema.optional(),
   preprocess: EvalPythonPreprocessor$outboundSchema.optional(),
   fewshot: EvalFewshotConfig$outboundSchema.optional(),
   numFewshot: z.number().int().optional(),

package/src/models/components/evaltasktype.ts

@@ -16,6 +16,7 @@ export const EvalTaskType = {
   Math: "math",
   Custom: "custom",
   Transcription: "transcription",
+  Embedding: "embedding",
 } as const;
 export type EvalTaskType = ClosedEnum<typeof EvalTaskType>;
 

package/src/models/components/index.ts

@@ -232,6 +232,7 @@ export * from "./errorevent.js";
 export * from "./evalartifactsobject.js";
 export * from "./evaldataset.js";
 export * from "./evaldatasetformat.js";
+export * from "./evalembeddingconfig.js";
 export * from "./evalfewshotconfig.js";
 export * from "./evalgenerationconfig.js";
 export * from "./evalhistoricalimportobject.js";