@toloka-platform/sdk 0.2.0-preview.2 → 0.2.0-preview.3

package/src/cli/project-context.ts

@@ -29,6 +29,7 @@ export type ProjectCreateApi = {
     readonly create: (input: {
       readonly teamId: string;
       readonly name?: string;
+      readonly deliveryOrderId?: string;
     }) => Promise<ProjectRecord>;
     readonly listAll: (params?: object) => AsyncIterable<ProjectRecord>;
   };
@@ -41,16 +42,11 @@ export const NO_PROJECT_SELECTED_MESSAGE =
   "No project selected. Run toloka project create for a new project, toloka project add for an existing project, or cd into a checkout.";
 
 const NETWORK_MISMATCH_MESSAGE =
-  "project add writes the machine registry; unset --api-key / --base-url / POTATO_API_KEY / POTATO_BASE_URL, or run login against that platform first.";
+  "project add writes the machine registry; unset --api-key / --base-url / TOLOKA_API_KEY / TOLOKA_BASE_URL, or run login against that platform first.";
 
 const ALIAS_PATTERN = /^[^/\s]+$/u;
 
-export type ProjectContextSource =
-  | "flag"
-  | "toloka-project-id"
-  | "potato-project-id"
-  | "toloka.json"
-  | "registry";
+export type ProjectContextSource = "flag" | "toloka-project-id" | "toloka.json" | "registry";
 
 export type ProjectContext = {
   readonly id: string;
@@ -93,7 +89,7 @@ const effectiveNetworkBaseUrl = (
   stored: StoredCredentials | undefined,
 ): string | undefined =>
   firstNonEmpty(flags?.baseUrl) ??
-  firstNonEmpty(environment?.POTATO_BASE_URL) ??
+  firstNonEmpty(environment?.TOLOKA_BASE_URL) ??
   resolveInternalBaseUrl(environment ?? {}) ??
   firstNonEmpty(stored?.baseUrl);
 
@@ -144,7 +140,6 @@ export function resolveProjectContext(
   }> = [
     { id: firstNonEmpty(flags.projectId), source: "flag" },
     { id: firstNonEmpty(environment.TOLOKA_PROJECT_ID), source: "toloka-project-id" },
-    { id: firstNonEmpty(environment.POTATO_PROJECT_ID), source: "potato-project-id" },
     { id: firstNonEmpty(options.state?.projectId), source: "toloka.json" },
     {
       id:
@@ -183,10 +178,10 @@ const assertStoredNetworkMatches = (
   environment: TolokaEnvironment,
   stored: StoredCredentials,
 ) => {
-  const flagOrEnvKey = firstNonEmpty(flags.apiKey) ?? firstNonEmpty(environment.POTATO_API_KEY);
+  const flagOrEnvKey = firstNonEmpty(flags.apiKey) ?? firstNonEmpty(environment.TOLOKA_API_KEY);
   const flagOrEnvBase =
     firstNonEmpty(flags.baseUrl) ??
-    firstNonEmpty(environment.POTATO_BASE_URL) ??
+    firstNonEmpty(environment.TOLOKA_BASE_URL) ??
     resolveInternalBaseUrl(environment);
   assert(flagOrEnvKey === undefined || flagOrEnvKey === stored.apiKey, NETWORK_MISMATCH_MESSAGE);
   assert(
@@ -464,6 +459,7 @@ export async function createProject(options: {
   readonly alias?: string;
   readonly name?: string;
   readonly teamId?: string;
+  readonly deliveryOrderId?: string;
   readonly sourceRoot?: string;
   readonly afterPreflight?: () => Promise<void>;
 }): Promise<{ readonly output: string; readonly projectId: string; readonly alias: string }> {
@@ -489,9 +485,11 @@ export async function createProject(options: {
   await options.afterPreflight?.();
 
   const teamId = await resolveTeamId(options.client, options.teamId);
+  const deliveryOrderId = firstNonEmpty(options.deliveryOrderId);
   const created = await options.client.projects.create({
     teamId,
     name: firstNonEmpty(options.name) ?? alias,
+    ...(deliveryOrderId === undefined ? {} : { deliveryOrderId }),
   });
 
   try {

package/src/cli/sdk-version.ts

@@ -0,0 +1,38 @@
+import packageMetadata from "../../package.json" with { type: "json" };
+
+export const SDK_PACKAGE_NAME = packageMetadata.name;
+export const SDK_VERSION = packageMetadata.version;
+
+export type ReleaseChannel = "latest" | "preview";
+
+export const parseSdkVersion = (version: string): readonly number[] | undefined => {
+  const match = /^(\d+)\.(\d+)\.(\d+)(?:-preview\.(\d+))?$/u.exec(version);
+  if (match === null) return undefined;
+  const [, major, minor, patch, preview] = match;
+  return [
+    Number(major),
+    Number(minor),
+    Number(patch),
+    preview === undefined ? Number.POSITIVE_INFINITY : Number(preview),
+  ];
+};
+
+export const releaseChannelFor = (version: string): ReleaseChannel | undefined => {
+  const parsed = parseSdkVersion(version);
+  if (parsed === undefined) return undefined;
+  return parsed[3] === Number.POSITIVE_INFINITY ? "latest" : "preview";
+};
+
+export const compareSdkVersions = (left: string, right: string): number | undefined => {
+  const parsedLeft = parseSdkVersion(left);
+  const parsedRight = parseSdkVersion(right);
+  if (parsedLeft === undefined || parsedRight === undefined) return undefined;
+  const length = Math.max(parsedLeft.length, parsedRight.length);
+  for (let index = 0; index < length; index += 1) {
+    const leftPart = parsedLeft[index] ?? 0;
+    const rightPart = parsedRight[index] ?? 0;
+    if (leftPart < rightPart) return -1;
+    if (leftPart > rightPart) return 1;
+  }
+  return 0;
+};

package/src/cli/skill.ts

@@ -4,6 +4,7 @@ import { copyFile, mkdir, mkdtemp, readFile, realpath, rename, rm, stat } from "
 import { dirname, join } from "node:path";
 import { assertSafeDestination, entryStats, hasErrorCode } from "./file-system";
 import { formatCliError } from "./format-error";
+import { releaseChannelFor, SDK_PACKAGE_NAME, SDK_VERSION } from "./sdk-version";
 
 export const SKILL_CATALOG_MANIFEST = "catalog.json";
 export const SKILL_INSTALL_ROOT = ".agents/skills";
@@ -15,6 +16,9 @@ const MAX_MANIFEST_BYTES = 1024 * 1024;
 const MAX_SKILL_ID_LENGTH = 64;
 const SKILL_ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u;
 const DIGEST_PATTERN = /^[a-f0-9]{64}$/u;
+const UPGRADE_HINT = `bun add --global ${SDK_PACKAGE_NAME}@${
+  releaseChannelFor(SDK_VERSION) ?? "latest"
+}`;
 
 export type SkillDescriptor = {
   readonly id: string;
@@ -269,7 +273,7 @@ export async function installSkill({
   const skill = catalog.skills.find((candidate) => candidate.id === skillId);
   assert(
     skill,
-    `Skill ${JSON.stringify(skillId)} is not in the catalog. Run \`toloka skill list\` to see the available skills.`,
+    `Skill ${JSON.stringify(skillId)} is not in the catalog bundled with ${SDK_PACKAGE_NAME} ${SDK_VERSION}. Run \`toloka skill list\` to see the available skills; a newer release may ship it (${UPGRADE_HINT}).`,
   );
   const relativeDestination = `${SKILL_INSTALL_ROOT}/${skill.id}`;
   await assertSafeDestination(sourceRoot, relativeDestination);
@@ -348,7 +352,7 @@ export async function skillSearchCommand({
     return;
   }
   io.stdout(
-    `No skills match ${JSON.stringify(query)}. Run \`toloka skill list\` to see the catalog.`,
+    `No skills match ${JSON.stringify(query)} in the catalog bundled with ${SDK_PACKAGE_NAME} ${SDK_VERSION}. Run \`toloka skill list\` to see it; a newer release may add more (${UPGRADE_HINT}).`,
   );
 }
 

package/src/cli/terminal-io.ts

@@ -0,0 +1,103 @@
+import { createInterface } from "node:readline/promises";
+import type { AuthPrompt } from "./auth";
+import type { CliIo } from "./context";
+import { pickPipeline } from "./runs";
+
+const readStandardInput = async (): Promise<string> => {
+  process.stdin.setEncoding("utf8");
+  const chunks: string[] = [];
+  for await (const chunk of process.stdin) chunks.push(String(chunk));
+  return chunks.join("");
+};
+
+const readMaskedLine = async (question: string): Promise<string> => {
+  process.stdout.write(question);
+  const wasRaw = process.stdin.isRaw;
+  process.stdin.setRawMode(true);
+  process.stdin.resume();
+  process.stdin.setEncoding("utf8");
+
+  const { promise, resolve, reject } = Promise.withResolvers<string>();
+  let value = "";
+
+  const restore = () => {
+    process.stdin.off("data", onData);
+    process.stdin.off("end", onEnd);
+    process.stdin.off("error", onError);
+    process.stdin.setRawMode(wasRaw);
+    process.stdin.pause();
+  };
+
+  const finish = () => {
+    restore();
+    process.stdout.write("\n");
+  };
+
+  const onEnd = () => {
+    finish();
+    reject(new Error("Standard input closed before an API key was entered."));
+  };
+
+  const onError = (error: Error) => {
+    finish();
+    reject(error);
+  };
+
+  const onData = (chunk: string) => {
+    for (const character of chunk) {
+      if (character === "\r" || character === "\n") {
+        finish();
+        resolve(value);
+        return;
+      }
+      if (character === "\u0003" || character === "\u0004") {
+        finish();
+        reject(
+          new Error(
+            character === "\u0003"
+              ? "Canceled."
+              : "Standard input closed before an API key was entered.",
+          ),
+        );
+        return;
+      }
+      if (character === "\u007f" || character === "\b") {
+        if (value.length > 0) {
+          value = value.slice(0, -1);
+          process.stdout.write("\b \b");
+        }
+        continue;
+      }
+      value += character;
+      process.stdout.write("*");
+    }
+  };
+
+  process.stdin.on("data", onData);
+  process.stdin.once("end", onEnd);
+  process.stdin.once("error", onError);
+
+  try {
+    return await promise;
+  } finally {
+    restore();
+  }
+};
+
+const promptFromTerminal: AuthPrompt = async (question, options = {}) => {
+  if (options.mask === true) return readMaskedLine(question);
+
+  const input = createInterface({ input: process.stdin, output: process.stdout, terminal: true });
+  try {
+    return await input.question(question);
+  } finally {
+    input.close();
+  }
+};
+
+export const terminalIo = (): CliIo => ({
+  stdout: (text) => console.log(text),
+  stderr: (text) => console.error(text),
+  readStdin: readStandardInput,
+  ...(process.stdin.isTTY ? { prompt: promptFromTerminal, select: pickPipeline } : {}),
+});

package/src/cli/trainings.ts

@@ -5,7 +5,14 @@ import { randomUUID } from "node:crypto";
 
 import { assert } from "es-toolkit";
 
-import type { PotatoClient, Training, TrainingCostEstimate, TrainingRun } from "../client";
+import type {
+  PotatoClient,
+  Training,
+  TrainingCostEstimate,
+  TrainingRun,
+  TrainingRunMetricHistory,
+  TrainingRunMetrics,
+} from "../client";
 import type { BaseModelId, TrainingConfigurationInput } from "../../openapi/gen/models";
 
 type TrainingMethod = TrainingConfigurationInput["method"];
@@ -33,6 +40,7 @@ export type TrainingDraftOptions = {
 type TrainingApi = Pick<PotatoClient, "trainings">;
 type TrainingDraftApi = Pick<PotatoClient, "trainings" | "files" | "datasets">;
 type TrainingCostApi = Pick<PotatoClient, "trainings" | "waitFor">;
+type TrainingRunApi = Pick<PotatoClient, "trainings" | "waitFor">;
 const GISTING_PROMPT_MAX_BYTES = 256 * 1024;
 type TrainingIo = { readonly stdout: (text: string) => void };
 type TableRow = readonly string[];
@@ -446,7 +454,10 @@ export const createTraining = async (
   io: TrainingIo,
 ): Promise<void> => {
   assert(input.datasetId, "--dataset is required.");
-  assert(!input.validationFilter || input.validationDatasetId, "--validation-filter requires --validation-dataset.");
+  assert(
+    !input.validationFilter || input.validationDatasetId,
+    "--validation-filter requires --validation-dataset.",
+  );
 
   const configuration = await buildTrainingConfiguration({
     client,
@@ -458,12 +469,7 @@ export const createTraining = async (
     name: input.name,
     configuration,
   });
-  writeOutput(
-    io,
-    input.json,
-    training,
-    `Created training ${training.id}`,
-  );
+  writeOutput(io, input.json, training, `Created training ${training.id}`);
 };
 
 export const updateTraining = async (
@@ -485,19 +491,13 @@ export const updateTraining = async (
       options: input,
     }),
   });
-  writeOutput(
-    io,
-    input.json,
-    updated,
-    `Updated training ${updated.id}`,
-  );
+  writeOutput(io, input.json, updated, `Updated training ${updated.id}`);
 };
 
-export const estimateTrainingCost = async (
+const prepareTrainingCostEstimate = async (
   client: TrainingCostApi,
-  input: { projectId: string; nameOrId: string; idempotencyKey?: string; json: boolean },
-  io: TrainingIo,
-): Promise<void> => {
+  input: { nameOrId: string; projectId: string; idempotencyKey?: string },
+): Promise<{ training: Training; estimate: TrainingCostEstimate }> => {
   const training = await resolveTraining(client, input.projectId, input.nameOrId);
   const inputSet = await client.trainings.prepareCostEstimateInputs({
     trainingId: training.id,
@@ -511,33 +511,53 @@ export const estimateTrainingCost = async (
     exportIds.map((exportId) => client.waitFor({ type: "datasetExport", exportId })),
   );
 
-  const estimate = await client.trainings.estimateCost({
-    trainingId: training.id,
-    inputSetId: inputSet.id,
-  });
+  return {
+    training,
+    estimate: await client.trainings.estimateCost({
+      trainingId: training.id,
+      inputSetId: inputSet.id,
+    }),
+  };
+};
+
+export const estimateTrainingCost = async (
+  client: TrainingCostApi,
+  input: { projectId: string; nameOrId: string; idempotencyKey?: string; json: boolean },
+  io: TrainingIo,
+): Promise<void> => {
+  const { estimate } = await prepareTrainingCostEstimate(client, input);
   writeOutput(io, input.json, estimate, formatCostEstimate(estimate));
 };
 
 export const createTrainingRun = async (
-  client: TrainingApi,
+  client: TrainingRunApi,
   input: {
     projectId: string;
     nameOrId: string;
-    estimateFile: string;
-    versionName: string;
+    estimateFile?: string;
+    versionName?: string;
     idempotencyKey?: string;
     json: boolean;
   },
   io: TrainingIo,
 ): Promise<void> => {
   const training = await resolveTraining(client, input.projectId, input.nameOrId);
-  const estimate = parseReadyEstimateFile(JSON.parse(await readFile(input.estimateFile, "utf8")));
+  const estimate = input.estimateFile
+    ? parseReadyEstimateFile(JSON.parse(await readFile(input.estimateFile, "utf8")))
+    : await (async () => {
+        const prepared = await prepareTrainingCostEstimate(client, {
+          projectId: input.projectId,
+          nameOrId: input.nameOrId,
+        });
+        assert(prepared.estimate.status === "ready", formatCostEstimate(prepared.estimate));
+        return parseReadyEstimateFile(prepared.estimate);
+      })();
   assert(estimate.trainingId === training.id, "The estimate file belongs to another Training.");
   const run = await client.trainings.createRun({
     trainingId: training.id,
     inputSetId: estimate.inputSetId,
     estimateFingerprint: estimate.fingerprint,
-    versionName: input.versionName,
+    versionName: input.versionName ?? training.name,
     idempotencyKey: input.idempotencyKey ?? randomUUID(),
   });
   writeOutput(io, input.json, run, formatTrainingRun(run));
@@ -625,3 +645,74 @@ export const readTrainingRunLogs = async (
     });
   }
 };
+
+const formatMetricValue = (point: {
+  value: number | null;
+  nonFinite: "nan" | "positive_infinity" | "negative_infinity" | null;
+}): string => {
+  if (point.nonFinite === "nan") return "NaN";
+  if (point.nonFinite === "positive_infinity") return "Infinity";
+  if (point.nonFinite === "negative_infinity") return "-Infinity";
+  return point.value === null ? "-" : String(point.value);
+};
+
+const METRICS_UNAVAILABLE: Record<"not_configured" | "unresolved", string> = {
+  not_configured: "This Training Run has no metrics: it started before metrics were available, or the server records none.",
+  unresolved: "Metrics are temporarily unreachable. The Training Run itself is unaffected; try again later.",
+};
+
+const formatTrainingRunMetrics = (metrics: TrainingRunMetrics): string => {
+  if (!metrics.available) {
+    return metrics.unavailableReason === null
+      ? "No metrics are available for this Training Run."
+      : METRICS_UNAVAILABLE[metrics.unavailableReason];
+  }
+  if (metrics.metrics.length === 0) return "No metrics have been reported yet.";
+
+  const rows = metrics.metrics.map((entry) => [
+    entry.key,
+    entry.metric ?? "-",
+    formatMetricValue(entry.latest),
+    String(entry.latest.step),
+  ]);
+  const table = formatTable([["Metric", "Meaning", "Latest", "Step"], ...rows]);
+  return metrics.truncated ? `${table}\n\nMore metrics exist than the server returns.` : table;
+};
+
+const formatTrainingRunMetricHistory = (history: TrainingRunMetricHistory): string => {
+  const header = formatTable([
+    ["Metric", history.key],
+    ["Meaning", history.metric ?? "-"],
+    ["Points", String(history.points.length)],
+  ]);
+  const points = formatTable([
+    ["Step", "Value"],
+    ...history.points.map((point) => [String(point.step), formatMetricValue(point)]),
+  ]);
+  const note = history.truncated
+    ? "\n\nNot every recorded point is shown: the series is thinned to fit --resolution, and a very long run keeps only its most recent points."
+    : "";
+  return `${header}\n\n${points}${note}`;
+};
+
+export const readTrainingRunMetrics = async (
+  client: TrainingApi,
+  input: { trainingRunId: string; json: boolean },
+  io: TrainingIo,
+): Promise<void> => {
+  const metrics = await client.trainings.getRunMetrics({ trainingRunId: input.trainingRunId });
+  writeOutput(io, input.json, metrics, formatTrainingRunMetrics(metrics));
+};
+
+export const readTrainingRunMetricHistory = async (
+  client: TrainingApi,
+  input: { trainingRunId: string; metricKey: string; resolution?: number; json: boolean },
+  io: TrainingIo,
+): Promise<void> => {
+  const history = await client.trainings.getRunMetricHistory({
+    trainingRunId: input.trainingRunId,
+    metricKey: input.metricKey,
+    resolution: input.resolution,
+  });
+  writeOutput(io, input.json, history, formatTrainingRunMetricHistory(history));
+};

package/src/cli/version.ts

@@ -3,7 +3,7 @@ import { homedir } from "node:os";
 import { dirname, join, resolve, sep } from "node:path";
 
 import { assert } from "es-toolkit";
-import { SDK_VERSION } from "./init";
+import { SDK_VERSION } from "./sdk-version";
 
 export type CliInstallChannel = "global-registry" | "linked" | "project-local";
 

package/src/client.ts

@@ -25,6 +25,7 @@ import type {
   ImportPipelineBody,
   InferredField as GeneratedInferredField,
   GetTrainingRunLogsParams,
+  GetTrainingRunMetricHistoryParams,
   ListDatasetFieldsParams,
   ListModelVersionsParams,
   ListDatasetItemsParams,
@@ -45,6 +46,8 @@ import type {
   TrainingCostEstimate as GeneratedTrainingCostEstimate,
   TrainingCostEstimateInputSet as GeneratedTrainingCostEstimateInputSet,
   TrainingRun as GeneratedTrainingRun,
+  TrainingRunMetricHistory as GeneratedTrainingRunMetricHistory,
+  TrainingRunMetrics as GeneratedTrainingRunMetrics,
   TrainingRunLogs as GeneratedTrainingRunLogs,
   ReplaceTrainingConfigurationBody,
   UpdateTrainingMetadataBody,
@@ -164,6 +167,8 @@ export type TrainingCostEstimateInputSet = BrandedResource<
 export type TrainingCostEstimate = BrandResourceUnion<GeneratedTrainingCostEstimate>;
 export type TrainingRun = BrandedResource<GeneratedTrainingRun, TrainingRunId>;
 export type TrainingRunLogs = GeneratedTrainingRunLogs;
+export type TrainingRunMetrics = GeneratedTrainingRunMetrics;
+export type TrainingRunMetricHistory = GeneratedTrainingRunMetricHistory;
 export type BaseModel = GeneratedBaseModel;
 export type BaseModelId = GeneratedBaseModelId;
 export type Pipeline = BrandedResource<GeneratedPipeline, PipelineId>;
@@ -482,6 +487,14 @@ const createClientImplementation = (config: ClientConfig) => {
       ...params
     }: { trainingRunId: TrainingRunId } & GetTrainingRunLogsParams) =>
       wire.getTrainingRunLogs(trainingRunId, params, requestOptions),
+    getRunMetrics: ({ trainingRunId }: { trainingRunId: TrainingRunId }) =>
+      wire.getTrainingRunMetrics(trainingRunId, requestOptions),
+    getRunMetricHistory: ({
+      trainingRunId,
+      metricKey,
+      ...params
+    }: { trainingRunId: TrainingRunId; metricKey: string } & GetTrainingRunMetricHistoryParams) =>
+      wire.getTrainingRunMetricHistory(trainingRunId, metricKey, params, requestOptions),
   };
 
   const datasets = {
@@ -624,6 +637,10 @@ const createClientImplementation = (config: ClientConfig) => {
       wire.getPipelineCheck(checkRunId, requestOptions),
   };
 
+  const deliveryOrders = {
+    list: async () => (await wire.listDeliveryOrders(requestOptions)).deliveryOrders,
+  };
+
   return {
     me: () => wire.getCurrentUser(requestOptions),
     projects,
@@ -636,6 +653,7 @@ const createClientImplementation = (config: ClientConfig) => {
     secrets,
     files,
     checks,
+    deliveryOrders,
     waitFor,
   };
 };

package/src/importer.ts

@@ -263,6 +263,32 @@ function transformExam<Exam extends { gradedEntities: Array<{ targetEntity: stri
   };
 }
 
+const generationCheckNames: readonly string[] = [
+  "ui",
+  "llmqa",
+  "qc_lint",
+  "ui_types_lint",
+  "exam_dataset",
+  "synthgen",
+];
+const quorumCheckNames: readonly string[] = [
+  "ui",
+  "llmqa",
+  "qc_lint",
+  "ui_types_lint",
+  "exam_dataset",
+];
+
+const normalizeImportedChecks = (
+  checks: Record<string, unknown[]> | undefined,
+  requiredNames: readonly string[],
+): Record<string, unknown[]> | undefined => {
+  if (checks === undefined) return undefined;
+  const normalized = { ...checks };
+  for (const name of requiredNames) normalized[name] ??= [];
+  return normalized;
+};
+
 function authoringSettings(node: GraphNode, fieldReference: FieldReference): unknown {
   switch (node.type) {
     case "root": {
@@ -296,9 +322,11 @@ function authoringSettings(node: GraphNode, fieldReference: FieldReference): unk
       };
     }
     case "generation": {
-      const { entities, qualityCriteria, exam, ...settings } = node.settings;
+      const { entities, qualityCriteria, exam, checks, ...settings } = node.settings;
+      const normalizedChecks = normalizeImportedChecks(checks, generationCheckNames);
       return {
         ...settings,
+        ...(normalizedChecks === undefined ? {} : { checks: normalizedChecks }),
         entities: {
           input: entities.input.map((entity) => transformEntity(entity, fieldReference)),
           output: entities.output.map((entity) => transformEntity(entity, fieldReference)),
@@ -326,9 +354,11 @@ function authoringSettings(node: GraphNode, fieldReference: FieldReference): unk
       };
     }
     case "quorum": {
-      const { entities, qualityCriteria, exam, ...settings } = node.settings;
+      const { entities, qualityCriteria, exam, checks, ...settings } = node.settings;
+      const normalizedChecks = normalizeImportedChecks(checks, quorumCheckNames);
       return {
         ...settings,
+        ...(normalizedChecks === undefined ? {} : { checks: normalizedChecks }),
         entities: {
           input: entities.input.map((entity) => transformEntity(entity, fieldReference)),
           output: entities.output.map((entity) => transformEntity(entity, fieldReference)),