@toloka-platform/sdk 0.11.1 → 0.11.3

package/src/cli/project.ts

@@ -10,6 +10,7 @@ import { resolvePipelineDefinition, validatePipelineDefinition } from "../author
 import type { Dataset, DatasetField, Pipeline, PotatoClient } from "../client";
 import { generatePipelineSourceTree } from "../importer";
 import { readTolokaState, TOLOKA_STATE_FILENAME, writeTolokaState } from "../state";
+import { platformRuleIssues } from "../validate";
 import type { TolokaState } from "../state";
 import { assertProjectSelected } from "./project-context";
 import { collectGraphSecretNames } from "./secrets";
@@ -82,9 +83,11 @@ export type ApplyProjectResult = {
   readonly output: string;
 };
 
-export type ValidateProjectOptions = PlanProjectOptions;
+export type ValidateProjectOptions = PlanProjectOptions & {
+  readonly wait?: { readonly timeoutMs: number; readonly pollIntervalMs: number };
+};
 
-export type PipelineValidationStatus = "valid" | "invalid" | "checks_pending";
+export type PipelineValidationStatus = "valid" | "invalid" | "checks_pending" | "checks_running";
 
 export type PipelineValidationResult = {
   readonly sourceDirectory: string;
@@ -92,6 +95,7 @@ export type PipelineValidationResult = {
   readonly report?: PipelineValidationReport;
   readonly status?: PipelineValidationStatus;
   readonly skipped?: string;
+  readonly note?: string;
 };
 
 export type ValidateProjectResult = {
@@ -261,6 +265,13 @@ export function diffPipelineGraphs(
   ];
 }
 
+type CheckedNode = Extract<PipelineGraphV1["nodes"][number], { type: "code" | "generation" | "quorum" }>;
+
+const withLiveChecks = <Node extends CheckedNode>(authored: Node, live: Node): Node =>
+  live.settings.checks === undefined
+    ? authored
+    : { ...authored, settings: { ...authored.settings, checks: { ...live.settings.checks, ...authored.settings.checks } } };
+
 const graphWithLiveState = (
   authoredGraph: PipelineGraphV1,
   liveGraph: PipelineGraphV1,
@@ -277,28 +288,43 @@ const graphWithLiveState = (
             ? {}
             : { itemLimit: liveRoot.settings.itemLimit }),
         };
+  const liveNodes = new Map(liveGraph.nodes.map((node) => [node.id, node]));
   return {
     ...authoredGraph,
     ...(liveGraph.layout === undefined ? {} : { layout: liveGraph.layout }),
-    nodes: authoredGraph.nodes.map((node) =>
-      node.type === "root" ? { ...node, settings: { ...liveRunScope, ...node.settings } } : node,
-    ),
+    nodes: authoredGraph.nodes.map((node) => {
+      const live = liveNodes.get(node.id);
+      switch (node.type) {
+        case "root":
+          return { ...node, settings: { ...liveRunScope, ...node.settings } };
+        case "code":
+          return live?.type === "code" ? withLiveChecks(node, live) : node;
+        case "generation":
+          return live?.type === "generation" ? withLiveChecks(node, live) : node;
+        case "quorum":
+          return live?.type === "quorum" ? withLiveChecks(node, live) : node;
+        case "match":
+          return node;
+      }
+    }),
   };
 };
 
 export const desiredDatasets = (source: ProjectSource): readonly DesiredDataset[] => {
   const fieldsByDataset = new Map<string, Map<string, PipelineGraphV1["fields"][number]>>();
   for (const pipeline of source.pipelines) {
-    let fields = fieldsByDataset.get(pipeline.definition.dataset);
+    const datasetName = pipeline.definition.dataset;
+    if (datasetName === undefined) continue;
+    let fields = fieldsByDataset.get(datasetName);
     if (fields === undefined) {
       fields = new Map();
-      fieldsByDataset.set(pipeline.definition.dataset, fields);
+      fieldsByDataset.set(datasetName, fields);
     }
     for (const field of pipeline.definition.graph.fields) {
       const existing = fields.get(field.name);
       assertCli(
         existing === undefined || valueKey(existing.jsonSchema) === valueKey(field.jsonSchema),
-        `Dataset "${pipeline.definition.dataset}" has conflicting schemas for field "${field.name}".`,
+        `Dataset "${datasetName}" has conflicting schemas for field "${field.name}".`,
         { hint: { description: "Give the field one JSON schema across all pipelines that share the dataset." } },
       );
       if (existing === undefined) fields.set(field.name, field);
@@ -452,6 +478,29 @@ const inspectDatasets = async (
   });
 };
 
+const pipelineGraphChanges = (
+  pipelineSource: ProjectPipelineSource,
+  snapshot: PipelineGraphSnapshot | undefined,
+  datasets: readonly DatasetInspection[],
+): readonly PipelinePlanChange[] => {
+  if (snapshot === undefined) return [];
+  const datasetName = pipelineSource.definition.dataset;
+  const liveDatasetId = snapshot.graph.nodes.find((node) => node.type === "root")?.settings.datasetId;
+  if (datasetName === undefined && liveDatasetId !== undefined)
+    return [{ kind: "node", action: "removed", label: "root dataset binding", id: "root" }];
+  const dataset =
+    datasetName === undefined
+      ? undefined
+      : datasets.find((entry) => entry.name === datasetName)?.dataset;
+  if (datasetName !== undefined && dataset === undefined)
+    return [{ kind: "node", action: "changed", label: "root dataset binding", id: "root" }];
+  const desiredGraph = resolvePipelineDefinition(
+    pipelineSource.definition,
+    dataset === undefined ? {} : { datasetId: dataset.id },
+  );
+  return diffPipelineGraphs(graphWithLiveState(desiredGraph, snapshot.graph), snapshot.graph);
+};
+
 const inspectPipelines = async (
   client: PotatoClient,
   source: ProjectSource,
@@ -474,27 +523,11 @@ const inspectPipelines = async (
   );
   return pipelinePairs.map(({ source: pipelineSource, pipeline }, index) => {
     const snapshot = snapshots[index];
-    const dataset = datasets.find(
-      (entry) => entry.name === pipelineSource.definition.dataset,
-    )?.dataset;
-    let graphChanges: readonly PipelinePlanChange[] = [];
-    if (pipeline !== undefined && snapshot !== undefined) {
-      graphChanges =
-        dataset === undefined
-          ? [{ kind: "node", action: "changed", label: "root dataset binding", id: "root" }]
-          : diffPipelineGraphs(
-              graphWithLiveState(
-                resolvePipelineDefinition(pipelineSource.definition, { datasetId: dataset.id }),
-                snapshot.graph,
-              ),
-              snapshot.graph,
-            );
-    }
     return {
       source: pipelineSource,
       ...(pipeline === undefined ? {} : { pipeline }),
       ...(snapshot === undefined ? {} : { snapshot }),
-      graphChanges,
+      graphChanges: pipelineGraphChanges(pipelineSource, snapshot, datasets),
     };
   });
 };
@@ -638,13 +671,8 @@ export async function importProject(
   );
   const datasetIds = new Set<string>();
   for (const snapshot of snapshots) {
-    const rootNode = snapshot.graph.nodes.find((node) => node.type === "root");
-    assertCli(
-      rootNode?.settings.datasetId,
-      "Every imported pipeline must have a bound dataset.",
-      { hint: { description: "Bind a dataset to the pipeline in the platform, then re-run toloka import." } },
-    );
-    datasetIds.add(rootNode.settings.datasetId);
+    const datasetId = snapshot.graph.nodes.find((node) => node.type === "root")?.settings.datasetId;
+    if (datasetId !== undefined) datasetIds.add(datasetId);
   }
   const projectDatasets = await collectAsync(
     client.datasets.listAll({ projectId: options.projectId }),
@@ -667,25 +695,22 @@ export async function importProject(
   for (const [index, pipeline] of pipelines.entries()) {
     const snapshot = snapshots[index];
     assertCli(snapshot, `Pipeline ${pipeline.id} has no graph snapshot.`, { hint: REPORT_HINT });
-    const rootNode = snapshot.graph.nodes.find((node) => node.type === "root");
-    assertCli(
-      rootNode?.settings.datasetId,
-      `Pipeline ${pipeline.id} has no dataset binding.`,
-      { hint: { description: "Bind a dataset to the pipeline in the platform, then re-run toloka import." } },
-    );
-    const dataset = datasetsById.get(rootNode.settings.datasetId);
+    const datasetId = snapshot.graph.nodes.find((node) => node.type === "root")?.settings.datasetId;
+    const dataset = datasetId === undefined ? undefined : datasetsById.get(datasetId);
     assertCli(
-      dataset,
-      `Pipeline ${pipeline.id} references unknown dataset ${rootNode.settings.datasetId}.`,
+      datasetId === undefined || dataset !== undefined,
+      `Pipeline ${pipeline.id} references unknown dataset ${datasetId}.`,
       { hint: { description: "Check toloka dataset list; the pipeline's dataset binding must point at a dataset of this project." } },
     );
-    const existingDatasetBinding = stateDatasets[dataset.name];
-    assertCli(
-      existingDatasetBinding === undefined || existingDatasetBinding.datasetId === dataset.id,
-      `Project has multiple referenced datasets named "${dataset.name}".`,
-      { hint: { description: "Rename the duplicate datasets in the platform, then re-run toloka import." } },
-    );
-    stateDatasets[dataset.name] = { datasetId: dataset.id };
+    if (dataset !== undefined) {
+      const existingDatasetBinding = stateDatasets[dataset.name];
+      assertCli(
+        existingDatasetBinding === undefined || existingDatasetBinding.datasetId === dataset.id,
+        `Project has multiple referenced datasets named "${dataset.name}".`,
+        { hint: { description: "Rename the duplicate datasets in the platform, then re-run toloka import." } },
+      );
+      stateDatasets[dataset.name] = { datasetId: dataset.id };
+    }
     const sourceDirectory = directories.get(pipeline.id);
     assertCli(
       sourceDirectory,
@@ -694,7 +719,10 @@ export async function importProject(
     );
     statePipelines[sourceDirectory] = { pipelineId: pipeline.id };
     for (const [relativePath, source] of Object.entries(
-      generatePipelineSourceTree(snapshot.graph, { dataset: dataset.name }),
+      generatePipelineSourceTree(snapshot.graph, {
+        ...(dataset === undefined ? {} : { dataset: dataset.name }),
+        invalidFieldSchemas: snapshot.invalidFieldSchemas,
+      }),
     )) {
       tree[`${sourceDirectory}/${relativePath}`] = source;
     }
@@ -777,6 +805,45 @@ export async function projectStatus(
   };
 }
 
+const applyPipeline = async (
+  client: PotatoClient,
+  projectId: string,
+  pipelineInspection: PipelineInspection,
+  datasetIds: ReadonlyMap<string, string>,
+): Promise<{ readonly pipelineId: string; readonly writes: number }> => {
+  const datasetName = pipelineInspection.source.definition.dataset;
+  const datasetId = datasetName === undefined ? undefined : datasetIds.get(datasetName);
+  assertCli(
+    datasetName === undefined || datasetId !== undefined,
+    `Dataset ${datasetName} has no resolved ID.`,
+    { hint: REPORT_HINT },
+  );
+  const desiredGraph = resolvePipelineDefinition(
+    pipelineInspection.source.definition,
+    datasetId === undefined ? {} : { datasetId },
+  );
+  if (pipelineInspection.pipeline === undefined) {
+    const pipeline = await client.pipelines.import({
+      projectId,
+      name: pipelineInspection.source.name,
+      graph: desiredGraph,
+    });
+    if (datasetId === undefined) return { pipelineId: pipeline.id, writes: 1 };
+    await client.pipelines.attachDataset({ pipelineId: pipeline.id, datasetId });
+    return { pipelineId: pipeline.id, writes: 2 };
+  }
+  const pipelineId = pipelineInspection.pipeline.id;
+  if (pipelineInspection.graphChanges.length === 0) return { pipelineId, writes: 0 };
+  const snapshot = pipelineInspection.snapshot;
+  assertCli(snapshot, `Pipeline ${pipelineId} has no graph snapshot.`, { hint: REPORT_HINT });
+  await client.pipelines.replaceGraph({
+    pipelineId,
+    graph: graphWithLiveState(desiredGraph, snapshot.graph),
+    expectedVersion: snapshot.version,
+  });
+  return { pipelineId, writes: 1 };
+};
+
 export async function applyProject(
   client: PotatoClient,
   options: ApplyProjectOptions,
@@ -787,6 +854,19 @@ export async function applyProject(
     `Refusing to delete pipelines absent from source: ${inspection.removedPipelines.join(", ")}.`,
     { hint: { description: "Restore the pipeline in source, or run toloka project untrack pipeline <name> to stop managing it." } },
   );
+  const detachedPipelines = inspection.pipelines
+    .filter((pipeline) =>
+      pipeline.graphChanges.some(
+        (change) => change.label === "root dataset binding" && change.action === "removed",
+      ),
+    )
+    .map((pipeline) => pipeline.source.sourceDirectory);
+  assertCli(
+    detachedPipelines.length === 0,
+    `Refusing to unbind the dataset of ${detachedPipelines.join(", ")}: the source declares no dataset, but the pipeline has one on the platform.`,
+    { hint: { description: "Declare the dataset in definePipeline; toloka import writes the name of the bound dataset." } },
+  );
+
   assertCli(
     inspection.removedDatasets.length === 0,
     `Refusing to delete datasets absent from source: ${inspection.removedDatasets.join(", ")}. Dataset items are never deleted.`,
@@ -806,6 +886,19 @@ export async function applyProject(
     { hint: { description: "Add the changed columns under new field names, or create a new dataset for the new schema." } },
   );
 
+  const ruleViolations = inspection.pipelines
+    .filter((pipeline) => pipeline.pipeline === undefined || pipeline.graphChanges.length > 0)
+    .flatMap((pipeline) =>
+      platformRuleIssues(pipeline.source.definition.graph).map(
+        (issue) => `- ${pipeline.source.sourceDirectory}, node ${issue.node}: ${issue.reason}`,
+      ),
+    );
+  assertCli(
+    ruleViolations.length === 0,
+    `The platform would reject these pipelines:\n${ruleViolations.join("\n")}`,
+    { hint: { description: "Fix the listed nodes in source; toloka check reports the same rules as warnings." } },
+  );
+
   let writes = 0;
   const datasetIds = new Map<string, string>();
   for (const datasetInspection of inspection.datasets) {
@@ -832,40 +925,9 @@ export async function applyProject(
 
   const pipelineIds = new Map<string, string>();
   for (const pipelineInspection of inspection.pipelines) {
-    const datasetId = datasetIds.get(pipelineInspection.source.definition.dataset);
-    assertCli(
-      datasetId,
-      `Dataset ${pipelineInspection.source.definition.dataset} has no resolved ID.`,
-      { hint: REPORT_HINT },
-    );
-    const desiredGraph = resolvePipelineDefinition(pipelineInspection.source.definition, {
-      datasetId,
-    });
-    if (pipelineInspection.pipeline === undefined) {
-      const pipeline = await client.pipelines.import({
-        projectId: inspection.projectId,
-        name: pipelineInspection.source.name,
-        graph: desiredGraph,
-      });
-      await client.pipelines.attachDataset({ pipelineId: pipeline.id, datasetId });
-      pipelineIds.set(pipelineInspection.source.sourceDirectory, pipeline.id);
-      writes += 2;
-      continue;
-    }
-    pipelineIds.set(pipelineInspection.source.sourceDirectory, pipelineInspection.pipeline.id);
-    if (pipelineInspection.graphChanges.length === 0) continue;
-    const snapshot = pipelineInspection.snapshot;
-    assertCli(
-      snapshot,
-      `Pipeline ${pipelineInspection.pipeline.id} has no graph snapshot.`,
-      { hint: REPORT_HINT },
-    );
-    await client.pipelines.replaceGraph({
-      pipelineId: pipelineInspection.pipeline.id,
-      graph: graphWithLiveState(desiredGraph, snapshot.graph),
-      expectedVersion: snapshot.version,
-    });
-    writes += 1;
+    const applied = await applyPipeline(client, inspection.projectId, pipelineInspection, datasetIds);
+    pipelineIds.set(pipelineInspection.source.sourceDirectory, applied.pipelineId);
+    writes += applied.writes;
   }
 
   const nextState: TolokaState = {
@@ -910,6 +972,10 @@ const issueLocation = (issue: { nodeId?: string; checkId?: string }) =>
   [issue.nodeId, issue.checkId].filter((value): value is string => value !== undefined).join("/");
 
 const PENDING_CHECK_CODE = "not_ran_node_check";
+const RUNNING_CHECK_CODE = "running_node_check";
+
+const hasRunningChecks = (report: PipelineValidationReport) =>
+  report.issues.some((issue) => issue.code === RUNNING_CHECK_CODE);
 
 const normalizeValidationIssue = (issue: PipelineValidationIssue): PipelineValidationIssue =>
   issue.code === "empty_node_checks"
@@ -948,72 +1014,156 @@ const normalizeValidationReport = (
 const formatValidationIssues = (report: PipelineValidationReport) =>
   report.issues.map((issue) => {
     const location = issueLocation(issue);
-    const severity = issue.code === PENDING_CHECK_CODE ? "pending" : issue.severity;
+    const severity =
+      issue.code === PENDING_CHECK_CODE
+        ? "pending"
+        : issue.code === RUNNING_CHECK_CODE
+          ? "running"
+          : issue.severity;
     return `  ${severity} ${issue.code}${location.length === 0 ? "" : ` ${location}`}: ${issue.message}`;
   });
 
+type NormalizedValidation = ReturnType<typeof normalizeValidationReport>;
+
+type RunningValidation = {
+  readonly index: number;
+  readonly validate: () => Promise<NormalizedValidation>;
+  readonly failure?: string;
+};
+
+const sleep = (ms: number) =>
+  new Promise<void>((resolve) => {
+    setTimeout(resolve, ms);
+  });
+
+const pollValidationEntry = async (
+  results: PipelineValidationResult[],
+  entry: RunningValidation,
+): Promise<RunningValidation | undefined> => {
+  try {
+    const { report, status } = await entry.validate();
+    results[entry.index] = { ...results[entry.index]!, report, status };
+    return hasRunningChecks(report) ? { index: entry.index, validate: entry.validate } : undefined;
+  } catch (error) {
+    return { ...entry, failure: error instanceof Error ? error.message : String(error) };
+  }
+};
+
+const waitForRunningChecks = async (
+  results: PipelineValidationResult[],
+  running: readonly RunningValidation[],
+  wait: NonNullable<ValidateProjectOptions["wait"]>,
+  startedAt: number,
+) => {
+  const deadline = startedAt + wait.timeoutMs;
+  let pending = running;
+  while (pending.length > 0 && Date.now() < deadline) {
+    await sleep(Math.min(wait.pollIntervalMs, deadline - Date.now()));
+    const next: RunningValidation[] = [];
+    for (const entry of pending) {
+      const retry = await pollValidationEntry(results, entry);
+      if (retry !== undefined) next.push(retry);
+    }
+    pending = next;
+  }
+  const waitedSeconds = Math.round((Date.now() - startedAt) / 1000);
+  for (const entry of pending) {
+    const result = results[entry.index]!;
+    results[entry.index] = {
+      ...result,
+      status: result.status === "invalid" ? "invalid" : "checks_running",
+      note:
+        entry.failure === undefined
+          ? `checks are still running after ${waitedSeconds}s; run toloka validate --wait again`
+          : `validation failed while waiting (${entry.failure}); run toloka validate --wait again`,
+    };
+  }
+};
+
+const validatePipelineEntry = async (
+  client: PotatoClient,
+  inspection: ProjectInspection,
+  pipeline: ProjectInspection["pipelines"][number],
+  wait: ValidateProjectOptions["wait"],
+): Promise<{
+  readonly result: PipelineValidationResult;
+  readonly poll?: () => Promise<NormalizedValidation>;
+}> => {
+  const { sourceDirectory } = pipeline.source;
+  const datasetName = pipeline.source.definition.dataset;
+  const dataset =
+    datasetName === undefined
+      ? undefined
+      : inspection.datasets.find((entry) => entry.name === datasetName)?.dataset;
+  if (pipeline.pipeline === undefined)
+    return { result: { sourceDirectory, skipped: "pipeline is not on the platform yet; run toloka apply first" } };
+  if (datasetName !== undefined && dataset === undefined)
+    return {
+      result: {
+        sourceDirectory,
+        pipelineId: pipeline.pipeline.id,
+        skipped: `dataset "${datasetName}" is not on the platform yet; run toloka apply first`,
+      },
+    };
+  const resolved = resolvePipelineDefinition(
+    pipeline.source.definition,
+    dataset === undefined ? {} : { datasetId: dataset.id },
+  );
+  const graph = pipeline.snapshot === undefined ? resolved : graphWithLiveState(resolved, pipeline.snapshot.graph);
+  const pipelineId = pipeline.pipeline.id;
+  const validatePipeline = async () =>
+    normalizeValidationReport(await client.pipelines.validateGraph({ pipelineId, graph }));
+  const { report, status } = await validatePipeline();
+  const waitForChecks = wait !== undefined && hasRunningChecks(report);
+  const unapplied = waitForChecks && pipeline.graphChanges.length > 0;
+  return {
+    result: {
+      sourceDirectory,
+      pipelineId,
+      report,
+      status,
+      ...(unapplied
+        ? { note: "lints run on the applied graph; run toloka apply, then toloka validate --wait" }
+        : {}),
+    },
+    ...(waitForChecks && !unapplied ? { poll: validatePipeline } : {}),
+  };
+};
+
+const formatValidationResult = (result: PipelineValidationResult): string[] => {
+  if (result.skipped !== undefined) return [`- ${result.sourceDirectory}: skipped (${result.skipped})`];
+  assertCli(result.report, `Pipeline ${result.sourceDirectory} is missing a validation report`, {
+    hint: REPORT_HINT,
+  });
+  assertCli(result.status, `Pipeline ${result.sourceDirectory} is missing a validation status`, {
+    hint: REPORT_HINT,
+  });
+  return [
+    `- ${result.sourceDirectory}: ${result.status.replace("_", "-")}`,
+    ...formatValidationIssues(result.report),
+    ...(result.note === undefined ? [] : [`  note: ${result.note}`]),
+  ];
+};
+
 export async function validateProject(
   client: PotatoClient,
   options: ValidateProjectOptions,
 ): Promise<ValidateProjectResult> {
+  const startedAt = Date.now();
   const inspection = await inspectProject(client, options);
+  const { wait } = options;
   const results: PipelineValidationResult[] = [];
+  const running: RunningValidation[] = [];
   for (const pipeline of inspection.pipelines) {
-    const dataset = inspection.datasets.find(
-      (entry) => entry.name === pipeline.source.definition.dataset,
-    )?.dataset;
-    if (pipeline.pipeline === undefined) {
-      results.push({
-        sourceDirectory: pipeline.source.sourceDirectory,
-        skipped: "pipeline is not on the platform yet; run toloka apply first",
-      });
-      continue;
-    }
-    if (dataset === undefined) {
-      results.push({
-        sourceDirectory: pipeline.source.sourceDirectory,
-        pipelineId: pipeline.pipeline.id,
-        skipped: `dataset "${pipeline.source.definition.dataset}" is not on the platform yet; run toloka apply first`,
-      });
-      continue;
-    }
-    const graph =
-      pipeline.snapshot === undefined
-        ? resolvePipelineDefinition(pipeline.source.definition, { datasetId: dataset.id })
-        : graphWithLiveState(
-            resolvePipelineDefinition(pipeline.source.definition, { datasetId: dataset.id }),
-            pipeline.snapshot.graph,
-          );
-    const normalized = normalizeValidationReport(
-      await client.pipelines.validateGraph({
-        pipelineId: pipeline.pipeline.id,
-        graph,
-      }),
-    );
-    results.push({
-      sourceDirectory: pipeline.source.sourceDirectory,
-      pipelineId: pipeline.pipeline.id,
-      report: normalized.report,
-      status: normalized.status,
-    });
+    const { result, poll } = await validatePipelineEntry(client, inspection, pipeline, wait);
+    results.push(result);
+    if (poll !== undefined) running.push({ index: results.length - 1, validate: poll });
   }
-  const valid = results.every((result) => result.status !== "invalid");
-  const lines = results.flatMap((result) => {
-    if (result.skipped !== undefined)
-      return [`- ${result.sourceDirectory}: skipped (${result.skipped})`];
-    assertCli(
-      result.report,
-      `Pipeline ${result.sourceDirectory} is missing a validation report`,
-      { hint: REPORT_HINT },
-    );
-    assertCli(
-      result.status,
-      `Pipeline ${result.sourceDirectory} is missing a validation status`,
-      { hint: REPORT_HINT },
-    );
-    const header = `- ${result.sourceDirectory}: ${result.status.replace("_", "-")}`;
-    return [header, ...formatValidationIssues(result.report)];
-  });
+  if (wait !== undefined) await waitForRunningChecks(results, running, wait, startedAt);
+  const valid = results.every(
+    (result) => result.status !== "invalid" && result.status !== "checks_running",
+  );
+  const lines = results.flatMap(formatValidationResult);
   return {
     projectId: inspection.projectId,
     results,

package/src/cli/trainings.ts

@@ -313,6 +313,16 @@ const uploadPromptFile = async (
   }
 };
 
+const assertDatasetIdOption = (
+  flag: "--dataset" | "--validation-dataset",
+  value: string | undefined,
+): void =>
+  assertCli(
+    value === undefined || /^ds\./i.test(value),
+    `${flag} takes a Dataset id (ds.…), got ${JSON.stringify(value)}.`,
+    { hint: { description: "Run toloka dataset list and pass the Dataset id (ds.…)." } },
+  );
+
 const resolveSelections = async ({
   client,
   current,
@@ -322,6 +332,8 @@ const resolveSelections = async ({
   current: Training["configuration"] | undefined;
   options: TrainingDraftOptions;
 }): Promise<ResolvedSelections> => {
+  assertDatasetIdOption("--dataset", options.datasetId);
+  assertDatasetIdOption("--validation-dataset", options.validationDatasetId);
   const trainingDatasetId = options.datasetId ?? current?.training.datasetId;
   assertCli(trainingDatasetId, "--dataset is required.", {
     hint: { description: "Pass --dataset <dataset-id>; toloka dataset list shows the ids." },

package/src/cli/ui.ts

@@ -0,0 +1,63 @@
+import type { TaskUiComponentDescription } from "../testing/types";
+import type { DescribeTaskUiComponents } from "../testing/task-ui-browser";
+import type { CliIo } from "./context";
+
+const formatList = (title: string, items: readonly string[]): string[] =>
+  items.length === 0 ? [] : [`${title}:`, ...items.map((item) => `- ${item}`)];
+
+export const formatComponentIndex = (
+  components: Readonly<Record<string, TaskUiComponentDescription>>,
+): string =>
+  [
+    ...Object.values(components).map(
+      ({ name, description, whenToUse }) => `${name}: ${description} When to use: ${whenToUse}`,
+    ),
+    "",
+    "Run `toloka ui components <name> [name...]` for the props, value shape, and example of a component.",
+  ].join("\n");
+
+export const formatComponentDetails = (
+  components: Readonly<Record<string, TaskUiComponentDescription>>,
+): string =>
+  Object.values(components)
+    .map((component) =>
+      [
+        `## ${component.name}`,
+        component.description,
+        `When to use: ${component.whenToUse}`,
+        ...formatList("Capabilities", component.capabilities),
+        ...formatList("Limitations", component.limitations),
+        ...formatList("Best practices", component.bestPractices),
+        "Props:",
+        component.propsType,
+        ...(component.valueType === undefined ? [] : ["Value:", component.valueType]),
+        ...(component.example === undefined
+          ? []
+          : ["Example:", component.example.jsxTemplate, "Example output:", component.example.output]),
+      ].join("\n"),
+    )
+    .join("\n\n");
+
+export const uiComponentsCommand = async ({
+  sourceRoot,
+  names,
+  json,
+  io,
+  describe,
+}: {
+  sourceRoot: string;
+  names: readonly string[];
+  json: boolean;
+  io: CliIo;
+  describe: DescribeTaskUiComponents;
+}): Promise<void> => {
+  const components = await describe({
+    resolveFrom: sourceRoot,
+    ...(names.length === 0 ? {} : { names: [...names] }),
+  });
+  if (json) {
+    io.stdout(JSON.stringify(components, undefined, 2));
+    return;
+  }
+  io.stdout(names.length === 0 ? formatComponentIndex(components) : formatComponentDetails(components));
+};

package/src/client.ts

@@ -652,6 +652,10 @@ const createClientImplementation = (config: ClientConfig) => {
     list: async () => (await wire.listDeliveryOrders(requestOptions)).deliveryOrders,
   };
 
+  const generationModels = {
+    list: () => wire.listGenerationModels(requestOptions),
+  };
+
   const audience = {
     specializations: {
       listPage: (params: ListAudienceSpecializationsParams = {}) =>
@@ -674,6 +678,7 @@ const createClientImplementation = (config: ClientConfig) => {
     pipelines,
     datasets,
     baseModels,
+    generationModels,
     models,
     trainings,
     runs,