@toloka-platform/sdk 0.1.0-preview.3

package/src/cli/project.ts

@@ -0,0 +1,927 @@
+import { assert } from "es-toolkit";
+import { mkdir, rm, stat, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import type {
+  PipelineGraphSnapshot,
+  PipelineGraphV1,
+  PipelineValidationReport,
+} from "../../openapi/gen/models";
+import { resolvePipelineDefinition, validatePipelineDefinition } from "../authoring";
+import type { Dataset, DatasetField, Pipeline, PotatoClient } from "../client";
+import { generatePipelineSourceTree } from "../importer";
+import { readTolokaState, TOLOKA_STATE_FILENAME, writeTolokaState } from "../state";
+import type { TolokaState } from "../state";
+import { MISSING_PROJECT_ID_MESSAGE } from "./project-context";
+import { collectGraphSecretNames } from "./secrets";
+import { loadProjectSource } from "./source";
+import type { ProjectPipelineSource, ProjectSource } from "./source";
+
+export type PipelinePlanChange = {
+  readonly kind: "node" | "connection" | "field" | "layout";
+  readonly action: "added" | "removed" | "changed";
+  readonly label: string;
+  readonly id?: string;
+};
+
+export type ProjectPlanChange = {
+  readonly kind:
+    | "pipeline"
+    | "node"
+    | "connection"
+    | "field"
+    | "layout"
+    | "dataset"
+    | "datasetField"
+    | "secret";
+  readonly action: "added" | "removed" | "changed" | "missing";
+  readonly label: string;
+  readonly pipeline?: string;
+  readonly dataset?: string;
+  readonly id?: string;
+};
+
+export type ImportProjectOptions = {
+  readonly sourceRoot: string;
+  readonly projectId: string;
+  readonly force?: boolean;
+};
+
+export type ImportProjectResult = {
+  readonly state: TolokaState;
+  readonly files: readonly string[];
+  readonly output: string;
+};
+
+export type PlanProjectOptions = {
+  readonly sourceRoot: string;
+  readonly source?: ProjectSource;
+  readonly projectId?: string;
+  readonly listAllSecrets?: boolean;
+};
+
+export type PlanProjectResult = {
+  readonly projectId: string;
+  readonly changes: readonly ProjectPlanChange[];
+  readonly missingSecrets: readonly string[];
+  readonly output: string;
+};
+
+export type ProjectStatusResult = {
+  readonly projectId: string;
+  readonly output: string;
+};
+
+export type ApplyProjectOptions = PlanProjectOptions;
+
+export type ApplyProjectResult = {
+  readonly action: "changed" | "unchanged";
+  readonly state: TolokaState;
+  readonly changes: readonly ProjectPlanChange[];
+  readonly writes: number;
+  readonly output: string;
+};
+
+export type ValidateProjectOptions = PlanProjectOptions;
+
+export type PipelineValidationResult = {
+  readonly sourceDirectory: string;
+  readonly pipelineId?: string;
+  readonly report?: PipelineValidationReport;
+  readonly skipped?: string;
+};
+
+export type ValidateProjectResult = {
+  readonly projectId: string;
+  readonly results: readonly PipelineValidationResult[];
+  readonly valid: boolean;
+  readonly output: string;
+};
+
+type DesiredDataset = {
+  readonly name: string;
+  readonly fields: ReadonlyArray<PipelineGraphV1["fields"][number]>;
+};
+
+type DatasetInspection = DesiredDataset & {
+  readonly dataset?: Dataset;
+  readonly liveFields: readonly DatasetField[];
+  readonly missingFields: ReadonlyArray<PipelineGraphV1["fields"][number]>;
+  readonly conflictingFields: ReadonlyArray<PipelineGraphV1["fields"][number]>;
+};
+
+type PipelineInspection = {
+  readonly source: ProjectPipelineSource;
+  readonly pipeline?: Pipeline;
+  readonly snapshot?: PipelineGraphSnapshot;
+  readonly graphChanges: readonly PipelinePlanChange[];
+};
+
+type ProjectInspection = {
+  readonly projectId: string;
+  readonly source: ProjectSource;
+  readonly state: TolokaState | undefined;
+  readonly datasets: readonly DatasetInspection[];
+  readonly pipelines: readonly PipelineInspection[];
+  readonly livePipelines: readonly Pipeline[];
+  readonly liveDatasets: readonly Dataset[];
+  readonly liveSecrets: readonly { readonly name: string }[];
+  readonly changes: readonly ProjectPlanChange[];
+  readonly missingSecrets: readonly string[];
+  readonly removedDatasets: readonly string[];
+  readonly removedPipelines: readonly string[];
+};
+
+const sourceDirectoryPattern = /^(?!\/)(?!.*\\)(?!.*(?:^|\/)\.\.?(?:\/|$))(?!.*\/\/).+$/u;
+
+const canonicalize = (value: unknown): unknown => {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (typeof value !== "object" || value === null) return value;
+  return Object.fromEntries(
+    Object.entries(value)
+      .toSorted(([left], [right]) => left.localeCompare(right))
+      .map(([key, entryValue]) => [key, canonicalize(entryValue)]),
+  );
+};
+
+const valueKey = (value: unknown) => JSON.stringify(canonicalize(value));
+
+const isPlainObject = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const projectAuthoredKeys = (authored: unknown, live: unknown): unknown => {
+  if (!isPlainObject(authored) || !isPlainObject(live)) return live;
+
+  return Object.fromEntries(
+    Object.keys(authored).map((key) => [key, projectAuthoredKeys(authored[key], live[key])]),
+  );
+};
+
+const matchesAuthored = (authored: unknown, live: unknown) =>
+  valueKey(authored) === valueKey(projectAuthoredKeys(authored, live));
+
+const collectAsync = async <Value>(values: AsyncIterable<Value>): Promise<Value[]> => {
+  const collected: Value[] = [];
+  for await (const value of values) collected.push(value);
+  return collected;
+};
+
+const nodeLabel = (node: PipelineGraphV1["nodes"][number]) => {
+  const about = Reflect.get(node.settings, "about");
+  if (typeof about === "object" && about !== null) {
+    const name = Reflect.get(about, "name");
+    if (typeof name === "string" && name.length > 0) return `${name} (${node.id})`;
+  }
+  return node.id;
+};
+
+const fieldLabel = (field: PipelineGraphV1["fields"][number]) => `${field.name} (${field.id})`;
+
+const connectionLabel = (
+  connection: PipelineGraphV1["connections"][number],
+  nodes: ReadonlyMap<string, PipelineGraphV1["nodes"][number]>,
+) => {
+  const fromNode = nodes.get(connection.from.node);
+  const toNode = nodes.get(connection.to.node);
+  const fromLabel = fromNode === undefined ? connection.from.node : nodeLabel(fromNode);
+  const toLabel = toNode === undefined ? connection.to.node : nodeLabel(toNode);
+  return `${fromLabel}.${connection.from.handle} -> ${toLabel}.${connection.to.handle}`;
+};
+
+const byId = <Item extends { readonly id: string }>(items: readonly Item[]) =>
+  new Map(items.map((item) => [item.id, item]));
+
+const diffKeyed = <Item>(
+  authored: ReadonlyMap<string, Item>,
+  live: ReadonlyMap<string, Item>,
+  equals: (left: Item, right: Item) => boolean,
+  toChange: (item: Item, action: "added" | "removed" | "changed") => PipelinePlanChange,
+): readonly PipelinePlanChange[] => {
+  const changes: PipelinePlanChange[] = [];
+  for (const [key, item] of [...authored].toSorted(([left], [right]) =>
+    left.localeCompare(right),
+  )) {
+    const liveItem = live.get(key);
+    if (liveItem === undefined) changes.push(toChange(item, "added"));
+    else if (!equals(item, liveItem)) changes.push(toChange(item, "changed"));
+  }
+  for (const [key, item] of [...live].toSorted(([left], [right]) => left.localeCompare(right))) {
+    if (!authored.has(key)) changes.push(toChange(item, "removed"));
+  }
+  return changes;
+};
+
+export function diffPipelineGraphs(
+  authoredGraph: PipelineGraphV1,
+  liveGraph: PipelineGraphV1,
+): readonly PipelinePlanChange[] {
+  const authoredNodes = byId(authoredGraph.nodes);
+  const liveNodes = byId(liveGraph.nodes);
+  const authoredConnections = new Map(
+    authoredGraph.connections.map((connection) => [valueKey(connection), connection]),
+  );
+  const liveConnections = new Map(
+    liveGraph.connections.map((connection) => [valueKey(connection), connection]),
+  );
+
+  return [
+    ...diffKeyed(
+      byId(authoredGraph.fields),
+      byId(liveGraph.fields),
+      (left, right) => valueKey(left) === valueKey(right),
+      (field, action) => ({
+        kind: "field",
+        action,
+        label: fieldLabel(field),
+        id: field.id,
+      }),
+    ),
+    ...diffKeyed(authoredNodes, liveNodes, matchesAuthored, (node, action) => ({
+      kind: "node",
+      action,
+      label: nodeLabel(node),
+      id: node.id,
+    })),
+    ...diffKeyed(
+      authoredConnections,
+      liveConnections,
+      () => true,
+      (connection, action) => ({
+        kind: "connection",
+        action,
+        label: connectionLabel(connection, action === "removed" ? liveNodes : authoredNodes),
+      }),
+    ),
+    ...(valueKey(authoredGraph.layout) === valueKey(liveGraph.layout)
+      ? []
+      : [{ kind: "layout" as const, action: "changed" as const, label: "pipeline layout" }]),
+  ];
+}
+
+const graphWithLiveState = (
+  authoredGraph: PipelineGraphV1,
+  liveGraph: PipelineGraphV1,
+): PipelineGraphV1 => {
+  const liveRoot = liveGraph.nodes.find((node) => node.type === "root");
+  const liveRunScope =
+    liveRoot === undefined
+      ? {}
+      : {
+          ...(liveRoot.settings.itemFilter === undefined
+            ? {}
+            : { itemFilter: liveRoot.settings.itemFilter }),
+          ...(liveRoot.settings.itemLimit === undefined
+            ? {}
+            : { itemLimit: liveRoot.settings.itemLimit }),
+        };
+  return {
+    ...authoredGraph,
+    ...(liveGraph.layout === undefined ? {} : { layout: liveGraph.layout }),
+    nodes: authoredGraph.nodes.map((node) =>
+      node.type === "root" ? { ...node, settings: { ...liveRunScope, ...node.settings } } : node,
+    ),
+  };
+};
+
+export const desiredDatasets = (source: ProjectSource): readonly DesiredDataset[] => {
+  const fieldsByDataset = new Map<string, Map<string, PipelineGraphV1["fields"][number]>>();
+  for (const pipeline of source.pipelines) {
+    let fields = fieldsByDataset.get(pipeline.definition.dataset);
+    if (fields === undefined) {
+      fields = new Map();
+      fieldsByDataset.set(pipeline.definition.dataset, fields);
+    }
+    for (const field of pipeline.definition.graph.fields) {
+      const existing = fields.get(field.name);
+      assert(
+        existing === undefined || valueKey(existing.jsonSchema) === valueKey(field.jsonSchema),
+        `Dataset "${pipeline.definition.dataset}" has conflicting schemas for field "${field.name}".`,
+      );
+      if (existing === undefined) fields.set(field.name, field);
+    }
+  }
+  return [...fieldsByDataset]
+    .toSorted(([left], [right]) => left.localeCompare(right))
+    .map(([name, fields]) => ({
+      name,
+      fields: [...fields.values()].toSorted((left, right) => left.name.localeCompare(right.name)),
+    }));
+};
+
+const validateSource = (source: ProjectSource) => {
+  const sourceDirectories = new Set<string>();
+  for (const pipeline of source.pipelines) {
+    assert(
+      sourceDirectoryPattern.test(pipeline.sourceDirectory),
+      `Invalid pipeline source directory ${pipeline.sourceDirectory}`,
+    );
+    assert(
+      !sourceDirectories.has(pipeline.sourceDirectory),
+      `Pipeline source directory ${pipeline.sourceDirectory} is declared more than once`,
+    );
+    sourceDirectories.add(pipeline.sourceDirectory);
+    validatePipelineDefinition(pipeline.definition);
+  }
+  desiredDatasets(source);
+};
+
+const liveDatasetFor = (
+  name: string,
+  binding: { readonly datasetId: string } | undefined,
+  datasets: readonly Dataset[],
+): Dataset | undefined => {
+  const bound =
+    binding === undefined
+      ? undefined
+      : datasets.find((dataset) => dataset.id === binding.datasetId);
+  if (bound !== undefined) {
+    assert(
+      bound.name === name,
+      `Dataset ${bound.id} is bound as "${name}" but is named "${bound.name}".`,
+    );
+    return bound;
+  }
+  const matching = datasets.filter((dataset) => dataset.name === name);
+  assert(matching.length <= 1, `Project has multiple datasets named "${name}".`);
+  return matching[0];
+};
+
+const livePipelineFor = (
+  source: ProjectPipelineSource,
+  binding: { readonly pipelineId: string } | undefined,
+  pipelines: readonly Pipeline[],
+): Pipeline | undefined => {
+  const bound =
+    binding === undefined
+      ? undefined
+      : pipelines.find((pipeline) => pipeline.id === binding.pipelineId);
+  if (bound !== undefined) return bound;
+  const matching = pipelines.filter((pipeline) => pipeline.name === source.name);
+  assert(matching.length <= 1, `Project has multiple pipelines named "${source.name}".`);
+  return matching[0];
+};
+
+const changeLine = (change: ProjectPlanChange) => {
+  const prefix =
+    change.action === "added"
+      ? "+"
+      : change.action === "removed"
+        ? "-"
+        : change.action === "missing"
+          ? "!"
+          : "~";
+  switch (change.kind) {
+    case "pipeline":
+      return `${prefix} Pipeline ${change.action}: ${change.label}`;
+    case "dataset":
+      return `${prefix} Dataset ${change.action}: ${change.label}`;
+    case "datasetField":
+      return `${prefix} Dataset field ${change.action}: ${change.dataset}.${change.label}`;
+    case "secret":
+      return `${prefix} Missing secret: ${change.label}`;
+    case "node":
+      return `${prefix} Node ${change.action} in ${change.pipeline}: ${change.label}`;
+    case "connection":
+      return `${prefix} Connection ${change.action} in ${change.pipeline}: ${change.label}`;
+    case "field":
+      return `${prefix} Graph field ${change.action} in ${change.pipeline}: ${change.label}`;
+    case "layout":
+      return `${prefix} Layout ${change.action} in ${change.pipeline}: ${change.label}`;
+  }
+};
+
+const planOutput = (changes: readonly ProjectPlanChange[], missingSecrets: readonly string[]) => {
+  if (changes.length === 0) return "No changes.";
+  const unit = changes.length === 1 ? "change" : "changes";
+  const lines = [`Plan: ${changes.length} ${unit}`, ...changes.map(changeLine)];
+  if (missingSecrets.length > 0) {
+    lines.push("", "Required secrets are missing:");
+    for (const name of missingSecrets) lines.push(`  ${name}`);
+    lines.push("Run `toloka secret set <name>` before apply.");
+  }
+  return lines.join("\n");
+};
+
+const inspectDatasets = async (
+  client: PotatoClient,
+  requiredDatasets: readonly DesiredDataset[],
+  bindings: TolokaState | undefined,
+  liveDatasets: readonly Dataset[],
+): Promise<readonly DatasetInspection[]> => {
+  const datasetPairs = requiredDatasets.map((desired) => ({
+    desired,
+    dataset: liveDatasetFor(desired.name, bindings?.datasets[desired.name], liveDatasets),
+  }));
+  const liveFields = await Promise.all(
+    datasetPairs.map(async ({ dataset }) =>
+      dataset === undefined
+        ? []
+        : collectAsync(client.datasets.listAllFields({ datasetId: dataset.id })),
+    ),
+  );
+  return datasetPairs.map(({ desired, dataset }, index) => {
+    const fields = liveFields[index] ?? [];
+    const byName = new Map(fields.map((field) => [field.name, field]));
+    return {
+      ...desired,
+      ...(dataset === undefined ? {} : { dataset }),
+      liveFields: fields,
+      missingFields: desired.fields.filter((field) => !byName.has(field.name)),
+      conflictingFields: desired.fields.filter((field) => {
+        const liveField = byName.get(field.name);
+        return (
+          liveField !== undefined && valueKey(liveField.jsonSchema) !== valueKey(field.jsonSchema)
+        );
+      }),
+    };
+  });
+};
+
+const inspectPipelines = async (
+  client: PotatoClient,
+  source: ProjectSource,
+  bindings: TolokaState | undefined,
+  datasets: readonly DatasetInspection[],
+  livePipelines: readonly Pipeline[],
+): Promise<readonly PipelineInspection[]> => {
+  const pipelinePairs = source.pipelines.map((pipelineSource) => ({
+    source: pipelineSource,
+    pipeline: livePipelineFor(
+      pipelineSource,
+      bindings?.pipelines[pipelineSource.sourceDirectory],
+      livePipelines,
+    ),
+  }));
+  const snapshots = await Promise.all(
+    pipelinePairs.map(async ({ pipeline }) =>
+      pipeline === undefined ? undefined : client.pipelines.getGraph({ pipelineId: pipeline.id }),
+    ),
+  );
+  return pipelinePairs.map(({ source: pipelineSource, pipeline }, index) => {
+    const snapshot = snapshots[index];
+    const dataset = datasets.find(
+      (entry) => entry.name === pipelineSource.definition.dataset,
+    )?.dataset;
+    let graphChanges: readonly PipelinePlanChange[] = [];
+    if (pipeline !== undefined && snapshot !== undefined) {
+      graphChanges =
+        dataset === undefined
+          ? [{ kind: "node", action: "changed", label: "root dataset binding", id: "root" }]
+          : diffPipelineGraphs(
+              graphWithLiveState(
+                resolvePipelineDefinition(pipelineSource.definition, { datasetId: dataset.id }),
+                snapshot.graph,
+              ),
+              snapshot.graph,
+            );
+    }
+    return {
+      source: pipelineSource,
+      ...(pipeline === undefined ? {} : { pipeline }),
+      ...(snapshot === undefined ? {} : { snapshot }),
+      graphChanges,
+    };
+  });
+};
+
+const collectChanges = (
+  datasets: readonly DatasetInspection[],
+  pipelines: readonly PipelineInspection[],
+  missingSecrets: readonly string[],
+  removedDatasets: readonly string[],
+  removedPipelines: readonly string[],
+): readonly ProjectPlanChange[] => {
+  const changes: ProjectPlanChange[] = [];
+  for (const sourceDirectory of removedPipelines) {
+    changes.push({ kind: "pipeline", action: "removed", label: sourceDirectory });
+  }
+  for (const name of removedDatasets)
+    changes.push({ kind: "dataset", action: "removed", label: name });
+  for (const dataset of datasets) {
+    if (dataset.dataset === undefined)
+      changes.push({ kind: "dataset", action: "added", label: dataset.name });
+    for (const field of dataset.missingFields) {
+      changes.push({
+        kind: "datasetField",
+        action: "added",
+        label: field.name,
+        dataset: dataset.name,
+      });
+    }
+    for (const field of dataset.conflictingFields) {
+      changes.push({
+        kind: "datasetField",
+        action: "changed",
+        label: field.name,
+        dataset: dataset.name,
+      });
+    }
+  }
+  for (const pipeline of pipelines) {
+    if (pipeline.pipeline === undefined) {
+      changes.push({ kind: "pipeline", action: "added", label: pipeline.source.sourceDirectory });
+    } else {
+      for (const graphChange of pipeline.graphChanges) {
+        changes.push({ ...graphChange, pipeline: pipeline.source.sourceDirectory });
+      }
+    }
+  }
+  for (const name of missingSecrets)
+    changes.push({ kind: "secret", action: "missing", label: name });
+  return changes;
+};
+
+const inspectProject = async (
+  client: PotatoClient,
+  options: PlanProjectOptions,
+): Promise<ProjectInspection> => {
+  const source = options.source ?? (await loadProjectSource(options.sourceRoot));
+  validateSource(source);
+  const state = await readTolokaState(options.sourceRoot);
+  const projectId = options.projectId ?? state?.projectId;
+  assert(projectId, MISSING_PROJECT_ID_MESSAGE);
+
+  const bindings = state?.projectId === projectId ? state : undefined;
+  const requiredSecrets = new Set(
+    source.pipelines.flatMap(({ definition }) => collectGraphSecretNames(definition.graph)),
+  );
+  const shouldListSecrets = options.listAllSecrets === true || requiredSecrets.size > 0;
+  const [livePipelines, liveDatasets, liveSecrets] = await Promise.all([
+    collectAsync(client.pipelines.listAll({ projectId })),
+    collectAsync(client.datasets.listAll({ projectId })),
+    shouldListSecrets ? client.secrets.list({ projectId }) : Promise.resolve([]),
+  ]);
+  const requiredDatasets = desiredDatasets(source);
+  const removedDatasets = Object.keys(bindings?.datasets ?? {})
+    .filter((name) => !requiredDatasets.some((dataset) => dataset.name === name))
+    .toSorted();
+  const removedPipelines = Object.keys(bindings?.pipelines ?? {})
+    .filter(
+      (sourceDirectory) =>
+        !source.pipelines.some((pipeline) => pipeline.sourceDirectory === sourceDirectory),
+    )
+    .toSorted();
+  const datasets = await inspectDatasets(client, requiredDatasets, bindings, liveDatasets);
+  const pipelines = await inspectPipelines(client, source, bindings, datasets, livePipelines);
+  const existingSecrets = new Set(liveSecrets.map((secret) => secret.name));
+  const missingSecrets = [...requiredSecrets]
+    .filter((name) => !existingSecrets.has(name))
+    .toSorted();
+  return {
+    projectId,
+    source,
+    state,
+    datasets,
+    pipelines,
+    livePipelines,
+    liveDatasets,
+    liveSecrets,
+    changes: collectChanges(datasets, pipelines, missingSecrets, removedDatasets, removedPipelines),
+    missingSecrets,
+    removedDatasets,
+    removedPipelines,
+  };
+};
+
+const sourceDirectoryBase = (name: string) =>
+  name
+    .toLowerCase()
+    .match(/[a-z0-9]+/gu)
+    ?.join("-") || "pipeline";
+
+const hash = (value: string) => {
+  let result = 0x811c9dc5;
+  for (let index = 0; index < value.length; index += 1) {
+    result ^= value.charCodeAt(index);
+    result = Math.imul(result, 0x01000193);
+  }
+  return (result >>> 0).toString(16).padStart(8, "0");
+};
+
+const pipelineSourceDirectories = (pipelines: readonly Pipeline[]) => {
+  const groups = Map.groupBy(pipelines, (pipeline) => sourceDirectoryBase(pipeline.name));
+  return new Map(
+    pipelines.map((pipeline) => {
+      const base = sourceDirectoryBase(pipeline.name);
+      const collision = (groups.get(base)?.length ?? 0) > 1;
+      return [pipeline.id, `pipelines/${collision ? `${base}-${hash(pipeline.id)}` : base}`];
+    }),
+  );
+};
+
+export async function importProject(
+  client: PotatoClient,
+  options: ImportProjectOptions,
+): Promise<ImportProjectResult> {
+  const pipelines = (
+    await collectAsync(client.pipelines.listAll({ projectId: options.projectId }))
+  ).toSorted(
+    (left, right) => left.name.localeCompare(right.name) || left.id.localeCompare(right.id),
+  );
+  const snapshots = await Promise.all(
+    pipelines.map((pipeline) => client.pipelines.getGraph({ pipelineId: pipeline.id })),
+  );
+  const datasetIds = new Set<string>();
+  for (const snapshot of snapshots) {
+    const rootNode = snapshot.graph.nodes.find((node) => node.type === "root");
+    assert(rootNode?.settings.datasetId, "Every imported pipeline must have a bound dataset.");
+    datasetIds.add(rootNode.settings.datasetId);
+  }
+  const projectDatasets = await collectAsync(
+    client.datasets.listAll({ projectId: options.projectId }),
+  );
+  const datasets: Dataset[] = [];
+  for (const datasetId of [...datasetIds].toSorted()) {
+    const dataset = projectDatasets.find((candidate) => candidate.id === datasetId);
+    assert(dataset, `Project ${options.projectId} has no dataset ${datasetId}.`);
+    datasets.push(dataset);
+  }
+  const datasetsById = new Map<string, Dataset>(datasets.map((dataset) => [dataset.id, dataset]));
+  const directories = pipelineSourceDirectories(pipelines);
+  const tree: Record<string, string> = {};
+  const statePipelines: Record<string, { readonly pipelineId: string }> = {};
+  const stateDatasets: Record<string, { readonly datasetId: string }> = {};
+  for (const [index, pipeline] of pipelines.entries()) {
+    const snapshot = snapshots[index];
+    assert(snapshot, `Pipeline ${pipeline.id} has no graph snapshot.`);
+    const rootNode = snapshot.graph.nodes.find((node) => node.type === "root");
+    assert(rootNode?.settings.datasetId, `Pipeline ${pipeline.id} has no dataset binding.`);
+    const dataset = datasetsById.get(rootNode.settings.datasetId);
+    assert(
+      dataset,
+      `Pipeline ${pipeline.id} references unknown dataset ${rootNode.settings.datasetId}.`,
+    );
+    const existingDatasetBinding = stateDatasets[dataset.name];
+    assert(
+      existingDatasetBinding === undefined || existingDatasetBinding.datasetId === dataset.id,
+      `Project has multiple referenced datasets named "${dataset.name}".`,
+    );
+    stateDatasets[dataset.name] = { datasetId: dataset.id };
+    const sourceDirectory = directories.get(pipeline.id);
+    assert(sourceDirectory, `Pipeline ${pipeline.id} has no source directory.`);
+    statePipelines[sourceDirectory] = { pipelineId: pipeline.id };
+    for (const [relativePath, source] of Object.entries(
+      generatePipelineSourceTree(snapshot.graph, { dataset: dataset.name }),
+    )) {
+      tree[`${sourceDirectory}/${relativePath}`] = source;
+    }
+  }
+
+  await mkdir(options.sourceRoot, { recursive: true });
+  const pipelinesRoot = join(options.sourceRoot, "pipelines");
+  let pipelinesRootExists = false;
+  try {
+    pipelinesRootExists = (await stat(pipelinesRoot)).isDirectory();
+  } catch (error) {
+    if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error;
+  }
+  assert(
+    !pipelinesRootExists || options.force === true,
+    `Folder already exists: ${pipelinesRoot}. Pass --force to replace it.`,
+  );
+  if (pipelinesRootExists) await rm(pipelinesRoot, { recursive: true, force: true });
+
+  for (const [relativePath, source] of Object.entries(tree).toSorted(([left], [right]) =>
+    left.localeCompare(right),
+  )) {
+    const destination = join(options.sourceRoot, relativePath);
+    await mkdir(dirname(destination), { recursive: true });
+    await writeFile(destination, source, "utf8");
+  }
+  const state: TolokaState = {
+    projectId: options.projectId,
+    pipelines: statePipelines,
+    datasets: stateDatasets,
+  };
+  await writeTolokaState(options.sourceRoot, state);
+  const files = [...Object.keys(tree), TOLOKA_STATE_FILENAME].toSorted();
+  const pipelineUnit = pipelines.length === 1 ? "pipeline" : "pipelines";
+  const datasetUnit = datasets.length === 1 ? "dataset" : "datasets";
+  return {
+    state,
+    files,
+    output: `Imported project ${options.projectId} with ${pipelines.length} ${pipelineUnit} and ${datasets.length} ${datasetUnit} into ${resolve(options.sourceRoot)}.`,
+  };
+}
+
+export async function planProject(
+  client: PotatoClient,
+  options: PlanProjectOptions,
+): Promise<PlanProjectResult> {
+  const inspection = await inspectProject(client, options);
+  return {
+    projectId: inspection.projectId,
+    changes: inspection.changes,
+    missingSecrets: inspection.missingSecrets,
+    output: planOutput(inspection.changes, inspection.missingSecrets),
+  };
+}
+
+export async function projectStatus(
+  client: PotatoClient,
+  options: PlanProjectOptions,
+): Promise<ProjectStatusResult> {
+  const projectId = options.projectId;
+  assert(projectId, MISSING_PROJECT_ID_MESSAGE);
+  const live = await client.projects.get({ projectId });
+  const inspection = await inspectProject(client, { ...options, listAllSecrets: true });
+  const pending = inspection.changes.length;
+  const pendingLine =
+    pending === 0
+      ? "Source has no pending changes."
+      : `Source has ${pending} pending change${pending === 1 ? "" : "s"}.`;
+  return {
+    projectId,
+    output: [
+      `Project:   ${projectId} (${live.name})`,
+      `Team:      ${live.teamId}`,
+      `Pipelines: ${inspection.livePipelines.length}`,
+      `Datasets:  ${inspection.liveDatasets.length}`,
+      `Secrets:   ${inspection.liveSecrets.length}`,
+      pendingLine,
+    ].join("\n"),
+  };
+}
+
+export async function applyProject(
+  client: PotatoClient,
+  options: ApplyProjectOptions,
+): Promise<ApplyProjectResult> {
+  const inspection = await inspectProject(client, options);
+  assert(
+    inspection.removedPipelines.length === 0,
+    `Refusing to delete pipelines absent from source: ${inspection.removedPipelines.join(", ")}.`,
+  );
+  assert(
+    inspection.removedDatasets.length === 0,
+    `Refusing to delete datasets absent from source: ${inspection.removedDatasets.join(", ")}. Dataset items are never deleted.`,
+  );
+  assert(
+    inspection.missingSecrets.length === 0,
+    `Required secrets are missing: ${inspection.missingSecrets.join(", ")}. Run toloka secret set <name> before apply.`,
+  );
+  const conflictingFields = inspection.datasets.flatMap((dataset) =>
+    dataset.conflictingFields.map((field) => `${dataset.name}.${field.name}`),
+  );
+  assert(
+    conflictingFields.length === 0,
+    `Dataset field schemas conflict with source: ${conflictingFields.join(", ")}. Existing fields are never rewritten.`,
+  );
+
+  let writes = 0;
+  const datasetIds = new Map<string, string>();
+  for (const datasetInspection of inspection.datasets) {
+    let dataset = datasetInspection.dataset;
+    if (dataset === undefined) {
+      dataset = await client.datasets.create({
+        projectId: inspection.projectId,
+        name: datasetInspection.name,
+      });
+      writes += 1;
+    }
+    datasetIds.set(datasetInspection.name, dataset.id);
+    if (datasetInspection.missingFields.length > 0) {
+      await client.datasets.createFields({
+        datasetId: dataset.id,
+        fields: datasetInspection.missingFields.map((field) => ({
+          name: field.name,
+          jsonSchema: field.jsonSchema,
+        })),
+      });
+      writes += datasetInspection.missingFields.length;
+    }
+  }
+
+  const pipelineIds = new Map<string, string>();
+  for (const pipelineInspection of inspection.pipelines) {
+    const datasetId = datasetIds.get(pipelineInspection.source.definition.dataset);
+    assert(
+      datasetId,
+      `Dataset ${pipelineInspection.source.definition.dataset} has no resolved ID.`,
+    );
+    const desiredGraph = resolvePipelineDefinition(pipelineInspection.source.definition, {
+      datasetId,
+    });
+    if (pipelineInspection.pipeline === undefined) {
+      const pipeline = await client.pipelines.import({
+        projectId: inspection.projectId,
+        name: pipelineInspection.source.name,
+        graph: desiredGraph,
+      });
+      await client.pipelines.attachDataset({ pipelineId: pipeline.id, datasetId });
+      pipelineIds.set(pipelineInspection.source.sourceDirectory, pipeline.id);
+      writes += 2;
+      continue;
+    }
+    pipelineIds.set(pipelineInspection.source.sourceDirectory, pipelineInspection.pipeline.id);
+    if (pipelineInspection.graphChanges.length === 0) continue;
+    const snapshot = pipelineInspection.snapshot;
+    assert(snapshot, `Pipeline ${pipelineInspection.pipeline.id} has no graph snapshot.`);
+    await client.pipelines.replaceGraph({
+      pipelineId: pipelineInspection.pipeline.id,
+      graph: graphWithLiveState(desiredGraph, snapshot.graph),
+      expectedVersion: snapshot.version,
+    });
+    writes += 1;
+  }
+
+  const nextState: TolokaState = {
+    projectId: inspection.projectId,
+    pipelines: Object.fromEntries(
+      inspection.source.pipelines.map((pipeline) => {
+        const pipelineId = pipelineIds.get(pipeline.sourceDirectory);
+        assert(pipelineId, `Pipeline ${pipeline.sourceDirectory} has no resolved ID.`);
+        return [pipeline.sourceDirectory, { pipelineId }];
+      }),
+    ),
+    datasets: Object.fromEntries(
+      inspection.datasets.map((dataset) => {
+        const datasetId = datasetIds.get(dataset.name);
+        assert(datasetId, `Dataset ${dataset.name} has no resolved ID.`);
+        return [dataset.name, { datasetId }];
+      }),
+    ),
+  };
+  if (valueKey(inspection.state) !== valueKey(nextState))
+    await writeTolokaState(options.sourceRoot, nextState);
+  const action = writes === 0 ? "unchanged" : "changed";
+  const output =
+    writes === 0
+      ? "No changes."
+      : `Applied ${writes} ${writes === 1 ? "write" : "writes"} to project ${inspection.projectId}.`;
+  return {
+    action,
+    state: nextState,
+    changes: inspection.changes,
+    writes,
+    output,
+  };
+}
+
+const issueLocation = (issue: { nodeId?: string; checkId?: string }) =>
+  [issue.nodeId, issue.checkId].filter((value): value is string => value !== undefined).join("/");
+
+const formatValidationIssues = (report: PipelineValidationReport) =>
+  report.issues.map((issue) => {
+    const location = issueLocation(issue);
+    return `  ${issue.severity} ${issue.code}${location.length === 0 ? "" : ` ${location}`}: ${issue.message}`;
+  });
+
+export async function validateProject(
+  client: PotatoClient,
+  options: ValidateProjectOptions,
+): Promise<ValidateProjectResult> {
+  const inspection = await inspectProject(client, options);
+  const results: PipelineValidationResult[] = [];
+  for (const pipeline of inspection.pipelines) {
+    const dataset = inspection.datasets.find(
+      (entry) => entry.name === pipeline.source.definition.dataset,
+    )?.dataset;
+    if (pipeline.pipeline === undefined) {
+      results.push({
+        sourceDirectory: pipeline.source.sourceDirectory,
+        skipped: "pipeline is not on the platform yet; run toloka apply first",
+      });
+      continue;
+    }
+    if (dataset === undefined) {
+      results.push({
+        sourceDirectory: pipeline.source.sourceDirectory,
+        pipelineId: pipeline.pipeline.id,
+        skipped: `dataset "${pipeline.source.definition.dataset}" is not on the platform yet; run toloka apply first`,
+      });
+      continue;
+    }
+    const graph =
+      pipeline.snapshot === undefined
+        ? resolvePipelineDefinition(pipeline.source.definition, { datasetId: dataset.id })
+        : graphWithLiveState(
+            resolvePipelineDefinition(pipeline.source.definition, { datasetId: dataset.id }),
+            pipeline.snapshot.graph,
+          );
+    const report = await client.pipelines.validateGraph({
+      pipelineId: pipeline.pipeline.id,
+      graph,
+    });
+    results.push({
+      sourceDirectory: pipeline.source.sourceDirectory,
+      pipelineId: pipeline.pipeline.id,
+      report,
+    });
+  }
+  const valid = results.every((result) => result.report?.valid !== false);
+  const lines = results.flatMap((result) => {
+    if (result.skipped !== undefined)
+      return [`- ${result.sourceDirectory}: skipped (${result.skipped})`];
+    assert(result.report, `Pipeline ${result.sourceDirectory} is missing a validation report`);
+    const header = result.report.valid
+      ? `- ${result.sourceDirectory}: valid`
+      : `- ${result.sourceDirectory}: invalid`;
+    return [header, ...formatValidationIssues(result.report)];
+  });
+  return {
+    projectId: inspection.projectId,
+    results,
+    valid,
+    output: lines.length === 0 ? "No pipelines to validate." : lines.join("\n"),
+  };
+}