@toloka-platform/sdk 0.1.0-preview.5 → 0.1.0-preview.6

package/openapi/gen/models/baseModel.ts

@@ -0,0 +1,9 @@
+import type { BaseModelIdProperty } from './baseModelIdProperty';
+
+export interface BaseModel {
+  id: BaseModelIdProperty;
+  displayName: string;
+  provider: string;
+  upstreamModel: string;
+  upstreamRevision: string;
+}

package/openapi/gen/models/baseModelId.ts

@@ -0,0 +1,2 @@
+
+export type BaseModelId = 'qwen3-4b' | 'qwen3-8b' | 'qwen3-32b' | 'qwen3-235b';

package/openapi/gen/models/baseModelIdProperty.ts

@@ -0,0 +1,2 @@
+
+export type BaseModelIdProperty = 'qwen3-4b' | 'qwen3-8b' | 'qwen3-32b' | 'qwen3-235b';

package/openapi/gen/models/createProjectModelBody.ts

@@ -0,0 +1,8 @@
+import type { CreateProjectModelBodyDescription } from './createProjectModelBodyDescription';
+import type { BaseModelId } from './baseModelId';
+
+export type CreateProjectModelBody = {
+  name: string;
+  description?: CreateProjectModelBodyDescription;
+  baseModelId: BaseModelId;
+};

package/openapi/gen/models/createProjectModelBodyDescription.ts

@@ -0,0 +1,2 @@
+
+export type CreateProjectModelBodyDescription = string | null;

package/openapi/gen/models/generationNodeSettingsAudienceAndPricingOneOfOneone.ts

@@ -6,4 +6,5 @@ export type GenerationNodeSettingsAudienceAndPricingOneOfOneone = {
   generationModelId?: GenerationNodeSettingsAudienceAndPricingOneOfOneoneGenerationModelId;
   reasoningEffort?: GenerationNodeSettingsAudienceAndPricingOneOfOneoneReasoningEffort;
   llmInstruction?: string;
+  maxCostPerAttempt: number;
 };

package/openapi/gen/models/index.ts

@@ -19,6 +19,9 @@ export * from './attachPipelineDatasetBodyItemFilterFieldValuesItemSourcesItem';
 export * from './attachPipelineDatasetBodyItemFilterFieldValuesItemSourcesItemOneOf';
 export * from './attachPipelineDatasetBodyItemFilterFieldValuesItemSourcesItemOneOfTwo';
 export * from './attachPipelineDatasetBodyItemFilterFieldValuesItemValue';
+export * from './baseModel';
+export * from './baseModelId';
+export * from './baseModelIdProperty';
 export * from './checkJob';
 export * from './checkJobError';
 export * from './checkJobResult';
@@ -96,6 +99,8 @@ export * from './createProjectBody';
 export * from './createProjectBodyDeliveryOrderId';
 export * from './createProjectFileUpload200';
 export * from './createProjectFileUploadBody';
+export * from './createProjectModelBody';
+export * from './createProjectModelBodyDescription';
 export * from './createdDatasetFields';
 export * from './currentUser';
 export * from './currentUserEmail';
@@ -249,10 +254,13 @@ export * from './jsonSchema';
 export * from './jsonSchemaAdditionalProperties';
 export * from './jsonSchemaType';
 export * from './jsonSchemaTypeAnyOfItem';
+export * from './listBaseModels200';
 export * from './listDatasetFields200';
 export * from './listDatasetFieldsParams';
 export * from './listDatasetItems200';
 export * from './listDatasetItemsParams';
+export * from './listModelVersions200';
+export * from './listModelVersionsParams';
 export * from './listPipelineChecks200';
 export * from './listPipelineRunItems200';
 export * from './listPipelineRunItemsParams';
@@ -261,6 +269,8 @@ export * from './listPipelineRunsParams';
 export * from './listProjectDatasets200';
 export * from './listProjectDatasetsParams';
 export * from './listProjectFiles200';
+export * from './listProjectModels200';
+export * from './listProjectModelsParams';
 export * from './listProjectPipelines200';
 export * from './listProjectPipelinesParams';
 export * from './listProjectSecrets200';
@@ -277,6 +287,12 @@ export * from './matchNodeSettingsConditionsItemPredicatesItemOperator';
 export * from './matchNodeSettingsConditionsItemPredicatesItemValue';
 export * from './matchNodeSettingsFieldProxies';
 export * from './matchNodeSettingsFieldProxiesInputItem';
+export * from './model';
+export * from './modelArchivedAt';
+export * from './modelDescription';
+export * from './modelId';
+export * from './modelVersion';
+export * from './modelVersionId';
 export * from './pagination';
 export * from './patchPipelineNodeBody';
 export * from './patchPipelineNodeBodySettings';

package/openapi/gen/models/listBaseModels200.ts

@@ -0,0 +1,5 @@
+import type { BaseModel } from './baseModel';
+
+export type ListBaseModels200 = {
+  baseModels: BaseModel[];
+};

package/openapi/gen/models/listModelVersions200.ts

@@ -0,0 +1,7 @@
+import type { ModelVersion } from './modelVersion';
+import type { Pagination } from './pagination';
+
+export type ListModelVersions200 = {
+  versions: ModelVersion[];
+  pagination: Pagination;
+};

package/openapi/gen/models/listModelVersionsParams.ts

@@ -0,0 +1,5 @@
+
+export type ListModelVersionsParams = {
+limit?: number;
+offset?: number;
+};

package/openapi/gen/models/listProjectModels200.ts

@@ -0,0 +1,7 @@
+import type { Model } from './model';
+import type { Pagination } from './pagination';
+
+export type ListProjectModels200 = {
+  models: Model[];
+  pagination: Pagination;
+};

package/openapi/gen/models/listProjectModelsParams.ts

@@ -0,0 +1,5 @@
+
+export type ListProjectModelsParams = {
+limit?: number;
+offset?: number;
+};

package/openapi/gen/models/model.ts

@@ -0,0 +1,16 @@
+import type { ModelId } from './modelId';
+import type { ProjectId } from './projectId';
+import type { ModelDescription } from './modelDescription';
+import type { BaseModelId } from './baseModelId';
+import type { ModelArchivedAt } from './modelArchivedAt';
+
+export interface Model {
+  id: ModelId;
+  projectId: ProjectId;
+  name: string;
+  description: ModelDescription;
+  baseModelId: BaseModelId;
+  archivedAt: ModelArchivedAt;
+  createdAt: string;
+  updatedAt: string;
+}

package/openapi/gen/models/modelArchivedAt.ts

@@ -0,0 +1,2 @@
+
+export type ModelArchivedAt = string | null;

package/openapi/gen/models/modelDescription.ts

@@ -0,0 +1,2 @@
+
+export type ModelDescription = string | null;

package/openapi/gen/models/modelId.ts

@@ -0,0 +1,2 @@
+
+export type ModelId = string;

package/openapi/gen/models/modelVersion.ts

@@ -0,0 +1,9 @@
+import type { ModelVersionId } from './modelVersionId';
+import type { ModelId } from './modelId';
+
+export interface ModelVersion {
+  id: ModelVersionId;
+  modelId: ModelId;
+  version: string;
+  createdAt: string;
+}

package/openapi/gen/models/modelVersionId.ts

@@ -0,0 +1,2 @@
+
+export type ModelVersionId = string;

package/openapi/gen/models/pipelineGraphSnapshotGraphNodesItemOneOfThreeoneSettingsAudienceAndPricingOneOfOneone.ts

@@ -6,4 +6,5 @@ export type PipelineGraphSnapshotGraphNodesItemOneOfThreeoneSettingsAudienceAndP
   generationModelId?: PipelineGraphSnapshotGraphNodesItemOneOfThreeoneSettingsAudienceAndPricingOneOfOneoneGenerationModelId;
   reasoningEffort?: PipelineGraphSnapshotGraphNodesItemOneOfThreeoneSettingsAudienceAndPricingOneOfOneoneReasoningEffort;
   llmInstruction?: string;
+  maxCostPerAttempt: number;
 };

package/openapi/gen/models/validatePipelineGraphBodyGraphNodesItemOneOfThreeoneSettingsAudienceAndPricingOneOfOneone.ts

@@ -6,4 +6,5 @@ export type ValidatePipelineGraphBodyGraphNodesItemOneOfThreeoneSettingsAudience
   generationModelId?: ValidatePipelineGraphBodyGraphNodesItemOneOfThreeoneSettingsAudienceAndPricingOneOfOneoneGenerationModelId;
   reasoningEffort?: ValidatePipelineGraphBodyGraphNodesItemOneOfThreeoneSettingsAudienceAndPricingOneOfOneoneReasoningEffort;
   llmInstruction?: string;
+  maxCostPerAttempt: number;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toloka-platform/sdk",
-  "version": "0.1.0-preview.5",
+  "version": "0.1.0-preview.6",
   "private": false,
   "description": "Preview. Build, inspect, and operate Toloka pipeline projects as TypeScript source. The public API may change.",
   "license": "Apache-2.0",

package/skills/catalog.json

@@ -5,8 +5,8 @@
       "name": "Build Toloka pipelines as code",
       "description": "Build, inspect, and change Toloka platform pipeline projects as TypeScript source with @toloka-platform/sdk and toloka. Use when the user wants to create, adopt, inspect, plan, or apply a pipeline project as code.",
       "tags": ["pipelines", "authoring", "sdk"],
-      "version": "0.1.0-preview.5",
-      "digest": "135d0b6ff6aa7bf26782bc26e6adbe7ee70c9db1565722ee356618211543d0bc",
+      "version": "0.1.0-preview.6",
+      "digest": "d45d1124d27e19ee43c260ddf0c259ccdc343c5d641c2aeb538d6fedaeb99183",
       "files": ["SKILL.md", "references/node-semantics.md", "references/workflow-and-cli.md"]
     },
     {
@@ -14,8 +14,8 @@
       "name": "Report a Toloka problem",
       "description": "Report a problem with the Toloka platform, CLI, or SDK as a redacted Markdown issue report sent through the toloka CLI only after explicit approval. Use when the user hits unexpected behavior, an error, or a bug and wants Toloka engineers to investigate.",
       "tags": ["support", "issue", "bug", "report"],
-      "version": "0.1.0-preview.5",
-      "digest": "9a673337ad1cbbcec7c2fadb2d49d38e923c797d049818fdaa15ff30ffd153f3",
+      "version": "0.1.0-preview.6",
+      "digest": "b71d06333786826edd51b909dfe8a20633870f235d03df4aae83d20c26e99205",
       "files": ["SKILL.md", "references/report-template.md"]
     }
   ]

package/skills/toloka-build/SKILL.md

@@ -66,7 +66,7 @@ A successful `toloka validate` is necessary, not proof the pipeline can be launc
 
 ## Keep data and credentials out of source
 
-- Never put dataset items in pipeline source or `toloka.json`. After apply has created or resolved the dataset, upload JSONL objects with `toloka dataset upload <dataset> <file>`. If a later upload fails as schema-incompatible, run `toloka dataset fields <dataset>` and copy the printed `field()` declarations into source.
+- Never put dataset items in pipeline source or `toloka.json`. After apply has created or resolved the dataset, upload `.json`, `.jsonl`, or `.csv` rows with `toloka dataset upload <dataset> <file>`. If a later upload fails as schema-incompatible, run `toloka dataset fields <dataset>` and copy the printed `field()` declarations into source.
 - Put secret references in node settings, but never secret values. Set values through stdin with `toloka secret set <name>`, or use `toloka secret set <name> --from-env <variable>`.
 
 ## Credentials

package/skills/toloka-build/references/workflow-and-cli.md

@@ -109,6 +109,13 @@ toloka pipeline runs [name] [--limit <n>]
 toloka pipeline cost [name] [--item-limit <n>]
 toloka pipeline results <runId> [--limit <n>] [--failed]
 toloka pipeline cancel <runId>
+toloka base-model list [--json]
+toloka base-model get <id> [--json]
+toloka model list [--json]
+toloka model get <name-or-id> [--json]
+toloka model versions <name-or-id> [--json]
+toloka model create <name> --base <base-model-id> [--description <text>] [--json]
+toloka model archive <name-or-id> [--json]
 toloka skill list [--json]
 toloka skill search <query> [--json]
 toloka skill install <skill-id> [--force]
@@ -121,6 +128,10 @@ Select the platform project with `toloka project add` or `toloka project create`
 
 `toloka pipeline cost [name] [--item-limit <n>]` forecasts what a run of a live pipeline would bill. It does not start a run and does not change the graph. Name the live pipeline or id; a TTY may pick among several. The typed client equivalent is `client.pipelines.estimateCost({ pipelineId, itemLimit? })`.
 
+## Manage model checkpoints
+
+The global Base Model Catalog is read-only and uses `toloka base-model`. `base-model list` prints aligned ID, name, provider, and upstream-model columns by default. `toloka model` manages project-owned lineages; creation requires one catalog `baseModelId` and creates metadata without a Model Version. Every Base Model and Model command accepts `--json` for machine-readable output. Model Version upload, validation, and download commands are deferred until the separate upload-processing contract is implemented.
+
 ## Adopt an existing live project
 
 1. Run `toloka init` in the intended source root if the agent entry points are not installed yet.
@@ -186,7 +197,7 @@ Declare the fields in source, then upload rows:
 toloka dataset upload <dataset> <file>
 ```
 
-The input file is JSONL: each non-empty line must parse as one JSON object. The dataset does not have to exist yet: when a pipeline in source binds it, upload creates it and declares that pipeline's columns, so rows land in real columns rather than in the raw `uploaded` blob. Upload refuses when no pipeline in source declares a name because it would have no column names to map onto. An id-shaped argument is never created; upload fails when that dataset id does not exist in the project. Upload appends items; apply never uploads, replaces, or deletes rows. Duplicate live names must be addressed by id.
+The CLI infers the input format from its extension and uploads the original bytes without conversion: `.json` is an array of row objects, `.jsonl` has one object per non-empty line, and `.csv` uses its header row as field names with every value ingested as a string. JSONL is validated locally before upload. If transfer or finalization fails, the CLI cancels the started upload before returning the error, so another upload can start immediately and may reuse the same filename. The dataset does not have to exist yet: when a pipeline in source binds it, upload creates it and declares that pipeline's columns, so rows land in real columns rather than in the raw `uploaded` blob. Upload refuses when no pipeline in source declares a name because it would have no column names to map onto. An id-shaped argument is never created; upload fails when that dataset id does not exist in the project. Upload appends items; applying source never deletes them.
 
 If a later upload fails with a schema-incompatibility error, inspect the inferred schema from the first successful upload:
 

package/skills/toloka-report-issue/references/report-template.md

@@ -26,7 +26,7 @@ The CLI exits 1 with `Potato API request failed with status 500`. The same comma
 
 ## Environment
 
-- toloka CLI 0.1.0-preview.5 (`toloka --version`), Bun 1.4.0, macOS 15
+- toloka CLI 0.1.0-preview.6 (`toloka --version`), Bun 1.4.0, macOS 15
 - Platform: https://platform.toloka.ai
 
 ## Redactions

package/src/cli/data.ts

@@ -1,7 +1,7 @@
 import { assert } from "es-toolkit";
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { basename, dirname } from "node:path";
-import type { Dataset, InferredField, PotatoClient } from "../client";
+import type { Dataset, InferredField, PotatoClient, UploadId } from "../client";
 import { PotatoApiError } from "../errors";
 import { generateFieldDeclarationsSource } from "../importer";
 import { readTolokaState } from "../state";
@@ -14,6 +14,14 @@ import type { ProjectSource } from "./source";
 const RAW_UPLOAD_FIELD_NAME = "uploaded";
 const DATASET_ID_PATTERN =
   /^ds\.[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
+const FINALIZE_UPLOAD_ATTEMPTS = 3;
+
+class DatasetUploadError extends Error {
+  constructor(message: string, cause: unknown) {
+    super(message, { cause });
+    this.name = "DatasetUploadError";
+  }
+}
 
 const canonicalDatasetId = (token: string): string | undefined =>
   DATASET_ID_PATTERN.test(token) ? token.toLowerCase() : undefined;
@@ -22,6 +30,16 @@ type UploadFieldDeclaration = NonNullable<
   Parameters<PotatoClient["datasets"]["startUpload"]>[0]["fields"]
 >[number];
 
+type DatasetUploadFormat = NonNullable<
+  Parameters<PotatoClient["datasets"]["startUpload"]>[0]["format"]
+>;
+
+const DATASET_UPLOAD_CONTENT_TYPES: Record<DatasetUploadFormat, string> = {
+  json: "application/json",
+  jsonl: "application/x-ndjson",
+  csv: "text/csv",
+};
+
 export type UploadDatasetOptions = {
   readonly sourceRoot: string;
   readonly datasetName: string;
@@ -201,29 +219,138 @@ const resolveLiveDataset = async (
   return { datasetId: created.id, name: created.name, nameIsUnique: true };
 };
 
-const readJsonLines = async (filePath: string): Promise<ReadonlyArray<Record<string, unknown>>> => {
-  const source = await readFile(filePath, "utf8");
-  return source
-    .split(/\r?\n/u)
-    .map((line, index) => ({ line, lineNumber: index + 1 }))
-    .filter(({ line }) => line.trim().length > 0)
-    .map(({ line, lineNumber }) => {
+const resolveDatasetUploadFormat = (filename: string): DatasetUploadFormat => {
+  const normalizedFilename = filename.toLowerCase();
+
+  if (normalizedFilename.endsWith(".json")) return "json";
+  if (normalizedFilename.endsWith(".jsonl")) return "jsonl";
+  if (normalizedFilename.endsWith(".csv")) return "csv";
+
+  assert(false, `${filename} must have a .json, .jsonl, or .csv extension`);
+};
+
+const readDatasetUpload = async (filePath: string) => {
+  const filename = basename(filePath);
+  const format = resolveDatasetUploadFormat(filename);
+  const payload = await readFile(filePath, "utf8");
+
+  if (format === "jsonl") {
+    for (const { line, lineNumber } of payload
+      .split(/\r?\n/u)
+      .map((line, index) => ({ line, lineNumber: index + 1 }))
+      .filter(({ line }) => line.length > 0)) {
       let value: unknown;
       try {
         value = JSON.parse(line);
       } catch {
-        assert(false, `${basename(filePath)}:${lineNumber} is not valid JSON`);
+        assert(false, `${filename}:${lineNumber} is not valid JSON`);
       }
-      assert(isJsonObject(value), `${basename(filePath)}:${lineNumber} must contain a JSON object`);
-      return value;
+      assert(isJsonObject(value), `${filename}:${lineNumber} must contain a JSON object`);
+    }
+  }
+
+  return { payload, format, contentType: DATASET_UPLOAD_CONTENT_TYPES[format] };
+};
+
+type StartedDatasetUpload = { readonly uploadUrl: string; readonly job: { readonly id: UploadId } };
+
+const cancelStartedUpload = async (
+  client: PotatoClient,
+  uploadId: StartedDatasetUpload["job"]["id"],
+  error: unknown,
+): Promise<never> => {
+  const reason = error instanceof Error ? error.message : String(error);
+
+  try {
+    await client.datasets.cancelUpload({ uploadId });
+  } catch (cancelError) {
+    const cancelReason = cancelError instanceof Error ? cancelError.message : String(cancelError);
+    throw new AggregateError(
+      [error, cancelError],
+      `Dataset upload ${uploadId} failed and could not be canceled. Upload error: ${reason}. Cancellation error: ${cancelReason}.`,
+    );
+  }
+
+  throw new DatasetUploadError(
+    `Dataset upload ${uploadId} failed and was canceled: ${reason}`,
+    error,
+  );
+};
+
+const transferAndFinalizeUpload = async ({
+  client,
+  started,
+  payload,
+  contentType,
+  fetch,
+  retryIntervalMs,
+}: {
+  client: PotatoClient;
+  started: StartedDatasetUpload;
+  payload: string;
+  contentType: string;
+  fetch: typeof globalThis.fetch;
+  retryIntervalMs: number;
+}): Promise<void> => {
+  let uploadResponse: Response;
+  try {
+    uploadResponse = await fetch(started.uploadUrl, {
+      method: "PUT",
+      headers: {
+        "Content-Type": contentType,
+        "x-ms-blob-type": "BlockBlob",
+      },
+      body: payload,
     });
+  } catch (error) {
+    return cancelStartedUpload(client, started.job.id, error);
+  }
+
+  if (!uploadResponse.ok) {
+    return cancelStartedUpload(
+      client,
+      started.job.id,
+      new DatasetUploadError(
+        `Dataset upload failed with HTTP ${uploadResponse.status}.`,
+        undefined,
+      ),
+    );
+  }
+
+  let transientFailures = 0;
+  while (true) {
+    try {
+      await client.datasets.finalizeUpload({ uploadId: started.job.id });
+      return;
+    } catch (error) {
+      if (!(error instanceof PotatoApiError && error.status === 503)) {
+        return cancelStartedUpload(client, started.job.id, error);
+      }
+
+      transientFailures += 1;
+      if (transientFailures === FINALIZE_UPLOAD_ATTEMPTS) {
+        return cancelStartedUpload(
+          client,
+          started.job.id,
+          new DatasetUploadError(
+            `Storage remained unavailable after ${FINALIZE_UPLOAD_ATTEMPTS} finalization attempts.`,
+            error,
+          ),
+        );
+      }
+
+      const { promise, resolve } = Promise.withResolvers<void>();
+      setTimeout(resolve, retryIntervalMs);
+      await promise;
+    }
+  }
 };
 
 export async function uploadDataset(
   client: PotatoClient,
   options: UploadDatasetOptions,
 ): Promise<UploadDatasetResult> {
-  const rows = await readJsonLines(options.filePath);
+  const { payload, format, contentType } = await readDatasetUpload(options.filePath);
   const resolved = await resolveLiveDataset(client, options, true);
   const datasetId = resolved.datasetId;
 
@@ -236,25 +363,21 @@ export async function uploadDataset(
     if (!fieldsByName.has(field.name)) fieldsByName.set(field.name, field);
   }
   const fields = [...fieldsByName.values()];
-  const payload = JSON.stringify(rows);
-  const payloadBytes = new TextEncoder().encode(payload);
   const started = await client.datasets.startUpload({
     datasetId,
     filename: basename(options.filePath),
-    format: "json",
-    declaredSizeBytes: payloadBytes.byteLength,
+    format,
+    declaredSizeBytes: new TextEncoder().encode(payload).byteLength,
     fields,
   });
-  const uploadResponse = await (options.fetch ?? globalThis.fetch)(started.uploadUrl, {
-    method: "PUT",
-    headers: {
-      "Content-Type": "application/json",
-      "x-ms-blob-type": "BlockBlob",
-    },
-    body: payloadBytes,
+  await transferAndFinalizeUpload({
+    client,
+    started,
+    payload,
+    contentType,
+    fetch: options.fetch ?? globalThis.fetch,
+    retryIntervalMs: options.waitFor?.intervalMs ?? 1000,
   });
-  assert(uploadResponse.ok, `Dataset upload failed with HTTP ${uploadResponse.status}.`);
-  await client.datasets.finalizeUpload({ uploadId: started.job.id });
   const completed = await client.waitFor(
     { type: "datasetUpload", uploadId: started.job.id },
     options.waitFor,