@toloka-platform/sdk 0.1.0-preview.5 → 0.1.0-preview.7

package/src/cli/data.ts

@@ -1,7 +1,7 @@
 import { assert } from "es-toolkit";
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { basename, dirname } from "node:path";
-import type { Dataset, InferredField, PotatoClient } from "../client";
+import type { Dataset, InferredField, PotatoClient, UploadId } from "../client";
 import { PotatoApiError } from "../errors";
 import { generateFieldDeclarationsSource } from "../importer";
 import { readTolokaState } from "../state";
@@ -14,6 +14,14 @@ import type { ProjectSource } from "./source";
 const RAW_UPLOAD_FIELD_NAME = "uploaded";
 const DATASET_ID_PATTERN =
   /^ds\.[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
+const FINALIZE_UPLOAD_ATTEMPTS = 3;
+
+class DatasetUploadError extends Error {
+  constructor(message: string, cause: unknown) {
+    super(message, { cause });
+    this.name = "DatasetUploadError";
+  }
+}
 
 const canonicalDatasetId = (token: string): string | undefined =>
   DATASET_ID_PATTERN.test(token) ? token.toLowerCase() : undefined;
@@ -22,6 +30,16 @@ type UploadFieldDeclaration = NonNullable<
   Parameters<PotatoClient["datasets"]["startUpload"]>[0]["fields"]
 >[number];
 
+type DatasetUploadFormat = NonNullable<
+  Parameters<PotatoClient["datasets"]["startUpload"]>[0]["format"]
+>;
+
+const DATASET_UPLOAD_CONTENT_TYPES: Record<DatasetUploadFormat, string> = {
+  json: "application/json",
+  jsonl: "application/x-ndjson",
+  csv: "text/csv",
+};
+
 export type UploadDatasetOptions = {
   readonly sourceRoot: string;
   readonly datasetName: string;
@@ -201,29 +219,138 @@ const resolveLiveDataset = async (
   return { datasetId: created.id, name: created.name, nameIsUnique: true };
 };
 
-const readJsonLines = async (filePath: string): Promise<ReadonlyArray<Record<string, unknown>>> => {
-  const source = await readFile(filePath, "utf8");
-  return source
-    .split(/\r?\n/u)
-    .map((line, index) => ({ line, lineNumber: index + 1 }))
-    .filter(({ line }) => line.trim().length > 0)
-    .map(({ line, lineNumber }) => {
+const resolveDatasetUploadFormat = (filename: string): DatasetUploadFormat => {
+  const normalizedFilename = filename.toLowerCase();
+
+  if (normalizedFilename.endsWith(".json")) return "json";
+  if (normalizedFilename.endsWith(".jsonl")) return "jsonl";
+  if (normalizedFilename.endsWith(".csv")) return "csv";
+
+  assert(false, `${filename} must have a .json, .jsonl, or .csv extension`);
+};
+
+const readDatasetUpload = async (filePath: string) => {
+  const filename = basename(filePath);
+  const format = resolveDatasetUploadFormat(filename);
+  const payload = await readFile(filePath, "utf8");
+
+  if (format === "jsonl") {
+    for (const { line, lineNumber } of payload
+      .split(/\r?\n/u)
+      .map((line, index) => ({ line, lineNumber: index + 1 }))
+      .filter(({ line }) => line.length > 0)) {
       let value: unknown;
       try {
         value = JSON.parse(line);
       } catch {
-        assert(false, `${basename(filePath)}:${lineNumber} is not valid JSON`);
+        assert(false, `${filename}:${lineNumber} is not valid JSON`);
       }
-      assert(isJsonObject(value), `${basename(filePath)}:${lineNumber} must contain a JSON object`);
-      return value;
+      assert(isJsonObject(value), `${filename}:${lineNumber} must contain a JSON object`);
+    }
+  }
+
+  return { payload, format, contentType: DATASET_UPLOAD_CONTENT_TYPES[format] };
+};
+
+type StartedDatasetUpload = { readonly uploadUrl: string; readonly job: { readonly id: UploadId } };
+
+const cancelStartedUpload = async (
+  client: PotatoClient,
+  uploadId: StartedDatasetUpload["job"]["id"],
+  error: unknown,
+): Promise<never> => {
+  const reason = error instanceof Error ? error.message : String(error);
+
+  try {
+    await client.datasets.cancelUpload({ uploadId });
+  } catch (cancelError) {
+    const cancelReason = cancelError instanceof Error ? cancelError.message : String(cancelError);
+    throw new AggregateError(
+      [error, cancelError],
+      `Dataset upload ${uploadId} failed and could not be canceled. Upload error: ${reason}. Cancellation error: ${cancelReason}.`,
+    );
+  }
+
+  throw new DatasetUploadError(
+    `Dataset upload ${uploadId} failed and was canceled: ${reason}`,
+    error,
+  );
+};
+
+const transferAndFinalizeUpload = async ({
+  client,
+  started,
+  payload,
+  contentType,
+  fetch,
+  retryIntervalMs,
+}: {
+  client: PotatoClient;
+  started: StartedDatasetUpload;
+  payload: string;
+  contentType: string;
+  fetch: typeof globalThis.fetch;
+  retryIntervalMs: number;
+}): Promise<void> => {
+  let uploadResponse: Response;
+  try {
+    uploadResponse = await fetch(started.uploadUrl, {
+      method: "PUT",
+      headers: {
+        "Content-Type": contentType,
+        "x-ms-blob-type": "BlockBlob",
+      },
+      body: payload,
     });
+  } catch (error) {
+    return cancelStartedUpload(client, started.job.id, error);
+  }
+
+  if (!uploadResponse.ok) {
+    return cancelStartedUpload(
+      client,
+      started.job.id,
+      new DatasetUploadError(
+        `Dataset upload failed with HTTP ${uploadResponse.status}.`,
+        undefined,
+      ),
+    );
+  }
+
+  let transientFailures = 0;
+  while (true) {
+    try {
+      await client.datasets.finalizeUpload({ uploadId: started.job.id });
+      return;
+    } catch (error) {
+      if (!(error instanceof PotatoApiError && error.status === 503)) {
+        return cancelStartedUpload(client, started.job.id, error);
+      }
+
+      transientFailures += 1;
+      if (transientFailures === FINALIZE_UPLOAD_ATTEMPTS) {
+        return cancelStartedUpload(
+          client,
+          started.job.id,
+          new DatasetUploadError(
+            `Storage remained unavailable after ${FINALIZE_UPLOAD_ATTEMPTS} finalization attempts.`,
+            error,
+          ),
+        );
+      }
+
+      const { promise, resolve } = Promise.withResolvers<void>();
+      setTimeout(resolve, retryIntervalMs);
+      await promise;
+    }
+  }
 };
 
 export async function uploadDataset(
   client: PotatoClient,
   options: UploadDatasetOptions,
 ): Promise<UploadDatasetResult> {
-  const rows = await readJsonLines(options.filePath);
+  const { payload, format, contentType } = await readDatasetUpload(options.filePath);
   const resolved = await resolveLiveDataset(client, options, true);
   const datasetId = resolved.datasetId;
 
@@ -236,25 +363,21 @@ export async function uploadDataset(
     if (!fieldsByName.has(field.name)) fieldsByName.set(field.name, field);
   }
   const fields = [...fieldsByName.values()];
-  const payload = JSON.stringify(rows);
-  const payloadBytes = new TextEncoder().encode(payload);
   const started = await client.datasets.startUpload({
     datasetId,
     filename: basename(options.filePath),
-    format: "json",
-    declaredSizeBytes: payloadBytes.byteLength,
+    format,
+    declaredSizeBytes: new TextEncoder().encode(payload).byteLength,
     fields,
   });
-  const uploadResponse = await (options.fetch ?? globalThis.fetch)(started.uploadUrl, {
-    method: "PUT",
-    headers: {
-      "Content-Type": "application/json",
-      "x-ms-blob-type": "BlockBlob",
-    },
-    body: payloadBytes,
+  await transferAndFinalizeUpload({
+    client,
+    started,
+    payload,
+    contentType,
+    fetch: options.fetch ?? globalThis.fetch,
+    retryIntervalMs: options.waitFor?.intervalMs ?? 1000,
   });
-  assert(uploadResponse.ok, `Dataset upload failed with HTTP ${uploadResponse.status}.`);
-  await client.datasets.finalizeUpload({ uploadId: started.job.id });
   const completed = await client.waitFor(
     { type: "datasetUpload", uploadId: started.job.id },
     options.waitFor,

package/src/cli/models.ts

@@ -0,0 +1,188 @@
+import { assert } from "es-toolkit";
+
+import type { BaseModel, BaseModelId, Model, ModelVersion, PotatoClient } from "../client";
+
+type ModelApi = Pick<PotatoClient, "baseModels" | "models">;
+type ModelIo = { readonly stdout: (text: string) => void };
+type TableRow = readonly string[];
+
+const BASE_MODEL_IDS = [
+  "qwen3-4b",
+  "qwen3-8b",
+  "qwen3-32b",
+  "qwen3-235b",
+] satisfies readonly BaseModelId[];
+
+const formatTable = (rows: readonly TableRow[]): string => {
+  const widths = rows.reduce<number[]>(
+    (current, row) => row.map((cell, index) => Math.max(current[index] ?? 0, cell.length)),
+    [],
+  );
+  return rows
+    .map((row) =>
+      row
+        .map((cell, index) => cell.padEnd(widths[index] ?? cell.length))
+        .join("  ")
+        .trimEnd(),
+    )
+    .join("\n");
+};
+
+const writeOutput = (io: ModelIo, json: boolean, value: unknown, formatted: string): void => {
+  io.stdout(json ? JSON.stringify(value, null, 2) : formatted);
+};
+
+const formatBaseModel = (model: BaseModel): string =>
+  formatTable([
+    ["ID", model.id],
+    ["Name", model.displayName],
+    ["Provider", model.provider],
+    ["Upstream model", model.upstreamModel],
+    ["Upstream revision", model.upstreamRevision],
+  ]);
+
+const formatModel = (model: Model): string =>
+  formatTable([
+    ["ID", model.id],
+    ["Project", model.projectId],
+    ["Name", model.name],
+    ["Base model", model.baseModelId],
+    ["Description", model.description ?? "-"],
+    ["Archived at", model.archivedAt ?? "-"],
+    ["Created at", model.createdAt],
+    ["Updated at", model.updatedAt],
+  ]);
+
+export const parseBaseModelId = (value: string): BaseModelId => {
+  const modelId = BASE_MODEL_IDS.find((candidate) => candidate === value);
+  assert(modelId, `Unknown base model ${value}.`);
+  return modelId;
+};
+
+const resolveModel = async (
+  client: ModelApi,
+  projectId: string,
+  nameOrId: string,
+): Promise<Model> => {
+  if (nameOrId.startsWith("mdl.")) return client.models.get({ projectId, modelId: nameOrId });
+  const matches: Model[] = [];
+  for await (const model of client.models.listAll({ projectId, limit: 100 })) {
+    if (model.name === nameOrId) matches.push(model);
+  }
+  assert(matches.length === 1, `Model ${nameOrId} was not found or is ambiguous.`);
+  return matches[0]!;
+};
+
+export const listBaseModels = async (
+  client: ModelApi,
+  input: { json: boolean },
+  io: ModelIo,
+): Promise<void> => {
+  const models = await client.baseModels.list();
+  writeOutput(
+    io,
+    input.json,
+    models,
+    models.length === 0
+      ? "No base models found."
+      : formatTable([
+          ["ID", "NAME", "PROVIDER", "UPSTREAM MODEL"],
+          ...models.map((model) => [
+            model.id,
+            model.displayName,
+            model.provider,
+            model.upstreamModel,
+          ]),
+        ]),
+  );
+};
+
+export const getBaseModel = async (
+  client: ModelApi,
+  input: { id: BaseModelId; json: boolean },
+  io: ModelIo,
+): Promise<void> => {
+  const model = await client.baseModels.get({ id: input.id });
+  writeOutput(io, input.json, model, formatBaseModel(model));
+};
+
+export const listModels = async (
+  client: ModelApi,
+  input: { projectId: string; json: boolean },
+  io: ModelIo,
+): Promise<void> => {
+  const page = await client.models.listPage({ projectId: input.projectId, limit: 100, offset: 0 });
+  writeOutput(
+    io,
+    input.json,
+    page.models,
+    page.models.length === 0
+      ? "No models found."
+      : formatTable([
+          ["ID", "NAME", "BASE MODEL"],
+          ...page.models.map((model) => [model.id, model.name, model.baseModelId]),
+        ]),
+  );
+};
+
+export const getModel = async (
+  client: ModelApi,
+  input: { projectId: string; nameOrId: string; json: boolean },
+  io: ModelIo,
+): Promise<void> => {
+  const model = await resolveModel(client, input.projectId, input.nameOrId);
+  writeOutput(io, input.json, model, formatModel(model));
+};
+
+export const listModelVersions = async (
+  client: ModelApi,
+  input: { projectId: string; nameOrId: string; json: boolean },
+  io: ModelIo,
+): Promise<void> => {
+  const model = await resolveModel(client, input.projectId, input.nameOrId);
+  const versions: ModelVersion[] = [];
+  for await (const version of client.models.listAllVersions({
+    projectId: input.projectId,
+    modelId: model.id,
+    limit: 100,
+  })) {
+    versions.push(version);
+  }
+  writeOutput(
+    io,
+    input.json,
+    versions,
+    versions.length === 0
+      ? "No Model Versions found."
+      : formatTable([
+          ["VERSION", "ID", "CREATED AT"],
+          ...versions.map((version) => [version.version, version.id, version.createdAt]),
+        ]),
+  );
+};
+
+export const createModel = async (
+  client: ModelApi,
+  input: {
+    projectId: string;
+    name: string;
+    description?: string;
+    baseModelId: BaseModelId;
+    json: boolean;
+  },
+  io: ModelIo,
+): Promise<void> => {
+  const { json, ...body } = input;
+  const model = await client.models.create(body);
+  writeOutput(io, json, model, formatModel(model));
+};
+
+export const archiveModel = async (
+  client: ModelApi,
+  input: { projectId: string; nameOrId: string; json: boolean },
+  io: ModelIo,
+): Promise<void> => {
+  const model = await resolveModel(client, input.projectId, input.nameOrId);
+  const archived = await client.models.archive({ projectId: input.projectId, modelId: model.id });
+  writeOutput(io, input.json, archived, `Archived Model ${archived.id}.`);
+};