@toloka-platform/sdk 0.1.0-preview.4 → 0.1.0-preview.6

package/src/cli/data.ts

@@ -1,7 +1,7 @@
 import { assert } from "es-toolkit";
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { basename, dirname } from "node:path";
-import type { Dataset, InferredField, PotatoClient } from "../client";
+import type { Dataset, InferredField, PotatoClient, UploadId } from "../client";
 import { PotatoApiError } from "../errors";
 import { generateFieldDeclarationsSource } from "../importer";
 import { readTolokaState } from "../state";
@@ -14,6 +14,14 @@ import type { ProjectSource } from "./source";
 const RAW_UPLOAD_FIELD_NAME = "uploaded";
 const DATASET_ID_PATTERN =
   /^ds\.[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
+const FINALIZE_UPLOAD_ATTEMPTS = 3;
+
+class DatasetUploadError extends Error {
+  constructor(message: string, cause: unknown) {
+    super(message, { cause });
+    this.name = "DatasetUploadError";
+  }
+}
 
 const canonicalDatasetId = (token: string): string | undefined =>
   DATASET_ID_PATTERN.test(token) ? token.toLowerCase() : undefined;
@@ -22,6 +30,16 @@ type UploadFieldDeclaration = NonNullable<
   Parameters<PotatoClient["datasets"]["startUpload"]>[0]["fields"]
 >[number];
 
+type DatasetUploadFormat = NonNullable<
+  Parameters<PotatoClient["datasets"]["startUpload"]>[0]["format"]
+>;
+
+const DATASET_UPLOAD_CONTENT_TYPES: Record<DatasetUploadFormat, string> = {
+  json: "application/json",
+  jsonl: "application/x-ndjson",
+  csv: "text/csv",
+};
+
 export type UploadDatasetOptions = {
   readonly sourceRoot: string;
   readonly datasetName: string;
@@ -201,29 +219,138 @@ const resolveLiveDataset = async (
   return { datasetId: created.id, name: created.name, nameIsUnique: true };
 };
 
-const readJsonLines = async (filePath: string): Promise<ReadonlyArray<Record<string, unknown>>> => {
-  const source = await readFile(filePath, "utf8");
-  return source
-    .split(/\r?\n/u)
-    .map((line, index) => ({ line, lineNumber: index + 1 }))
-    .filter(({ line }) => line.trim().length > 0)
-    .map(({ line, lineNumber }) => {
+const resolveDatasetUploadFormat = (filename: string): DatasetUploadFormat => {
+  const normalizedFilename = filename.toLowerCase();
+
+  if (normalizedFilename.endsWith(".json")) return "json";
+  if (normalizedFilename.endsWith(".jsonl")) return "jsonl";
+  if (normalizedFilename.endsWith(".csv")) return "csv";
+
+  assert(false, `${filename} must have a .json, .jsonl, or .csv extension`);
+};
+
+const readDatasetUpload = async (filePath: string) => {
+  const filename = basename(filePath);
+  const format = resolveDatasetUploadFormat(filename);
+  const payload = await readFile(filePath, "utf8");
+
+  if (format === "jsonl") {
+    for (const { line, lineNumber } of payload
+      .split(/\r?\n/u)
+      .map((line, index) => ({ line, lineNumber: index + 1 }))
+      .filter(({ line }) => line.length > 0)) {
       let value: unknown;
       try {
         value = JSON.parse(line);
       } catch {
-        assert(false, `${basename(filePath)}:${lineNumber} is not valid JSON`);
+        assert(false, `${filename}:${lineNumber} is not valid JSON`);
       }
-      assert(isJsonObject(value), `${basename(filePath)}:${lineNumber} must contain a JSON object`);
-      return value;
+      assert(isJsonObject(value), `${filename}:${lineNumber} must contain a JSON object`);
+    }
+  }
+
+  return { payload, format, contentType: DATASET_UPLOAD_CONTENT_TYPES[format] };
+};
+
+type StartedDatasetUpload = { readonly uploadUrl: string; readonly job: { readonly id: UploadId } };
+
+const cancelStartedUpload = async (
+  client: PotatoClient,
+  uploadId: StartedDatasetUpload["job"]["id"],
+  error: unknown,
+): Promise<never> => {
+  const reason = error instanceof Error ? error.message : String(error);
+
+  try {
+    await client.datasets.cancelUpload({ uploadId });
+  } catch (cancelError) {
+    const cancelReason = cancelError instanceof Error ? cancelError.message : String(cancelError);
+    throw new AggregateError(
+      [error, cancelError],
+      `Dataset upload ${uploadId} failed and could not be canceled. Upload error: ${reason}. Cancellation error: ${cancelReason}.`,
+    );
+  }
+
+  throw new DatasetUploadError(
+    `Dataset upload ${uploadId} failed and was canceled: ${reason}`,
+    error,
+  );
+};
+
+const transferAndFinalizeUpload = async ({
+  client,
+  started,
+  payload,
+  contentType,
+  fetch,
+  retryIntervalMs,
+}: {
+  client: PotatoClient;
+  started: StartedDatasetUpload;
+  payload: string;
+  contentType: string;
+  fetch: typeof globalThis.fetch;
+  retryIntervalMs: number;
+}): Promise<void> => {
+  let uploadResponse: Response;
+  try {
+    uploadResponse = await fetch(started.uploadUrl, {
+      method: "PUT",
+      headers: {
+        "Content-Type": contentType,
+        "x-ms-blob-type": "BlockBlob",
+      },
+      body: payload,
     });
+  } catch (error) {
+    return cancelStartedUpload(client, started.job.id, error);
+  }
+
+  if (!uploadResponse.ok) {
+    return cancelStartedUpload(
+      client,
+      started.job.id,
+      new DatasetUploadError(
+        `Dataset upload failed with HTTP ${uploadResponse.status}.`,
+        undefined,
+      ),
+    );
+  }
+
+  let transientFailures = 0;
+  while (true) {
+    try {
+      await client.datasets.finalizeUpload({ uploadId: started.job.id });
+      return;
+    } catch (error) {
+      if (!(error instanceof PotatoApiError && error.status === 503)) {
+        return cancelStartedUpload(client, started.job.id, error);
+      }
+
+      transientFailures += 1;
+      if (transientFailures === FINALIZE_UPLOAD_ATTEMPTS) {
+        return cancelStartedUpload(
+          client,
+          started.job.id,
+          new DatasetUploadError(
+            `Storage remained unavailable after ${FINALIZE_UPLOAD_ATTEMPTS} finalization attempts.`,
+            error,
+          ),
+        );
+      }
+
+      const { promise, resolve } = Promise.withResolvers<void>();
+      setTimeout(resolve, retryIntervalMs);
+      await promise;
+    }
+  }
 };
 
 export async function uploadDataset(
   client: PotatoClient,
   options: UploadDatasetOptions,
 ): Promise<UploadDatasetResult> {
-  const rows = await readJsonLines(options.filePath);
+  const { payload, format, contentType } = await readDatasetUpload(options.filePath);
   const resolved = await resolveLiveDataset(client, options, true);
   const datasetId = resolved.datasetId;
 
@@ -236,25 +363,21 @@ export async function uploadDataset(
     if (!fieldsByName.has(field.name)) fieldsByName.set(field.name, field);
   }
   const fields = [...fieldsByName.values()];
-  const payload = JSON.stringify(rows);
-  const payloadBytes = new TextEncoder().encode(payload);
   const started = await client.datasets.startUpload({
     datasetId,
     filename: basename(options.filePath),
-    format: "json",
-    declaredSizeBytes: payloadBytes.byteLength,
+    format,
+    declaredSizeBytes: new TextEncoder().encode(payload).byteLength,
     fields,
   });
-  const uploadResponse = await (options.fetch ?? globalThis.fetch)(started.uploadUrl, {
-    method: "PUT",
-    headers: {
-      "Content-Type": "application/json",
-      "x-ms-blob-type": "BlockBlob",
-    },
-    body: payloadBytes,
+  await transferAndFinalizeUpload({
+    client,
+    started,
+    payload,
+    contentType,
+    fetch: options.fetch ?? globalThis.fetch,
+    retryIntervalMs: options.waitFor?.intervalMs ?? 1000,
   });
-  assert(uploadResponse.ok, `Dataset upload failed with HTTP ${uploadResponse.status}.`);
-  await client.datasets.finalizeUpload({ uploadId: started.job.id });
   const completed = await client.waitFor(
     { type: "datasetUpload", uploadId: started.job.id },
     options.waitFor,

package/src/cli/file-system.ts

@@ -0,0 +1,34 @@
+import { assert } from "es-toolkit";
+import { lstat } from "node:fs/promises";
+import { join } from "node:path";
+
+export const hasErrorCode = (error: unknown, ...codes: readonly string[]) =>
+  error instanceof Error &&
+  "code" in error &&
+  typeof error.code === "string" &&
+  codes.includes(error.code);
+
+export const isMissingFileError = (error: unknown) => hasErrorCode(error, "ENOENT");
+
+export const entryStats = async (path: string) => {
+  try {
+    return await lstat(path);
+  } catch (error) {
+    if (isMissingFileError(error)) return undefined;
+    throw error;
+  }
+};
+
+export const assertSafeDestination = async (
+  targetDirectory: string,
+  relativePath: string,
+  action = "write",
+) => {
+  let currentPath = targetDirectory;
+  for (const segment of relativePath.split("/")) {
+    currentPath = join(currentPath, segment);
+    const entry = await entryStats(currentPath);
+    if (entry === undefined) return;
+    assert(!entry.isSymbolicLink(), `Refusing to ${action} through symbolic link ${currentPath}.`);
+  }
+};

package/src/cli/format-error.ts

@@ -0,0 +1,19 @@
+import { PotatoApiError } from "../errors";
+
+export const apiErrorDetails = (error: PotatoApiError): string | undefined => {
+  const details =
+    typeof error.body === "string"
+      ? error.body
+      : error.body === undefined
+        ? undefined
+        : JSON.stringify(error.body);
+  return details === undefined || details.length === 0 ? undefined : details;
+};
+
+export const formatCliError = (error: unknown): string => {
+  if (error instanceof PotatoApiError) {
+    const details = apiErrorDetails(error);
+    return details === undefined ? error.message : `${error.message}\n${details}`;
+  }
+  return error instanceof Error ? error.message : String(error);
+};