@toloka-platform/sdk 0.1.0-preview.3

package/src/cli/data.ts

@@ -0,0 +1,415 @@
+import { assert } from "es-toolkit";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { basename, dirname } from "node:path";
+import type { Dataset, InferredField, PotatoClient } from "../client";
+import { PotatoApiError } from "../errors";
+import { generateFieldDeclarationsSource } from "../importer";
+import { readTolokaState } from "../state";
+import type { WaitForOptions } from "../wait-for";
+import { desiredDatasets } from "./project";
+import { MISSING_PROJECT_ID_MESSAGE } from "./project-context";
+import { loadProjectSource } from "./source";
+import type { ProjectSource } from "./source";
+
+const RAW_UPLOAD_FIELD_NAME = "uploaded";
+const DATASET_ID_PATTERN =
+  /^ds\.[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
+
+const canonicalDatasetId = (token: string): string | undefined =>
+  DATASET_ID_PATTERN.test(token) ? token.toLowerCase() : undefined;
+
+type UploadFieldDeclaration = NonNullable<
+  Parameters<PotatoClient["datasets"]["startUpload"]>[0]["fields"]
+>[number];
+
+export type UploadDatasetOptions = {
+  readonly sourceRoot: string;
+  readonly datasetName: string;
+  readonly filePath: string;
+  readonly projectId?: string;
+  readonly source?: ProjectSource;
+  readonly fetch?: typeof globalThis.fetch;
+  readonly waitFor?: WaitForOptions;
+};
+
+export type GenerateDatasetFieldDeclarationsOptions = {
+  readonly sourceRoot: string;
+  readonly datasetName: string;
+  readonly projectId?: string;
+};
+
+export type GenerateDatasetFieldDeclarationsResult = {
+  readonly datasetId: string;
+  readonly inferredFields: ReadonlyArray<InferredField>;
+  readonly declarations: string;
+  readonly output: string;
+};
+
+export type UploadDatasetResult = {
+  readonly datasetId: string;
+  readonly rowsUploaded: number;
+  readonly inferredFields: ReadonlyArray<InferredField>;
+  readonly output: string;
+  readonly warning?: string;
+};
+
+export type SetProjectSecretOptions = {
+  readonly projectId: string;
+  readonly name: string;
+  readonly value: string;
+};
+
+export type SetProjectSecretResult = {
+  readonly name: string;
+  readonly output: string;
+};
+
+export type DatasetCommandResult = {
+  readonly output: string;
+};
+
+export type LoadDatasetOptions = {
+  readonly sourceRoot: string;
+  readonly datasetName: string;
+  readonly outPath: string;
+  readonly projectId?: string;
+  readonly limit?: number;
+  readonly timeoutMs?: number;
+  readonly fetch?: typeof globalThis.fetch;
+};
+
+type DatasetRef = {
+  readonly datasetId: string;
+  readonly name: string;
+  readonly nameIsUnique: boolean;
+};
+
+const isJsonObject = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const summarizeJsonSchema = (schema: unknown): string => {
+  if (!isJsonObject(schema)) return "unknown";
+  if (Array.isArray(schema.anyOf)) {
+    const variants = [...new Set(schema.anyOf.map(summarizeJsonSchema))];
+    return variants.length === 0 ? "unknown" : variants.join(" | ");
+  }
+  const schemaType = schema.type;
+  const types =
+    typeof schemaType === "string"
+      ? [schemaType]
+      : Array.isArray(schemaType)
+        ? schemaType.filter((value): value is string => typeof value === "string")
+        : [];
+  if (types.length === 0) return "unknown";
+  return types
+    .map((type) => {
+      if (type !== "array") return type;
+      const itemType = summarizeJsonSchema(schema.items);
+      return itemType === "unknown" ? "array" : `array<${itemType}>`;
+    })
+    .join(" | ");
+};
+
+const formatInferredFields = (fields: ReadonlyArray<InferredField>): string => {
+  if (fields.length === 0) return "No inferred fields are available yet.";
+  return [
+    "Inferred fields:",
+    ...fields
+      .toSorted((left, right) => left.jsonPath.localeCompare(right.jsonPath))
+      .map(
+        (field) =>
+          `${"  ".repeat(field.depth)}- ${field.jsonPath}: ${summarizeJsonSchema(field.jsonSchema)}`,
+      ),
+  ].join("\n");
+};
+
+const declaredDatasetFields = async (options: {
+  readonly sourceRoot: string;
+  readonly datasetName: string;
+  readonly source?: ProjectSource;
+}): Promise<ReadonlyArray<UploadFieldDeclaration>> => {
+  const source = options.source ?? (await loadProjectSource(options.sourceRoot));
+  const declared = desiredDatasets(source).find((dataset) => dataset.name === options.datasetName);
+  return declared === undefined
+    ? []
+    : declared.fields.map((field) => ({ name: field.name, jsonSchema: field.jsonSchema }));
+};
+
+const formatDatasetChoice = (dataset: Dataset): string => `${dataset.name} (${dataset.id})`;
+
+const collectLiveDatasets = async (client: PotatoClient, projectId: string): Promise<Dataset[]> => {
+  const datasets: Dataset[] = [];
+  for await (const dataset of client.datasets.listAll({ projectId })) datasets.push(dataset);
+  return datasets.toSorted((left, right) => {
+    const byName = left.name.localeCompare(right.name);
+    return byName === 0 ? left.id.localeCompare(right.id) : byName;
+  });
+};
+
+const resolveLiveDataset = async (
+  client: PotatoClient,
+  options: GenerateDatasetFieldDeclarationsOptions & { readonly source?: ProjectSource },
+  createWhenMissing: boolean,
+): Promise<DatasetRef> => {
+  const state = await readTolokaState(options.sourceRoot);
+  const projectId = options.projectId ?? state?.projectId;
+  assert(projectId, MISSING_PROJECT_ID_MESSAGE);
+  const token = options.datasetName;
+  const live = await collectLiveDatasets(client, projectId);
+  const nameIsUnique = (name: string) =>
+    live.filter((dataset) => dataset.name === name).length <= 1;
+  const datasetId = canonicalDatasetId(token);
+
+  const byId = live.find(
+    (dataset) => dataset.id === token || dataset.id.toLowerCase() === datasetId,
+  );
+  if (byId !== undefined) {
+    return { datasetId: byId.id, name: byId.name, nameIsUnique: nameIsUnique(byId.name) };
+  }
+
+  const byName = live.filter((dataset) => dataset.name === token);
+  assert(
+    byName.length <= 1,
+    `Project ${projectId} has multiple datasets named "${token}": ${byName.map(formatDatasetChoice).join(", ")}.`,
+  );
+  if (byName.length === 1) {
+    const [dataset] = byName;
+    assert(dataset, `Dataset "${token}" has no resolved ID.`);
+    return { datasetId: dataset.id, name: dataset.name, nameIsUnique: true };
+  }
+
+  const binding = state?.projectId === projectId ? state.datasets[token] : undefined;
+  if (binding !== undefined) {
+    const bound = live.find((dataset) => dataset.id === binding.datasetId);
+    if (bound !== undefined) {
+      return { datasetId: bound.id, name: bound.name, nameIsUnique: nameIsUnique(bound.name) };
+    }
+  }
+
+  assert(
+    createWhenMissing && datasetId === undefined,
+    live.length === 0
+      ? `Dataset "${token}" does not exist in project ${projectId}. Run toloka apply first.`
+      : `Dataset "${token}" does not exist in project ${projectId}. This project has: ${live.map(formatDatasetChoice).join(", ")}.`,
+  );
+
+  assert(
+    (await declaredDatasetFields(options)).length > 0,
+    `Dataset "${token}" does not exist in project ${projectId} and no pipeline in source declares it. Bind a pipeline to it in source so its columns are known, or upload to an existing dataset.`,
+  );
+  const created = await client.datasets.create({ projectId, name: token });
+  return { datasetId: created.id, name: created.name, nameIsUnique: true };
+};
+
+const readJsonLines = async (filePath: string): Promise<ReadonlyArray<Record<string, unknown>>> => {
+  const source = await readFile(filePath, "utf8");
+  return source
+    .split(/\r?\n/u)
+    .map((line, index) => ({ line, lineNumber: index + 1 }))
+    .filter(({ line }) => line.trim().length > 0)
+    .map(({ line, lineNumber }) => {
+      let value: unknown;
+      try {
+        value = JSON.parse(line);
+      } catch {
+        assert(false, `${basename(filePath)}:${lineNumber} is not valid JSON`);
+      }
+      assert(isJsonObject(value), `${basename(filePath)}:${lineNumber} must contain a JSON object`);
+      return value;
+    });
+};
+
+export async function uploadDataset(
+  client: PotatoClient,
+  options: UploadDatasetOptions,
+): Promise<UploadDatasetResult> {
+  const rows = await readJsonLines(options.filePath);
+  const resolved = await resolveLiveDataset(client, options, true);
+  const datasetId = resolved.datasetId;
+
+  const fieldsByName = new Map<string, UploadFieldDeclaration>();
+  for await (const field of client.datasets.listAllFields({ datasetId })) {
+    if (field.internal || field.name === RAW_UPLOAD_FIELD_NAME) continue;
+    fieldsByName.set(field.name, { name: field.name, jsonSchema: field.jsonSchema });
+  }
+  for (const field of await declaredDatasetFields({ ...options, datasetName: resolved.name })) {
+    if (!fieldsByName.has(field.name)) fieldsByName.set(field.name, field);
+  }
+  const fields = [...fieldsByName.values()];
+  const payload = JSON.stringify(rows);
+  const payloadBytes = new TextEncoder().encode(payload);
+  const started = await client.datasets.startUpload({
+    datasetId,
+    filename: basename(options.filePath),
+    format: "json",
+    declaredSizeBytes: payloadBytes.byteLength,
+    fields,
+  });
+  const uploadResponse = await (options.fetch ?? globalThis.fetch)(started.uploadUrl, {
+    method: "PUT",
+    headers: {
+      "Content-Type": "application/json",
+      "x-ms-blob-type": "BlockBlob",
+    },
+    body: payloadBytes,
+  });
+  assert(uploadResponse.ok, `Dataset upload failed with HTTP ${uploadResponse.status}.`);
+  await client.datasets.finalizeUpload({ uploadId: started.job.id });
+  const completed = await client.waitFor(
+    { type: "datasetUpload", uploadId: started.job.id },
+    options.waitFor,
+  );
+  const fieldsHint = resolved.nameIsUnique ? resolved.name : resolved.datasetId;
+  const compatibilityHint =
+    completed.status === "failed" &&
+    completed.error?.startsWith("Schema incompatible with existing upload") === true
+      ? `\nRun toloka dataset fields ${JSON.stringify(fieldsHint)} to inspect the inferred schema stored from the first successful upload.`
+      : "";
+  assert(
+    completed.status === "succeeded",
+    `${completed.error ?? `Dataset upload ${completed.id} failed.`}${compatibilityHint}`,
+  );
+
+  const uploadedOutput = `Uploaded ${completed.result.itemsIngested} rows to dataset ${resolved.name}.`;
+  let inferredFields: ReadonlyArray<InferredField> = [];
+  let inferenceReport: string | undefined;
+  let warning: string | undefined;
+  try {
+    const inferred = await client.datasets.getInferredFields({ datasetId });
+    inferredFields = inferred.fields;
+    inferenceReport = formatInferredFields(inferredFields);
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    warning = `Upload succeeded, but inferred fields could not be fetched: ${reason}`;
+  }
+  return {
+    datasetId,
+    rowsUploaded: completed.result.itemsIngested,
+    inferredFields,
+    output:
+      inferenceReport === undefined ? uploadedOutput : `${uploadedOutput}\n${inferenceReport}`,
+    ...(warning === undefined ? {} : { warning }),
+  };
+}
+
+export async function generateDatasetFieldDeclarations(
+  client: PotatoClient,
+  options: GenerateDatasetFieldDeclarationsOptions,
+): Promise<GenerateDatasetFieldDeclarationsResult> {
+  const resolved = await resolveLiveDataset(client, options, false);
+  const inferred = await client.datasets.getInferredFields({ datasetId: resolved.datasetId });
+  const topLevelFields = inferred.fields.filter((field) => field.depth === 1);
+  const declarations = generateFieldDeclarationsSource(
+    topLevelFields.map(({ name, jsonSchema }) => ({ name, jsonSchema })),
+  );
+  return {
+    datasetId: resolved.datasetId,
+    inferredFields: inferred.fields,
+    declarations,
+    output:
+      topLevelFields.length === 0
+        ? `Dataset ${resolved.name} has no inferred fields yet.`
+        : declarations,
+  };
+}
+
+export async function listProjectDatasets(
+  client: PotatoClient,
+  options: { readonly projectId: string },
+): Promise<DatasetCommandResult> {
+  const live = await collectLiveDatasets(client, options.projectId);
+  if (live.length === 0) {
+    return { output: "This project has no datasets." };
+  }
+  return {
+    output: live.map((dataset) => `${dataset.name}\t${dataset.id}`).join("\n"),
+  };
+}
+
+const downloadExportRows = async (options: {
+  readonly downloadUrl: string;
+  readonly exportId: string;
+  readonly fetch: typeof globalThis.fetch;
+}): Promise<Record<string, unknown>[]> => {
+  const download = await options.fetch(options.downloadUrl);
+  assert(download.ok, `Dataset export download failed with HTTP ${download.status}.`);
+  const text = await download.text();
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch (error) {
+    throw new Error(`Dataset export ${options.exportId} is not valid JSON.`, { cause: error });
+  }
+  assert(Array.isArray(parsed), `Dataset export ${options.exportId} is not a JSON array.`);
+  const rows: Record<string, unknown>[] = [];
+  for (const [index, element] of parsed.entries()) {
+    assert(
+      isJsonObject(element),
+      `Dataset export ${options.exportId} row ${index} is not a JSON object.`,
+    );
+    rows.push(element);
+  }
+  return rows;
+};
+
+export async function loadDataset(
+  client: PotatoClient,
+  options: LoadDatasetOptions,
+): Promise<DatasetCommandResult> {
+  const resolved = await resolveLiveDataset(client, options, false);
+  const fieldNames: string[] = [];
+  for await (const field of client.datasets.listAllFields({ datasetId: resolved.datasetId })) {
+    if (field.internal || field.name === RAW_UPLOAD_FIELD_NAME) continue;
+    fieldNames.push(field.name);
+  }
+  assert(fieldNames.length > 0, `Dataset ${resolved.name} has no fields to export.`);
+
+  const started = await client.datasets.startExport({
+    datasetId: resolved.datasetId,
+    fields: fieldNames,
+  });
+  let completed;
+  try {
+    completed = await client.waitFor(
+      { type: "datasetExport", exportId: started.id },
+      options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs },
+    );
+  } catch (error) {
+    assert(
+      !(error instanceof PotatoApiError && error.code === "wait_timeout"),
+      `Stopped waiting for dataset export ${started.id}; raise --timeout to wait longer.`,
+    );
+    throw error;
+  }
+
+  assert(
+    completed.result.downloadUrl,
+    `Dataset export ${completed.id} succeeded without a download URL.`,
+  );
+
+  const rows = await downloadExportRows({
+    downloadUrl: completed.result.downloadUrl,
+    exportId: completed.id,
+    fetch: options.fetch ?? globalThis.fetch,
+  });
+
+  const limited = options.limit === undefined ? rows : rows.slice(0, options.limit);
+  await mkdir(dirname(options.outPath), { recursive: true });
+  await writeFile(
+    options.outPath,
+    limited.length === 0 ? "" : `${limited.map((row) => JSON.stringify(row)).join("\n")}\n`,
+  );
+  return { output: `Wrote ${limited.length} rows to ${options.outPath}.` };
+}
+
+export async function setProjectSecret(
+  client: PotatoClient,
+  options: SetProjectSecretOptions,
+): Promise<SetProjectSecretResult> {
+  await client.secrets.set({
+    projectId: options.projectId,
+    name: options.name,
+    value: options.value,
+  });
+  return { name: options.name, output: `Set secret ${options.name}.` };
+}

package/src/cli/index.ts

@@ -0,0 +1,44 @@
+export { createProgram, runTolokaCli } from "./program";
+export type { CliIo, CreatedProgram, ProgramContext } from "./program";
+
+export { resolveTolokaConfig } from "./config";
+export type { ResolvedTolokaConfig, TolokaCliFlags, TolokaEnvironment } from "./config";
+export { setProjectSecret, uploadDataset } from "./data";
+export type {
+  SetProjectSecretOptions,
+  SetProjectSecretResult,
+  UploadDatasetOptions,
+  UploadDatasetResult,
+} from "./data";
+export { initProject } from "./init";
+export type { InitProjectResult } from "./init";
+export {
+  applyProject,
+  diffPipelineGraphs,
+  importProject,
+  planProject,
+  validateProject,
+} from "./project";
+export type {
+  ApplyProjectOptions,
+  ApplyProjectResult,
+  ImportProjectOptions,
+  ImportProjectResult,
+  PipelinePlanChange,
+  PipelineValidationResult,
+  PlanProjectOptions,
+  PlanProjectResult,
+  ProjectPlanChange,
+  ValidateProjectOptions,
+  ValidateProjectResult,
+} from "./project";
+export { collectGraphSecretNames } from "./secrets";
+export { loadProjectSource } from "./source";
+export type { ProjectPipelineSource, ProjectSource } from "./source";
+export {
+  readTolokaState,
+  TOLOKA_STATE_FILENAME,
+  tolokaStateSchema,
+  writeTolokaState,
+} from "../state";
+export type { TolokaDatasetBinding, TolokaPipelineBinding, TolokaState } from "../state";