@toloka-platform/sdk 0.2.0-preview.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +5 -6
  2. package/dist/cli/toloka.js +136 -136
  3. package/dist/cli/toloka.js.jsc +0 -0
  4. package/dist/cli/toloka.js.jsc.sha256 +1 -1
  5. package/dist/cli/toloka.js.sha256 +1 -1
  6. package/docs/cli-v2.md +3 -2
  7. package/docs/cli.md +35 -7
  8. package/docs/getting-started.md +4 -2
  9. package/docs/limitations.md +2 -2
  10. package/openapi/gen/models/baseModel.ts +1 -0
  11. package/openapi/gen/models/currentUser.ts +2 -0
  12. package/openapi/gen/models/currentUserTeamsItem.ts +6 -0
  13. package/openapi/gen/models/gistingTrainingConfiguration.ts +6 -2
  14. package/openapi/gen/models/gistingTrainingConfigurationBatchSize.ts +2 -0
  15. package/openapi/gen/models/gistingTrainingConfigurationGistTokens.ts +2 -0
  16. package/openapi/gen/models/gistingTrainingConfigurationInput.ts +6 -2
  17. package/openapi/gen/models/gistingTrainingConfigurationInputBatchSize.ts +2 -0
  18. package/openapi/gen/models/gistingTrainingConfigurationInputGistTokens.ts +2 -0
  19. package/openapi/gen/models/index.ts +9 -4
  20. package/openapi/gen/models/sftTrainingConfiguration.ts +6 -3
  21. package/openapi/gen/models/sftTrainingConfigurationBatchSize.ts +2 -0
  22. package/openapi/gen/models/sftTrainingConfigurationInput.ts +6 -3
  23. package/openapi/gen/models/sftTrainingConfigurationInputBatchSize.ts +2 -0
  24. package/openapi/gen/models/sftTrainingConfigurationInputLoraRank.ts +2 -0
  25. package/openapi/gen/models/sftTrainingConfigurationLoraRank.ts +2 -0
  26. package/package.json +2 -2
  27. package/skills/catalog.json +11 -11
  28. package/skills/toloka-build/SKILL.md +29 -18
  29. package/skills/toloka-build/references/node-semantics.md +7 -7
  30. package/skills/toloka-build/references/workflow-and-cli.md +15 -12
  31. package/skills/toloka-fine-tuning/SKILL.md +145 -62
  32. package/skills/toloka-get-started/SKILL.md +37 -21
  33. package/skills/toloka-gisting/SKILL.md +156 -66
  34. package/skills/toloka-report-issue/references/report-template.md +1 -1
  35. package/src/cli/analytics.ts +50 -23
  36. package/src/cli/auth.ts +31 -12
  37. package/src/cli/auto-update.ts +12 -6
  38. package/src/cli/check.ts +76 -29
  39. package/src/cli/cli-entry.ts +3 -2
  40. package/src/cli/cli-error.ts +43 -0
  41. package/src/cli/commands/dataset.ts +13 -3
  42. package/src/cli/commands/models.ts +6 -7
  43. package/src/cli/commands/pipeline.ts +6 -7
  44. package/src/cli/commands/project.ts +9 -16
  45. package/src/cli/commands/report.ts +6 -2
  46. package/src/cli/commands/secret.ts +5 -4
  47. package/src/cli/commands/source.ts +4 -4
  48. package/src/cli/commands/status.ts +2 -3
  49. package/src/cli/commands/training.ts +50 -17
  50. package/src/cli/config.ts +2 -2
  51. package/src/cli/context.ts +55 -17
  52. package/src/cli/credentials.ts +12 -4
  53. package/src/cli/data.ts +96 -34
  54. package/src/cli/delivery-orders.ts +18 -1
  55. package/src/cli/file-system.ts +6 -2
  56. package/src/cli/format-error.ts +45 -2
  57. package/src/cli/init.ts +13 -7
  58. package/src/cli/models.ts +43 -12
  59. package/src/cli/program.ts +5 -56
  60. package/src/cli/project-context.ts +117 -32
  61. package/src/cli/project.ts +80 -28
  62. package/src/cli/report.ts +24 -9
  63. package/src/cli/runs.ts +33 -8
  64. package/src/cli/settings.ts +13 -5
  65. package/src/cli/skill.ts +61 -22
  66. package/src/cli/source.ts +3 -2
  67. package/src/cli/terminal-io.ts +13 -6
  68. package/src/cli/trainings.ts +129 -35
  69. package/src/cli/version.ts +2 -2
  70. package/src/client.ts +3 -9
  71. package/src/transport.ts +13 -1
  72. package/openapi/gen/models/gistingTrainingConfigurationInputRegime.ts +0 -2
  73. package/openapi/gen/models/gistingTrainingConfigurationRegime.ts +0 -2
  74. package/openapi/gen/models/sftTrainingConfigurationInputRegime.ts +0 -2
  75. package/openapi/gen/models/sftTrainingConfigurationRegime.ts +0 -2
@@ -1,17 +1,20 @@
1
- import { assert } from "es-toolkit";
2
1
  import type { Command } from "commander";
3
2
  import {
4
- nonNegativeInteger,
5
3
  positiveInteger,
6
4
  resolveContext,
5
+ trainingBatchSize,
7
6
  trainingDraftOptions,
7
+ trainingEpochs,
8
+ trainingGistTokens,
9
+ trainingLearningRate,
10
+ trainingLoraRank,
11
+ trainingMaxSequenceLength,
8
12
  trainingMethod,
9
- trainingRegime,
10
13
  type CommandHost,
11
14
  type RawTrainingDraftOptions,
12
15
  } from "../context";
13
16
  import { parseBaseModelId } from "../models";
14
- import { MISSING_PROJECT_ID_MESSAGE } from "../project-context";
17
+ import { assertProjectSelected } from "../project-context";
15
18
  import {
16
19
  cancelTrainingRun,
17
20
  createTraining,
@@ -37,7 +40,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
37
40
  .description("list active Trainings owned by the selected project")
38
41
  .action(async (options: { json: boolean }) => {
39
42
  const { client, projectId } = await resolveContext(globals(), context);
40
- assert(projectId, MISSING_PROJECT_ID_MESSAGE);
43
+ assertProjectSelected(projectId);
41
44
  await listTrainings(client, { projectId, json: options.json }, context.io);
42
45
  });
43
46
  training
@@ -47,7 +50,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
47
50
  .description("show one Training draft")
48
51
  .action(async (nameOrId: string, options: { json: boolean }) => {
49
52
  const { client, projectId } = await resolveContext(globals(), context);
50
- assert(projectId, MISSING_PROJECT_ID_MESSAGE);
53
+ assertProjectSelected(projectId);
51
54
  await getTraining(client, { projectId, nameOrId, json: options.json }, context.io);
52
55
  });
53
56
  training
@@ -60,15 +63,30 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
60
63
  .option("--validation-dataset <id>", "optional validation Dataset ID")
61
64
  .option("--train-filter <field=value>", "training Dataset field equality filter")
62
65
  .option("--validation-filter <field=value>", "validation Dataset field equality filter")
63
- .option("--regime <regime>", "Python training regime", trainingRegime)
64
- .option("--epochs <count>", "training epochs", nonNegativeInteger)
65
- .option("--eval-steps <count>", "SFT eval steps", positiveInteger)
66
+ .option("--lora-rank <rank>", "SFT LoRA adapter rank: 8, 16, or 32 (default 16)", trainingLoraRank)
67
+ .option("--gist-tokens <count>", "gisting soft tokens: 8, 32, or 64 (default 32)", trainingGistTokens)
68
+ .option("--epochs <count>", "training epochs, 1 through 5 (default 3)", trainingEpochs)
69
+ .option(
70
+ "--learning-rate <rate>",
71
+ "optimizer learning rate (default 0.00015 for SFT, 0.03 for gisting)",
72
+ trainingLearningRate,
73
+ )
74
+ .option(
75
+ "--batch-size <count>",
76
+ "sequences per optimizer step: 8, 16, 32, or 64 (default 8)",
77
+ trainingBatchSize,
78
+ )
79
+ .option(
80
+ "--max-sequence-length <tokens>",
81
+ "token budget per example, 64 through 16384 (default 12288 for SFT, 3072 for gisting)",
82
+ trainingMaxSequenceLength,
83
+ )
66
84
  .option("--prompt-file <path>", "gisting prompt text file")
67
85
  .option("--json", "print JSON", false)
68
86
  .description("create a normalized Training draft")
69
87
  .action(async (name: string, options: RawTrainingDraftOptions & { json: boolean }) => {
70
88
  const { client, projectId } = await resolveContext(globals(), context);
71
- assert(projectId, MISSING_PROJECT_ID_MESSAGE);
89
+ assertProjectSelected(projectId);
72
90
  await createTraining(
73
91
  client,
74
92
  {
@@ -92,15 +110,30 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
92
110
  "--validation-filter <field=value>",
93
111
  "replacement validation Dataset field equality filter",
94
112
  )
95
- .option("--regime <regime>", "Python training regime", trainingRegime)
96
- .option("--epochs <count>", "training epochs", nonNegativeInteger)
97
- .option("--eval-steps <count>", "SFT eval steps", positiveInteger)
113
+ .option("--lora-rank <rank>", "SFT LoRA adapter rank: 8, 16, or 32 (default 16)", trainingLoraRank)
114
+ .option("--gist-tokens <count>", "gisting soft tokens: 8, 32, or 64 (default 32)", trainingGistTokens)
115
+ .option("--epochs <count>", "training epochs, 1 through 5 (default 3)", trainingEpochs)
116
+ .option(
117
+ "--learning-rate <rate>",
118
+ "optimizer learning rate (default 0.00015 for SFT, 0.03 for gisting)",
119
+ trainingLearningRate,
120
+ )
121
+ .option(
122
+ "--batch-size <count>",
123
+ "sequences per optimizer step: 8, 16, 32, or 64 (default 8)",
124
+ trainingBatchSize,
125
+ )
126
+ .option(
127
+ "--max-sequence-length <tokens>",
128
+ "token budget per example, 64 through 16384 (default 12288 for SFT, 3072 for gisting)",
129
+ trainingMaxSequenceLength,
130
+ )
98
131
  .option("--prompt-file <path>", "replacement gisting prompt text file")
99
132
  .option("--json", "print JSON", false)
100
133
  .description("replace the complete Training draft")
101
134
  .action(async (nameOrId: string, options: RawTrainingDraftOptions & { json: boolean }) => {
102
135
  const { client, projectId } = await resolveContext(globals(), context);
103
- assert(projectId, MISSING_PROJECT_ID_MESSAGE);
136
+ assertProjectSelected(projectId);
104
137
  await updateTraining(
105
138
  client,
106
139
  {
@@ -120,7 +153,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
120
153
  .description("validate Training inputs and calculate the maximum charge")
121
154
  .action(async (nameOrId: string, options: { idempotencyKey?: string; json: boolean }) => {
122
155
  const { client, projectId } = await resolveContext(globals(), context);
123
- assert(projectId, MISSING_PROJECT_ID_MESSAGE);
156
+ assertProjectSelected(projectId);
124
157
  await estimateTrainingCost(
125
158
  client,
126
159
  { projectId, nameOrId, idempotencyKey: options.idempotencyKey, json: options.json },
@@ -146,7 +179,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
146
179
  },
147
180
  ) => {
148
181
  const { client, projectId } = await resolveContext(globals(), context);
149
- assert(projectId, MISSING_PROJECT_ID_MESSAGE);
182
+ assertProjectSelected(projectId);
150
183
  await createTrainingRun(client, { projectId, nameOrId, ...options }, context.io);
151
184
  },
152
185
  );
@@ -157,7 +190,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
157
190
  .description("list Training Run history")
158
191
  .action(async (nameOrId: string, options: { json: boolean }) => {
159
192
  const { client, projectId } = await resolveContext(globals(), context);
160
- assert(projectId, MISSING_PROJECT_ID_MESSAGE);
193
+ assertProjectSelected(projectId);
161
194
  await listTrainingRuns(client, { projectId, nameOrId, json: options.json }, context.io);
162
195
  });
163
196
  training
package/src/cli/config.ts CHANGED
@@ -1,7 +1,7 @@
1
- import { assert } from "es-toolkit";
2
1
  import type { TolokaState } from "../state";
3
2
  import type { StoredCredentials } from "./credentials";
4
3
  import { resolveProjectContext } from "./project-context";
4
+ import { assertCli } from "./cli-error";
5
5
 
6
6
  export type TolokaCliFlags = {
7
7
  readonly apiKey?: string;
@@ -76,6 +76,6 @@ export function resolveTolokaConfig(
76
76
  stored,
77
77
  effectiveBaseUrl: baseUrl,
78
78
  });
79
- assert(apiKey, missingApiKeyMessage);
79
+ assertCli(apiKey, missingApiKeyMessage, { hint: { disabled: true } });
80
80
  return { apiKey, baseUrl, ...(project === undefined ? {} : { projectId: project.id }) };
81
81
  }
@@ -30,7 +30,6 @@ export type ProgramContext = {
30
30
  readonly injectedClient?: PotatoClient;
31
31
  readonly runChecks?: typeof checkProject;
32
32
  readonly analytics?: CliAnalytics;
33
- readonly internalUser?: boolean;
34
33
  };
35
34
 
36
35
  export type GlobalOptions = {
@@ -89,11 +88,52 @@ export const trainingMethod = (raw: string): "sft" | "gisting" => {
89
88
  return raw;
90
89
  };
91
90
 
92
- export const trainingRegime = (raw: string): "low" | "medium" | "high" => {
93
- if (raw !== "low" && raw !== "medium" && raw !== "high") {
94
- throw new InvalidArgumentError('Expected regime "low", "medium", or "high".');
91
+ export const trainingLoraRank = (raw: string): 8 | 16 | 32 => {
92
+ const value = Number(raw);
93
+ if (value !== 8 && value !== 16 && value !== 32) {
94
+ throw new InvalidArgumentError("Expected a LoRA rank of 8, 16, or 32.");
95
95
  }
96
- return raw;
96
+ return value;
97
+ };
98
+
99
+ export const trainingGistTokens = (raw: string): 8 | 32 | 64 => {
100
+ const value = Number(raw);
101
+ if (value !== 8 && value !== 32 && value !== 64) {
102
+ throw new InvalidArgumentError("Expected 8, 32, or 64 gist tokens.");
103
+ }
104
+ return value;
105
+ };
106
+
107
+ export const trainingEpochs = (raw: string): number => {
108
+ const value = Number(raw);
109
+ if (!Number.isInteger(value) || value < 1 || value > 5) {
110
+ throw new InvalidArgumentError("Expected epochs from 1 through 5.");
111
+ }
112
+ return value;
113
+ };
114
+
115
+ export const trainingLearningRate = (raw: string): number => {
116
+ const value = Number(raw);
117
+ if (!Number.isFinite(value) || value <= 0) {
118
+ throw new InvalidArgumentError("Expected a positive learning rate, for example 0.00015.");
119
+ }
120
+ return value;
121
+ };
122
+
123
+ export const trainingBatchSize = (raw: string): 8 | 16 | 32 | 64 => {
124
+ const value = Number(raw);
125
+ if (value !== 8 && value !== 16 && value !== 32 && value !== 64) {
126
+ throw new InvalidArgumentError("Expected a batch size of 8, 16, 32, or 64.");
127
+ }
128
+ return value;
129
+ };
130
+
131
+ export const trainingMaxSequenceLength = (raw: string): number => {
132
+ const value = Number(raw);
133
+ if (!Number.isInteger(value) || value < 64 || value > 16384) {
134
+ throw new InvalidArgumentError("Expected a maximum sequence length from 64 through 16384 tokens.");
135
+ }
136
+ return value;
97
137
  };
98
138
 
99
139
  export type RawTrainingDraftOptions = {
@@ -104,9 +144,12 @@ export type RawTrainingDraftOptions = {
104
144
  validationDataset?: string;
105
145
  trainFilter?: string;
106
146
  validationFilter?: string;
107
- regime?: TrainingDraftOptions["regime"];
147
+ loraRank?: TrainingDraftOptions["loraRank"];
148
+ gistTokens?: TrainingDraftOptions["gistTokens"];
108
149
  epochs?: number;
109
- evalSteps?: number;
150
+ learningRate?: number;
151
+ batchSize?: TrainingDraftOptions["batchSize"];
152
+ maxSequenceLength?: number;
110
153
  promptFile?: string;
111
154
  };
112
155
 
@@ -118,9 +161,12 @@ export const trainingDraftOptions = (options: RawTrainingDraftOptions): Training
118
161
  validationDatasetId: options.validationDataset,
119
162
  trainingFilter: options.trainFilter,
120
163
  validationFilter: options.validationFilter,
121
- regime: options.regime,
164
+ loraRank: options.loraRank,
165
+ gistTokens: options.gistTokens,
122
166
  epochs: options.epochs,
123
- evalSteps: options.evalSteps,
167
+ learningRate: options.learningRate,
168
+ batchSize: options.batchSize,
169
+ maxSequenceLength: options.maxSequenceLength,
124
170
  promptFilePath: options.promptFile,
125
171
  });
126
172
 
@@ -163,14 +209,6 @@ export const identifyClientUser = async (
163
209
  } catch {}
164
210
  };
165
211
 
166
- export const resolveInternalUser = async (client: PotatoClient): Promise<boolean | undefined> => {
167
- try {
168
- return (await client.me()).internal;
169
- } catch {
170
- return undefined;
171
- }
172
- };
173
-
174
212
  export const resolveContext = async (
175
213
  globals: GlobalOptions,
176
214
  context: ProgramContext,
@@ -1,7 +1,8 @@
1
- import { assert } from "es-toolkit";
2
1
  import { chmod, lstat, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
3
2
  import { homedir } from "node:os";
4
3
  import { dirname, join } from "node:path";
4
+ import { assertCli, TolokaCliError } from "./cli-error";
5
+ import type { TolokaCliErrorOptions } from "./cli-error";
5
6
 
6
7
  export type StoredProjectAlias = {
7
8
  readonly id: string;
@@ -28,8 +29,8 @@ export type CommandCredentialLookup = {
28
29
  readonly warning?: string;
29
30
  };
30
31
 
31
- export class CredentialsError extends Error {
32
- constructor(message: string, options?: { readonly cause?: unknown }) {
32
+ export class CredentialsError extends TolokaCliError {
33
+ constructor(message: string, options: TolokaCliErrorOptions) {
33
34
  super(message, options);
34
35
  this.name = "CredentialsError";
35
36
  }
@@ -168,6 +169,7 @@ export async function findStoredCredentials(
168
169
  if (primary === "invalid") {
169
170
  throw new CredentialsError(
170
171
  `Invalid credentials file at ${path}. Run toloka logout, then toloka login.`,
172
+ { hint: { disabled: true } },
171
173
  );
172
174
  }
173
175
  return { credentials: primary.credentials, path, legacy: false };
@@ -181,6 +183,7 @@ export async function findStoredCredentials(
181
183
  if (legacy === "invalid") {
182
184
  throw new CredentialsError(
183
185
  `Invalid credentials file at ${legacyPath}. Run toloka logout, then toloka login.`,
186
+ { hint: { disabled: true } },
184
187
  );
185
188
  }
186
189
  return { credentials: legacy.credentials, path: legacyPath, legacy: true };
@@ -211,6 +214,7 @@ export async function requireStoredCredentials(
211
214
  if (found !== undefined) return found;
212
215
  throw new CredentialsError(
213
216
  `No stored credentials found at ${credentialsPath(environment)}. Run toloka login.`,
217
+ { hint: { disabled: true } },
214
218
  );
215
219
  }
216
220
 
@@ -228,7 +232,11 @@ const serializedCredentials = (credentials: StoredCredentials): StoredCredential
228
232
  const assertNotSymbolicLink = async (path: string): Promise<void> => {
229
233
  try {
230
234
  const entry = await lstat(path);
231
- assert(!entry.isSymbolicLink(), `Refusing to write through symbolic link ${path}.`);
235
+ assertCli(
236
+ !entry.isSymbolicLink(),
237
+ `Refusing to write through symbolic link ${path}.`,
238
+ { hint: { description: "Replace the symbolic link with a regular file or directory, then retry." } },
239
+ );
232
240
  } catch (error) {
233
241
  if (error instanceof Error && "code" in error && error.code === "ENOENT") return;
234
242
  throw error;
package/src/cli/data.ts CHANGED
@@ -1,4 +1,3 @@
1
- import { assert } from "es-toolkit";
2
1
  import { mkdir, readFile, writeFile } from "node:fs/promises";
3
2
  import { basename, dirname } from "node:path";
4
3
  import type { Dataset, InferredField, PotatoClient, UploadId } from "../client";
@@ -7,18 +6,20 @@ import { generateFieldDeclarationsSource } from "../importer";
7
6
  import { readTolokaState } from "../state";
8
7
  import type { WaitForOptions } from "../wait-for";
9
8
  import { desiredDatasets } from "./project";
10
- import { MISSING_PROJECT_ID_MESSAGE } from "./project-context";
9
+ import { assertProjectSelected } from "./project-context";
11
10
  import { loadProjectSource } from "./source";
12
11
  import type { ProjectSource } from "./source";
12
+ import { assertCli, REPORT_HINT, RETRY_OR_REPORT_HINT, TolokaCliError } from "./cli-error";
13
+ import type { TolokaCliErrorOptions } from "./cli-error";
13
14
 
14
15
  const RAW_UPLOAD_FIELD_NAME = "uploaded";
15
16
  const DATASET_ID_PATTERN =
16
17
  /^ds\.[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
17
18
  const FINALIZE_UPLOAD_ATTEMPTS = 3;
18
19
 
19
- class DatasetUploadError extends Error {
20
- constructor(message: string, cause: unknown) {
21
- super(message, { cause });
20
+ class DatasetUploadError extends TolokaCliError {
21
+ constructor(message: string, options: TolokaCliErrorOptions) {
22
+ super(message, options);
22
23
  this.name = "DatasetUploadError";
23
24
  }
24
25
  }
@@ -171,7 +172,7 @@ const resolveLiveDataset = async (
171
172
  ): Promise<DatasetRef> => {
172
173
  const state = await readTolokaState(options.sourceRoot);
173
174
  const projectId = options.projectId ?? state?.projectId;
174
- assert(projectId, MISSING_PROJECT_ID_MESSAGE);
175
+ assertProjectSelected(projectId);
175
176
  const token = options.datasetName;
176
177
  const live = await collectLiveDatasets(client, projectId);
177
178
  const nameIsUnique = (name: string) =>
@@ -186,13 +187,14 @@ const resolveLiveDataset = async (
186
187
  }
187
188
 
188
189
  const byName = live.filter((dataset) => dataset.name === token);
189
- assert(
190
+ assertCli(
190
191
  byName.length <= 1,
191
192
  `Project ${projectId} has multiple datasets named "${token}": ${byName.map(formatDatasetChoice).join(", ")}.`,
193
+ { hint: { description: "Pass the dataset id (ds.…) instead of the name." } },
192
194
  );
193
195
  if (byName.length === 1) {
194
196
  const [dataset] = byName;
195
- assert(dataset, `Dataset "${token}" has no resolved ID.`);
197
+ assertCli(dataset, `Dataset "${token}" has no resolved ID.`, { hint: REPORT_HINT });
196
198
  return { datasetId: dataset.id, name: dataset.name, nameIsUnique: true };
197
199
  }
198
200
 
@@ -204,16 +206,18 @@ const resolveLiveDataset = async (
204
206
  }
205
207
  }
206
208
 
207
- assert(
209
+ assertCli(
208
210
  createWhenMissing && datasetId === undefined,
209
211
  live.length === 0
210
- ? `Dataset "${token}" does not exist in project ${projectId}. Run toloka apply first.`
212
+ ? `Dataset "${token}" does not exist in project ${projectId}. Run toloka dataset create ${JSON.stringify(token)} first, or toloka apply for a dataset a pipeline declares.`
211
213
  : `Dataset "${token}" does not exist in project ${projectId}. This project has: ${live.map(formatDatasetChoice).join(", ")}.`,
214
+ { hint: { disabled: true } },
212
215
  );
213
216
 
214
- assert(
217
+ assertCli(
215
218
  (await declaredDatasetFields(options)).length > 0,
216
- `Dataset "${token}" does not exist in project ${projectId} and no pipeline in source declares it. Bind a pipeline to it in source so its columns are known, or upload to an existing dataset.`,
219
+ `Dataset "${token}" does not exist in project ${projectId} and no pipeline in source declares it. Run toloka dataset create ${JSON.stringify(token)} to create it empty, bind a pipeline to it in source so its columns are known, or upload to an existing dataset.`,
220
+ { hint: { disabled: true } },
217
221
  );
218
222
  const created = await client.datasets.create({ projectId, name: token });
219
223
  return { datasetId: created.id, name: created.name, nameIsUnique: true };
@@ -226,7 +230,11 @@ const resolveDatasetUploadFormat = (filename: string): DatasetUploadFormat => {
226
230
  if (normalizedFilename.endsWith(".jsonl")) return "jsonl";
227
231
  if (normalizedFilename.endsWith(".csv")) return "csv";
228
232
 
229
- assert(false, `${filename} must have a .json, .jsonl, or .csv extension`);
233
+ assertCli(
234
+ false,
235
+ `${filename} must have a .json, .jsonl, or .csv extension`,
236
+ { hint: { description: "Rename the file with the extension that matches its format." } },
237
+ );
230
238
  };
231
239
 
232
240
  const readDatasetUpload = async (filePath: string) => {
@@ -243,9 +251,17 @@ const readDatasetUpload = async (filePath: string) => {
243
251
  try {
244
252
  value = JSON.parse(line);
245
253
  } catch {
246
- assert(false, `${filename}:${lineNumber} is not valid JSON`);
254
+ assertCli(
255
+ false,
256
+ `${filename}:${lineNumber} is not valid JSON`,
257
+ { hint: { description: "Fix the JSON on that line; every line of a JSONL file must be one complete object." } },
258
+ );
247
259
  }
248
- assert(isJsonObject(value), `${filename}:${lineNumber} must contain a JSON object`);
260
+ assertCli(
261
+ isJsonObject(value),
262
+ `${filename}:${lineNumber} must contain a JSON object`,
263
+ { hint: { description: "Wrap the value in an object with named fields, one object per line." } },
264
+ );
249
265
  }
250
266
  }
251
267
 
@@ -265,16 +281,22 @@ const cancelStartedUpload = async (
265
281
  await client.datasets.cancelUpload({ uploadId });
266
282
  } catch (cancelError) {
267
283
  const cancelReason = cancelError instanceof Error ? cancelError.message : String(cancelError);
268
- throw new AggregateError(
269
- [error, cancelError],
284
+ throw new DatasetUploadError(
270
285
  `Dataset upload ${uploadId} failed and could not be canceled. Upload error: ${reason}. Cancellation error: ${cancelReason}.`,
286
+ {
287
+ cause: new AggregateError([error, cancelError]),
288
+ hint: {
289
+ description:
290
+ "Check toloka dataset list before re-running toloka dataset upload; an upload that never finalized is cleaned up automatically.",
291
+ },
292
+ },
271
293
  );
272
294
  }
273
295
 
274
- throw new DatasetUploadError(
275
- `Dataset upload ${uploadId} failed and was canceled: ${reason}`,
276
- error,
277
- );
296
+ throw new DatasetUploadError(`Dataset upload ${uploadId} failed and was canceled: ${reason}`, {
297
+ cause: error,
298
+ hint: { description: "Fix the upload error above, then re-run toloka dataset upload." },
299
+ });
278
300
  };
279
301
 
280
302
  const transferAndFinalizeUpload = async ({
@@ -310,10 +332,9 @@ const transferAndFinalizeUpload = async ({
310
332
  return cancelStartedUpload(
311
333
  client,
312
334
  started.job.id,
313
- new DatasetUploadError(
314
- `Dataset upload failed with HTTP ${uploadResponse.status}.`,
315
- undefined,
316
- ),
335
+ new DatasetUploadError(`Dataset upload failed with HTTP ${uploadResponse.status}.`, {
336
+ hint: RETRY_OR_REPORT_HINT,
337
+ }),
317
338
  );
318
339
  }
319
340
 
@@ -334,7 +355,7 @@ const transferAndFinalizeUpload = async ({
334
355
  started.job.id,
335
356
  new DatasetUploadError(
336
357
  `Storage remained unavailable after ${FINALIZE_UPLOAD_ATTEMPTS} finalization attempts.`,
337
- error,
358
+ { cause: error, hint: RETRY_OR_REPORT_HINT },
338
359
  ),
339
360
  );
340
361
  }
@@ -346,6 +367,28 @@ const transferAndFinalizeUpload = async ({
346
367
  }
347
368
  };
348
369
 
370
+ export type CreateDatasetResult = {
371
+ readonly datasetId: string;
372
+ readonly output: string;
373
+ };
374
+
375
+ export async function createDataset(
376
+ client: Pick<PotatoClient, "datasets">,
377
+ options: { readonly projectId: string; readonly name: string },
378
+ ): Promise<CreateDatasetResult> {
379
+ const name = options.name.trim();
380
+ assertCli(
381
+ name.length > 0,
382
+ "Dataset name cannot be empty.",
383
+ { hint: { description: "Pass a non-empty dataset name." } },
384
+ );
385
+ const created = await client.datasets.create({ projectId: options.projectId, name });
386
+ return {
387
+ datasetId: created.id,
388
+ output: `Created dataset ${created.name} (${created.id}). Upload rows with toloka dataset upload ${created.id} <file>.`,
389
+ };
390
+ }
391
+
349
392
  export async function uploadDataset(
350
393
  client: PotatoClient,
351
394
  options: UploadDatasetOptions,
@@ -388,9 +431,10 @@ export async function uploadDataset(
388
431
  completed.error?.startsWith("Schema incompatible with existing upload") === true
389
432
  ? `\nRun toloka dataset fields ${JSON.stringify(fieldsHint)} to inspect the inferred schema stored from the first successful upload.`
390
433
  : "";
391
- assert(
434
+ assertCli(
392
435
  completed.status === "succeeded",
393
436
  `${completed.error ?? `Dataset upload ${completed.id} failed.`}${compatibilityHint}`,
437
+ { hint: { disabled: true } },
394
438
  );
395
439
 
396
440
  const uploadedOutput = `Uploaded ${completed.result.itemsIngested} rows to dataset ${resolved.name}.`;
@@ -455,20 +499,32 @@ const downloadExportRows = async (options: {
455
499
  readonly fetch: typeof globalThis.fetch;
456
500
  }): Promise<Record<string, unknown>[]> => {
457
501
  const download = await options.fetch(options.downloadUrl);
458
- assert(download.ok, `Dataset export download failed with HTTP ${download.status}.`);
502
+ assertCli(
503
+ download.ok,
504
+ `Dataset export download failed with HTTP ${download.status}.`,
505
+ { hint: RETRY_OR_REPORT_HINT },
506
+ );
459
507
  const text = await download.text();
460
508
  let parsed: unknown;
461
509
  try {
462
510
  parsed = JSON.parse(text);
463
511
  } catch (error) {
464
- throw new Error(`Dataset export ${options.exportId} is not valid JSON.`, { cause: error });
512
+ throw new TolokaCliError(`Dataset export ${options.exportId} is not valid JSON.`, {
513
+ cause: error,
514
+ hint: REPORT_HINT,
515
+ });
465
516
  }
466
- assert(Array.isArray(parsed), `Dataset export ${options.exportId} is not a JSON array.`);
517
+ assertCli(
518
+ Array.isArray(parsed),
519
+ `Dataset export ${options.exportId} is not a JSON array.`,
520
+ { hint: REPORT_HINT },
521
+ );
467
522
  const rows: Record<string, unknown>[] = [];
468
523
  for (const [index, element] of parsed.entries()) {
469
- assert(
524
+ assertCli(
470
525
  isJsonObject(element),
471
526
  `Dataset export ${options.exportId} row ${index} is not a JSON object.`,
527
+ { hint: REPORT_HINT },
472
528
  );
473
529
  rows.push(element);
474
530
  }
@@ -485,7 +541,11 @@ export async function loadDataset(
485
541
  if (field.internal || field.name === RAW_UPLOAD_FIELD_NAME) continue;
486
542
  fieldNames.push(field.name);
487
543
  }
488
- assert(fieldNames.length > 0, `Dataset ${resolved.name} has no fields to export.`);
544
+ assertCli(
545
+ fieldNames.length > 0,
546
+ `Dataset ${resolved.name} has no fields to export.`,
547
+ { hint: { description: "Upload rows first with toloka dataset upload <name> <file>." } },
548
+ );
489
549
 
490
550
  const started = await client.datasets.startExport({
491
551
  datasetId: resolved.datasetId,
@@ -498,16 +558,18 @@ export async function loadDataset(
498
558
  options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs },
499
559
  );
500
560
  } catch (error) {
501
- assert(
561
+ assertCli(
502
562
  !(error instanceof PotatoApiError && error.code === "wait_timeout"),
503
563
  `Stopped waiting for dataset export ${started.id}; raise --timeout to wait longer.`,
564
+ { hint: { disabled: true } },
504
565
  );
505
566
  throw error;
506
567
  }
507
568
 
508
- assert(
569
+ assertCli(
509
570
  completed.result.downloadUrl,
510
571
  `Dataset export ${completed.id} succeeded without a download URL.`,
572
+ { hint: REPORT_HINT },
511
573
  );
512
574
 
513
575
  const rows = await downloadExportRows({
@@ -1,3 +1,6 @@
1
+ import { PotatoApiError } from "../errors";
2
+ import { TolokaCliError } from "./cli-error";
3
+
1
4
  export type DeliveryOrderRecord = {
2
5
  readonly id: string;
3
6
  readonly name: string;
@@ -15,7 +18,21 @@ export type DeliveryOrderListApi = {
15
18
  export async function listDeliveryOrders(options: {
16
19
  readonly client: DeliveryOrderListApi;
17
20
  }): Promise<{ readonly output: string }> {
18
- const orders = await options.client.deliveryOrders.list();
21
+ let orders: readonly DeliveryOrderRecord[];
22
+ try {
23
+ orders = await options.client.deliveryOrders.list();
24
+ } catch (error) {
25
+ if (error instanceof PotatoApiError && error.status === 403) {
26
+ throw new TolokaCliError("Delivery orders are available to internal teams only.", {
27
+ cause: error,
28
+ hint: {
29
+ description:
30
+ "Run toloka whoami; when it prints Internal: false, create the project without --delivery-order-id.",
31
+ },
32
+ });
33
+ }
34
+ throw error;
35
+ }
19
36
  if (orders.length === 0) {
20
37
  return { output: "No delivery orders visible." };
21
38
  }
@@ -1,6 +1,6 @@
1
- import { assert } from "es-toolkit";
2
1
  import { lstat } from "node:fs/promises";
3
2
  import { join } from "node:path";
3
+ import { assertCli } from "./cli-error";
4
4
 
5
5
  export const hasErrorCode = (error: unknown, ...codes: readonly string[]) =>
6
6
  error instanceof Error &&
@@ -29,6 +29,10 @@ export const assertSafeDestination = async (
29
29
  currentPath = join(currentPath, segment);
30
30
  const entry = await entryStats(currentPath);
31
31
  if (entry === undefined) return;
32
- assert(!entry.isSymbolicLink(), `Refusing to ${action} through symbolic link ${currentPath}.`);
32
+ assertCli(
33
+ !entry.isSymbolicLink(),
34
+ `Refusing to ${action} through symbolic link ${currentPath}.`,
35
+ { hint: { description: "Replace the symbolic link with a regular directory, or point the command at the real path." } },
36
+ );
33
37
  }
34
38
  };