@toloka-platform/sdk 0.2.0-preview.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -6
- package/dist/cli/toloka.js +136 -136
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/cli-v2.md +3 -2
- package/docs/cli.md +35 -7
- package/docs/getting-started.md +4 -2
- package/docs/limitations.md +2 -2
- package/openapi/gen/models/baseModel.ts +1 -0
- package/openapi/gen/models/currentUser.ts +2 -0
- package/openapi/gen/models/currentUserTeamsItem.ts +6 -0
- package/openapi/gen/models/gistingTrainingConfiguration.ts +6 -2
- package/openapi/gen/models/gistingTrainingConfigurationBatchSize.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationGistTokens.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationInput.ts +6 -2
- package/openapi/gen/models/gistingTrainingConfigurationInputBatchSize.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationInputGistTokens.ts +2 -0
- package/openapi/gen/models/index.ts +9 -4
- package/openapi/gen/models/sftTrainingConfiguration.ts +6 -3
- package/openapi/gen/models/sftTrainingConfigurationBatchSize.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationInput.ts +6 -3
- package/openapi/gen/models/sftTrainingConfigurationInputBatchSize.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationInputLoraRank.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationLoraRank.ts +2 -0
- package/package.json +2 -2
- package/skills/catalog.json +11 -11
- package/skills/toloka-build/SKILL.md +29 -18
- package/skills/toloka-build/references/node-semantics.md +7 -7
- package/skills/toloka-build/references/workflow-and-cli.md +15 -12
- package/skills/toloka-fine-tuning/SKILL.md +145 -62
- package/skills/toloka-get-started/SKILL.md +37 -21
- package/skills/toloka-gisting/SKILL.md +156 -66
- package/skills/toloka-report-issue/references/report-template.md +1 -1
- package/src/cli/analytics.ts +50 -23
- package/src/cli/auth.ts +31 -12
- package/src/cli/auto-update.ts +12 -6
- package/src/cli/check.ts +76 -29
- package/src/cli/cli-entry.ts +3 -2
- package/src/cli/cli-error.ts +43 -0
- package/src/cli/commands/dataset.ts +13 -3
- package/src/cli/commands/models.ts +6 -7
- package/src/cli/commands/pipeline.ts +6 -7
- package/src/cli/commands/project.ts +9 -16
- package/src/cli/commands/report.ts +6 -2
- package/src/cli/commands/secret.ts +5 -4
- package/src/cli/commands/source.ts +4 -4
- package/src/cli/commands/status.ts +2 -3
- package/src/cli/commands/training.ts +50 -17
- package/src/cli/config.ts +2 -2
- package/src/cli/context.ts +55 -17
- package/src/cli/credentials.ts +12 -4
- package/src/cli/data.ts +96 -34
- package/src/cli/delivery-orders.ts +18 -1
- package/src/cli/file-system.ts +6 -2
- package/src/cli/format-error.ts +45 -2
- package/src/cli/init.ts +13 -7
- package/src/cli/models.ts +43 -12
- package/src/cli/program.ts +5 -56
- package/src/cli/project-context.ts +117 -32
- package/src/cli/project.ts +80 -28
- package/src/cli/report.ts +24 -9
- package/src/cli/runs.ts +33 -8
- package/src/cli/settings.ts +13 -5
- package/src/cli/skill.ts +61 -22
- package/src/cli/source.ts +3 -2
- package/src/cli/terminal-io.ts +13 -6
- package/src/cli/trainings.ts +129 -35
- package/src/cli/version.ts +2 -2
- package/src/client.ts +3 -9
- package/src/transport.ts +13 -1
- package/openapi/gen/models/gistingTrainingConfigurationInputRegime.ts +0 -2
- package/openapi/gen/models/gistingTrainingConfigurationRegime.ts +0 -2
- package/openapi/gen/models/sftTrainingConfigurationInputRegime.ts +0 -2
- package/openapi/gen/models/sftTrainingConfigurationRegime.ts +0 -2
|
@@ -1,17 +1,20 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import type { Command } from "commander";
|
|
3
2
|
import {
|
|
4
|
-
nonNegativeInteger,
|
|
5
3
|
positiveInteger,
|
|
6
4
|
resolveContext,
|
|
5
|
+
trainingBatchSize,
|
|
7
6
|
trainingDraftOptions,
|
|
7
|
+
trainingEpochs,
|
|
8
|
+
trainingGistTokens,
|
|
9
|
+
trainingLearningRate,
|
|
10
|
+
trainingLoraRank,
|
|
11
|
+
trainingMaxSequenceLength,
|
|
8
12
|
trainingMethod,
|
|
9
|
-
trainingRegime,
|
|
10
13
|
type CommandHost,
|
|
11
14
|
type RawTrainingDraftOptions,
|
|
12
15
|
} from "../context";
|
|
13
16
|
import { parseBaseModelId } from "../models";
|
|
14
|
-
import {
|
|
17
|
+
import { assertProjectSelected } from "../project-context";
|
|
15
18
|
import {
|
|
16
19
|
cancelTrainingRun,
|
|
17
20
|
createTraining,
|
|
@@ -37,7 +40,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
37
40
|
.description("list active Trainings owned by the selected project")
|
|
38
41
|
.action(async (options: { json: boolean }) => {
|
|
39
42
|
const { client, projectId } = await resolveContext(globals(), context);
|
|
40
|
-
|
|
43
|
+
assertProjectSelected(projectId);
|
|
41
44
|
await listTrainings(client, { projectId, json: options.json }, context.io);
|
|
42
45
|
});
|
|
43
46
|
training
|
|
@@ -47,7 +50,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
47
50
|
.description("show one Training draft")
|
|
48
51
|
.action(async (nameOrId: string, options: { json: boolean }) => {
|
|
49
52
|
const { client, projectId } = await resolveContext(globals(), context);
|
|
50
|
-
|
|
53
|
+
assertProjectSelected(projectId);
|
|
51
54
|
await getTraining(client, { projectId, nameOrId, json: options.json }, context.io);
|
|
52
55
|
});
|
|
53
56
|
training
|
|
@@ -60,15 +63,30 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
60
63
|
.option("--validation-dataset <id>", "optional validation Dataset ID")
|
|
61
64
|
.option("--train-filter <field=value>", "training Dataset field equality filter")
|
|
62
65
|
.option("--validation-filter <field=value>", "validation Dataset field equality filter")
|
|
63
|
-
.option("--
|
|
64
|
-
.option("--
|
|
65
|
-
.option("--
|
|
66
|
+
.option("--lora-rank <rank>", "SFT LoRA adapter rank: 8, 16, or 32 (default 16)", trainingLoraRank)
|
|
67
|
+
.option("--gist-tokens <count>", "gisting soft tokens: 8, 32, or 64 (default 32)", trainingGistTokens)
|
|
68
|
+
.option("--epochs <count>", "training epochs, 1 through 5 (default 3)", trainingEpochs)
|
|
69
|
+
.option(
|
|
70
|
+
"--learning-rate <rate>",
|
|
71
|
+
"optimizer learning rate (default 0.00015 for SFT, 0.03 for gisting)",
|
|
72
|
+
trainingLearningRate,
|
|
73
|
+
)
|
|
74
|
+
.option(
|
|
75
|
+
"--batch-size <count>",
|
|
76
|
+
"sequences per optimizer step: 8, 16, 32, or 64 (default 8)",
|
|
77
|
+
trainingBatchSize,
|
|
78
|
+
)
|
|
79
|
+
.option(
|
|
80
|
+
"--max-sequence-length <tokens>",
|
|
81
|
+
"token budget per example, 64 through 16384 (default 12288 for SFT, 3072 for gisting)",
|
|
82
|
+
trainingMaxSequenceLength,
|
|
83
|
+
)
|
|
66
84
|
.option("--prompt-file <path>", "gisting prompt text file")
|
|
67
85
|
.option("--json", "print JSON", false)
|
|
68
86
|
.description("create a normalized Training draft")
|
|
69
87
|
.action(async (name: string, options: RawTrainingDraftOptions & { json: boolean }) => {
|
|
70
88
|
const { client, projectId } = await resolveContext(globals(), context);
|
|
71
|
-
|
|
89
|
+
assertProjectSelected(projectId);
|
|
72
90
|
await createTraining(
|
|
73
91
|
client,
|
|
74
92
|
{
|
|
@@ -92,15 +110,30 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
92
110
|
"--validation-filter <field=value>",
|
|
93
111
|
"replacement validation Dataset field equality filter",
|
|
94
112
|
)
|
|
95
|
-
.option("--
|
|
96
|
-
.option("--
|
|
97
|
-
.option("--
|
|
113
|
+
.option("--lora-rank <rank>", "SFT LoRA adapter rank: 8, 16, or 32 (default 16)", trainingLoraRank)
|
|
114
|
+
.option("--gist-tokens <count>", "gisting soft tokens: 8, 32, or 64 (default 32)", trainingGistTokens)
|
|
115
|
+
.option("--epochs <count>", "training epochs, 1 through 5 (default 3)", trainingEpochs)
|
|
116
|
+
.option(
|
|
117
|
+
"--learning-rate <rate>",
|
|
118
|
+
"optimizer learning rate (default 0.00015 for SFT, 0.03 for gisting)",
|
|
119
|
+
trainingLearningRate,
|
|
120
|
+
)
|
|
121
|
+
.option(
|
|
122
|
+
"--batch-size <count>",
|
|
123
|
+
"sequences per optimizer step: 8, 16, 32, or 64 (default 8)",
|
|
124
|
+
trainingBatchSize,
|
|
125
|
+
)
|
|
126
|
+
.option(
|
|
127
|
+
"--max-sequence-length <tokens>",
|
|
128
|
+
"token budget per example, 64 through 16384 (default 12288 for SFT, 3072 for gisting)",
|
|
129
|
+
trainingMaxSequenceLength,
|
|
130
|
+
)
|
|
98
131
|
.option("--prompt-file <path>", "replacement gisting prompt text file")
|
|
99
132
|
.option("--json", "print JSON", false)
|
|
100
133
|
.description("replace the complete Training draft")
|
|
101
134
|
.action(async (nameOrId: string, options: RawTrainingDraftOptions & { json: boolean }) => {
|
|
102
135
|
const { client, projectId } = await resolveContext(globals(), context);
|
|
103
|
-
|
|
136
|
+
assertProjectSelected(projectId);
|
|
104
137
|
await updateTraining(
|
|
105
138
|
client,
|
|
106
139
|
{
|
|
@@ -120,7 +153,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
120
153
|
.description("validate Training inputs and calculate the maximum charge")
|
|
121
154
|
.action(async (nameOrId: string, options: { idempotencyKey?: string; json: boolean }) => {
|
|
122
155
|
const { client, projectId } = await resolveContext(globals(), context);
|
|
123
|
-
|
|
156
|
+
assertProjectSelected(projectId);
|
|
124
157
|
await estimateTrainingCost(
|
|
125
158
|
client,
|
|
126
159
|
{ projectId, nameOrId, idempotencyKey: options.idempotencyKey, json: options.json },
|
|
@@ -146,7 +179,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
146
179
|
},
|
|
147
180
|
) => {
|
|
148
181
|
const { client, projectId } = await resolveContext(globals(), context);
|
|
149
|
-
|
|
182
|
+
assertProjectSelected(projectId);
|
|
150
183
|
await createTrainingRun(client, { projectId, nameOrId, ...options }, context.io);
|
|
151
184
|
},
|
|
152
185
|
);
|
|
@@ -157,7 +190,7 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
157
190
|
.description("list Training Run history")
|
|
158
191
|
.action(async (nameOrId: string, options: { json: boolean }) => {
|
|
159
192
|
const { client, projectId } = await resolveContext(globals(), context);
|
|
160
|
-
|
|
193
|
+
assertProjectSelected(projectId);
|
|
161
194
|
await listTrainingRuns(client, { projectId, nameOrId, json: options.json }, context.io);
|
|
162
195
|
});
|
|
163
196
|
training
|
package/src/cli/config.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import type { TolokaState } from "../state";
|
|
3
2
|
import type { StoredCredentials } from "./credentials";
|
|
4
3
|
import { resolveProjectContext } from "./project-context";
|
|
4
|
+
import { assertCli } from "./cli-error";
|
|
5
5
|
|
|
6
6
|
export type TolokaCliFlags = {
|
|
7
7
|
readonly apiKey?: string;
|
|
@@ -76,6 +76,6 @@ export function resolveTolokaConfig(
|
|
|
76
76
|
stored,
|
|
77
77
|
effectiveBaseUrl: baseUrl,
|
|
78
78
|
});
|
|
79
|
-
|
|
79
|
+
assertCli(apiKey, missingApiKeyMessage, { hint: { disabled: true } });
|
|
80
80
|
return { apiKey, baseUrl, ...(project === undefined ? {} : { projectId: project.id }) };
|
|
81
81
|
}
|
package/src/cli/context.ts
CHANGED
|
@@ -30,7 +30,6 @@ export type ProgramContext = {
|
|
|
30
30
|
readonly injectedClient?: PotatoClient;
|
|
31
31
|
readonly runChecks?: typeof checkProject;
|
|
32
32
|
readonly analytics?: CliAnalytics;
|
|
33
|
-
readonly internalUser?: boolean;
|
|
34
33
|
};
|
|
35
34
|
|
|
36
35
|
export type GlobalOptions = {
|
|
@@ -89,11 +88,52 @@ export const trainingMethod = (raw: string): "sft" | "gisting" => {
|
|
|
89
88
|
return raw;
|
|
90
89
|
};
|
|
91
90
|
|
|
92
|
-
export const
|
|
93
|
-
|
|
94
|
-
|
|
91
|
+
export const trainingLoraRank = (raw: string): 8 | 16 | 32 => {
|
|
92
|
+
const value = Number(raw);
|
|
93
|
+
if (value !== 8 && value !== 16 && value !== 32) {
|
|
94
|
+
throw new InvalidArgumentError("Expected a LoRA rank of 8, 16, or 32.");
|
|
95
95
|
}
|
|
96
|
-
return
|
|
96
|
+
return value;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
export const trainingGistTokens = (raw: string): 8 | 32 | 64 => {
|
|
100
|
+
const value = Number(raw);
|
|
101
|
+
if (value !== 8 && value !== 32 && value !== 64) {
|
|
102
|
+
throw new InvalidArgumentError("Expected 8, 32, or 64 gist tokens.");
|
|
103
|
+
}
|
|
104
|
+
return value;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
export const trainingEpochs = (raw: string): number => {
|
|
108
|
+
const value = Number(raw);
|
|
109
|
+
if (!Number.isInteger(value) || value < 1 || value > 5) {
|
|
110
|
+
throw new InvalidArgumentError("Expected epochs from 1 through 5.");
|
|
111
|
+
}
|
|
112
|
+
return value;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
export const trainingLearningRate = (raw: string): number => {
|
|
116
|
+
const value = Number(raw);
|
|
117
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
118
|
+
throw new InvalidArgumentError("Expected a positive learning rate, for example 0.00015.");
|
|
119
|
+
}
|
|
120
|
+
return value;
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
export const trainingBatchSize = (raw: string): 8 | 16 | 32 | 64 => {
|
|
124
|
+
const value = Number(raw);
|
|
125
|
+
if (value !== 8 && value !== 16 && value !== 32 && value !== 64) {
|
|
126
|
+
throw new InvalidArgumentError("Expected a batch size of 8, 16, 32, or 64.");
|
|
127
|
+
}
|
|
128
|
+
return value;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
export const trainingMaxSequenceLength = (raw: string): number => {
|
|
132
|
+
const value = Number(raw);
|
|
133
|
+
if (!Number.isInteger(value) || value < 64 || value > 16384) {
|
|
134
|
+
throw new InvalidArgumentError("Expected a maximum sequence length from 64 through 16384 tokens.");
|
|
135
|
+
}
|
|
136
|
+
return value;
|
|
97
137
|
};
|
|
98
138
|
|
|
99
139
|
export type RawTrainingDraftOptions = {
|
|
@@ -104,9 +144,12 @@ export type RawTrainingDraftOptions = {
|
|
|
104
144
|
validationDataset?: string;
|
|
105
145
|
trainFilter?: string;
|
|
106
146
|
validationFilter?: string;
|
|
107
|
-
|
|
147
|
+
loraRank?: TrainingDraftOptions["loraRank"];
|
|
148
|
+
gistTokens?: TrainingDraftOptions["gistTokens"];
|
|
108
149
|
epochs?: number;
|
|
109
|
-
|
|
150
|
+
learningRate?: number;
|
|
151
|
+
batchSize?: TrainingDraftOptions["batchSize"];
|
|
152
|
+
maxSequenceLength?: number;
|
|
110
153
|
promptFile?: string;
|
|
111
154
|
};
|
|
112
155
|
|
|
@@ -118,9 +161,12 @@ export const trainingDraftOptions = (options: RawTrainingDraftOptions): Training
|
|
|
118
161
|
validationDatasetId: options.validationDataset,
|
|
119
162
|
trainingFilter: options.trainFilter,
|
|
120
163
|
validationFilter: options.validationFilter,
|
|
121
|
-
|
|
164
|
+
loraRank: options.loraRank,
|
|
165
|
+
gistTokens: options.gistTokens,
|
|
122
166
|
epochs: options.epochs,
|
|
123
|
-
|
|
167
|
+
learningRate: options.learningRate,
|
|
168
|
+
batchSize: options.batchSize,
|
|
169
|
+
maxSequenceLength: options.maxSequenceLength,
|
|
124
170
|
promptFilePath: options.promptFile,
|
|
125
171
|
});
|
|
126
172
|
|
|
@@ -163,14 +209,6 @@ export const identifyClientUser = async (
|
|
|
163
209
|
} catch {}
|
|
164
210
|
};
|
|
165
211
|
|
|
166
|
-
export const resolveInternalUser = async (client: PotatoClient): Promise<boolean | undefined> => {
|
|
167
|
-
try {
|
|
168
|
-
return (await client.me()).internal;
|
|
169
|
-
} catch {
|
|
170
|
-
return undefined;
|
|
171
|
-
}
|
|
172
|
-
};
|
|
173
|
-
|
|
174
212
|
export const resolveContext = async (
|
|
175
213
|
globals: GlobalOptions,
|
|
176
214
|
context: ProgramContext,
|
package/src/cli/credentials.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import { chmod, lstat, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
3
2
|
import { homedir } from "node:os";
|
|
4
3
|
import { dirname, join } from "node:path";
|
|
4
|
+
import { assertCli, TolokaCliError } from "./cli-error";
|
|
5
|
+
import type { TolokaCliErrorOptions } from "./cli-error";
|
|
5
6
|
|
|
6
7
|
export type StoredProjectAlias = {
|
|
7
8
|
readonly id: string;
|
|
@@ -28,8 +29,8 @@ export type CommandCredentialLookup = {
|
|
|
28
29
|
readonly warning?: string;
|
|
29
30
|
};
|
|
30
31
|
|
|
31
|
-
export class CredentialsError extends
|
|
32
|
-
constructor(message: string, options
|
|
32
|
+
export class CredentialsError extends TolokaCliError {
|
|
33
|
+
constructor(message: string, options: TolokaCliErrorOptions) {
|
|
33
34
|
super(message, options);
|
|
34
35
|
this.name = "CredentialsError";
|
|
35
36
|
}
|
|
@@ -168,6 +169,7 @@ export async function findStoredCredentials(
|
|
|
168
169
|
if (primary === "invalid") {
|
|
169
170
|
throw new CredentialsError(
|
|
170
171
|
`Invalid credentials file at ${path}. Run toloka logout, then toloka login.`,
|
|
172
|
+
{ hint: { disabled: true } },
|
|
171
173
|
);
|
|
172
174
|
}
|
|
173
175
|
return { credentials: primary.credentials, path, legacy: false };
|
|
@@ -181,6 +183,7 @@ export async function findStoredCredentials(
|
|
|
181
183
|
if (legacy === "invalid") {
|
|
182
184
|
throw new CredentialsError(
|
|
183
185
|
`Invalid credentials file at ${legacyPath}. Run toloka logout, then toloka login.`,
|
|
186
|
+
{ hint: { disabled: true } },
|
|
184
187
|
);
|
|
185
188
|
}
|
|
186
189
|
return { credentials: legacy.credentials, path: legacyPath, legacy: true };
|
|
@@ -211,6 +214,7 @@ export async function requireStoredCredentials(
|
|
|
211
214
|
if (found !== undefined) return found;
|
|
212
215
|
throw new CredentialsError(
|
|
213
216
|
`No stored credentials found at ${credentialsPath(environment)}. Run toloka login.`,
|
|
217
|
+
{ hint: { disabled: true } },
|
|
214
218
|
);
|
|
215
219
|
}
|
|
216
220
|
|
|
@@ -228,7 +232,11 @@ const serializedCredentials = (credentials: StoredCredentials): StoredCredential
|
|
|
228
232
|
const assertNotSymbolicLink = async (path: string): Promise<void> => {
|
|
229
233
|
try {
|
|
230
234
|
const entry = await lstat(path);
|
|
231
|
-
|
|
235
|
+
assertCli(
|
|
236
|
+
!entry.isSymbolicLink(),
|
|
237
|
+
`Refusing to write through symbolic link ${path}.`,
|
|
238
|
+
{ hint: { description: "Replace the symbolic link with a regular file or directory, then retry." } },
|
|
239
|
+
);
|
|
232
240
|
} catch (error) {
|
|
233
241
|
if (error instanceof Error && "code" in error && error.code === "ENOENT") return;
|
|
234
242
|
throw error;
|
package/src/cli/data.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
2
|
import { basename, dirname } from "node:path";
|
|
4
3
|
import type { Dataset, InferredField, PotatoClient, UploadId } from "../client";
|
|
@@ -7,18 +6,20 @@ import { generateFieldDeclarationsSource } from "../importer";
|
|
|
7
6
|
import { readTolokaState } from "../state";
|
|
8
7
|
import type { WaitForOptions } from "../wait-for";
|
|
9
8
|
import { desiredDatasets } from "./project";
|
|
10
|
-
import {
|
|
9
|
+
import { assertProjectSelected } from "./project-context";
|
|
11
10
|
import { loadProjectSource } from "./source";
|
|
12
11
|
import type { ProjectSource } from "./source";
|
|
12
|
+
import { assertCli, REPORT_HINT, RETRY_OR_REPORT_HINT, TolokaCliError } from "./cli-error";
|
|
13
|
+
import type { TolokaCliErrorOptions } from "./cli-error";
|
|
13
14
|
|
|
14
15
|
const RAW_UPLOAD_FIELD_NAME = "uploaded";
|
|
15
16
|
const DATASET_ID_PATTERN =
|
|
16
17
|
/^ds\.[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
17
18
|
const FINALIZE_UPLOAD_ATTEMPTS = 3;
|
|
18
19
|
|
|
19
|
-
class DatasetUploadError extends
|
|
20
|
-
constructor(message: string,
|
|
21
|
-
super(message,
|
|
20
|
+
class DatasetUploadError extends TolokaCliError {
|
|
21
|
+
constructor(message: string, options: TolokaCliErrorOptions) {
|
|
22
|
+
super(message, options);
|
|
22
23
|
this.name = "DatasetUploadError";
|
|
23
24
|
}
|
|
24
25
|
}
|
|
@@ -171,7 +172,7 @@ const resolveLiveDataset = async (
|
|
|
171
172
|
): Promise<DatasetRef> => {
|
|
172
173
|
const state = await readTolokaState(options.sourceRoot);
|
|
173
174
|
const projectId = options.projectId ?? state?.projectId;
|
|
174
|
-
|
|
175
|
+
assertProjectSelected(projectId);
|
|
175
176
|
const token = options.datasetName;
|
|
176
177
|
const live = await collectLiveDatasets(client, projectId);
|
|
177
178
|
const nameIsUnique = (name: string) =>
|
|
@@ -186,13 +187,14 @@ const resolveLiveDataset = async (
|
|
|
186
187
|
}
|
|
187
188
|
|
|
188
189
|
const byName = live.filter((dataset) => dataset.name === token);
|
|
189
|
-
|
|
190
|
+
assertCli(
|
|
190
191
|
byName.length <= 1,
|
|
191
192
|
`Project ${projectId} has multiple datasets named "${token}": ${byName.map(formatDatasetChoice).join(", ")}.`,
|
|
193
|
+
{ hint: { description: "Pass the dataset id (ds.…) instead of the name." } },
|
|
192
194
|
);
|
|
193
195
|
if (byName.length === 1) {
|
|
194
196
|
const [dataset] = byName;
|
|
195
|
-
|
|
197
|
+
assertCli(dataset, `Dataset "${token}" has no resolved ID.`, { hint: REPORT_HINT });
|
|
196
198
|
return { datasetId: dataset.id, name: dataset.name, nameIsUnique: true };
|
|
197
199
|
}
|
|
198
200
|
|
|
@@ -204,16 +206,18 @@ const resolveLiveDataset = async (
|
|
|
204
206
|
}
|
|
205
207
|
}
|
|
206
208
|
|
|
207
|
-
|
|
209
|
+
assertCli(
|
|
208
210
|
createWhenMissing && datasetId === undefined,
|
|
209
211
|
live.length === 0
|
|
210
|
-
? `Dataset "${token}" does not exist in project ${projectId}. Run toloka apply
|
|
212
|
+
? `Dataset "${token}" does not exist in project ${projectId}. Run toloka dataset create ${JSON.stringify(token)} first, or toloka apply for a dataset a pipeline declares.`
|
|
211
213
|
: `Dataset "${token}" does not exist in project ${projectId}. This project has: ${live.map(formatDatasetChoice).join(", ")}.`,
|
|
214
|
+
{ hint: { disabled: true } },
|
|
212
215
|
);
|
|
213
216
|
|
|
214
|
-
|
|
217
|
+
assertCli(
|
|
215
218
|
(await declaredDatasetFields(options)).length > 0,
|
|
216
|
-
`Dataset "${token}" does not exist in project ${projectId} and no pipeline in source declares it.
|
|
219
|
+
`Dataset "${token}" does not exist in project ${projectId} and no pipeline in source declares it. Run toloka dataset create ${JSON.stringify(token)} to create it empty, bind a pipeline to it in source so its columns are known, or upload to an existing dataset.`,
|
|
220
|
+
{ hint: { disabled: true } },
|
|
217
221
|
);
|
|
218
222
|
const created = await client.datasets.create({ projectId, name: token });
|
|
219
223
|
return { datasetId: created.id, name: created.name, nameIsUnique: true };
|
|
@@ -226,7 +230,11 @@ const resolveDatasetUploadFormat = (filename: string): DatasetUploadFormat => {
|
|
|
226
230
|
if (normalizedFilename.endsWith(".jsonl")) return "jsonl";
|
|
227
231
|
if (normalizedFilename.endsWith(".csv")) return "csv";
|
|
228
232
|
|
|
229
|
-
|
|
233
|
+
assertCli(
|
|
234
|
+
false,
|
|
235
|
+
`${filename} must have a .json, .jsonl, or .csv extension`,
|
|
236
|
+
{ hint: { description: "Rename the file with the extension that matches its format." } },
|
|
237
|
+
);
|
|
230
238
|
};
|
|
231
239
|
|
|
232
240
|
const readDatasetUpload = async (filePath: string) => {
|
|
@@ -243,9 +251,17 @@ const readDatasetUpload = async (filePath: string) => {
|
|
|
243
251
|
try {
|
|
244
252
|
value = JSON.parse(line);
|
|
245
253
|
} catch {
|
|
246
|
-
|
|
254
|
+
assertCli(
|
|
255
|
+
false,
|
|
256
|
+
`${filename}:${lineNumber} is not valid JSON`,
|
|
257
|
+
{ hint: { description: "Fix the JSON on that line; every line of a JSONL file must be one complete object." } },
|
|
258
|
+
);
|
|
247
259
|
}
|
|
248
|
-
|
|
260
|
+
assertCli(
|
|
261
|
+
isJsonObject(value),
|
|
262
|
+
`${filename}:${lineNumber} must contain a JSON object`,
|
|
263
|
+
{ hint: { description: "Wrap the value in an object with named fields, one object per line." } },
|
|
264
|
+
);
|
|
249
265
|
}
|
|
250
266
|
}
|
|
251
267
|
|
|
@@ -265,16 +281,22 @@ const cancelStartedUpload = async (
|
|
|
265
281
|
await client.datasets.cancelUpload({ uploadId });
|
|
266
282
|
} catch (cancelError) {
|
|
267
283
|
const cancelReason = cancelError instanceof Error ? cancelError.message : String(cancelError);
|
|
268
|
-
throw new
|
|
269
|
-
[error, cancelError],
|
|
284
|
+
throw new DatasetUploadError(
|
|
270
285
|
`Dataset upload ${uploadId} failed and could not be canceled. Upload error: ${reason}. Cancellation error: ${cancelReason}.`,
|
|
286
|
+
{
|
|
287
|
+
cause: new AggregateError([error, cancelError]),
|
|
288
|
+
hint: {
|
|
289
|
+
description:
|
|
290
|
+
"Check toloka dataset list before re-running toloka dataset upload; an upload that never finalized is cleaned up automatically.",
|
|
291
|
+
},
|
|
292
|
+
},
|
|
271
293
|
);
|
|
272
294
|
}
|
|
273
295
|
|
|
274
|
-
throw new DatasetUploadError(
|
|
275
|
-
|
|
276
|
-
error,
|
|
277
|
-
);
|
|
296
|
+
throw new DatasetUploadError(`Dataset upload ${uploadId} failed and was canceled: ${reason}`, {
|
|
297
|
+
cause: error,
|
|
298
|
+
hint: { description: "Fix the upload error above, then re-run toloka dataset upload." },
|
|
299
|
+
});
|
|
278
300
|
};
|
|
279
301
|
|
|
280
302
|
const transferAndFinalizeUpload = async ({
|
|
@@ -310,10 +332,9 @@ const transferAndFinalizeUpload = async ({
|
|
|
310
332
|
return cancelStartedUpload(
|
|
311
333
|
client,
|
|
312
334
|
started.job.id,
|
|
313
|
-
new DatasetUploadError(
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
),
|
|
335
|
+
new DatasetUploadError(`Dataset upload failed with HTTP ${uploadResponse.status}.`, {
|
|
336
|
+
hint: RETRY_OR_REPORT_HINT,
|
|
337
|
+
}),
|
|
317
338
|
);
|
|
318
339
|
}
|
|
319
340
|
|
|
@@ -334,7 +355,7 @@ const transferAndFinalizeUpload = async ({
|
|
|
334
355
|
started.job.id,
|
|
335
356
|
new DatasetUploadError(
|
|
336
357
|
`Storage remained unavailable after ${FINALIZE_UPLOAD_ATTEMPTS} finalization attempts.`,
|
|
337
|
-
error,
|
|
358
|
+
{ cause: error, hint: RETRY_OR_REPORT_HINT },
|
|
338
359
|
),
|
|
339
360
|
);
|
|
340
361
|
}
|
|
@@ -346,6 +367,28 @@ const transferAndFinalizeUpload = async ({
|
|
|
346
367
|
}
|
|
347
368
|
};
|
|
348
369
|
|
|
370
|
+
export type CreateDatasetResult = {
|
|
371
|
+
readonly datasetId: string;
|
|
372
|
+
readonly output: string;
|
|
373
|
+
};
|
|
374
|
+
|
|
375
|
+
export async function createDataset(
|
|
376
|
+
client: Pick<PotatoClient, "datasets">,
|
|
377
|
+
options: { readonly projectId: string; readonly name: string },
|
|
378
|
+
): Promise<CreateDatasetResult> {
|
|
379
|
+
const name = options.name.trim();
|
|
380
|
+
assertCli(
|
|
381
|
+
name.length > 0,
|
|
382
|
+
"Dataset name cannot be empty.",
|
|
383
|
+
{ hint: { description: "Pass a non-empty dataset name." } },
|
|
384
|
+
);
|
|
385
|
+
const created = await client.datasets.create({ projectId: options.projectId, name });
|
|
386
|
+
return {
|
|
387
|
+
datasetId: created.id,
|
|
388
|
+
output: `Created dataset ${created.name} (${created.id}). Upload rows with toloka dataset upload ${created.id} <file>.`,
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
|
|
349
392
|
export async function uploadDataset(
|
|
350
393
|
client: PotatoClient,
|
|
351
394
|
options: UploadDatasetOptions,
|
|
@@ -388,9 +431,10 @@ export async function uploadDataset(
|
|
|
388
431
|
completed.error?.startsWith("Schema incompatible with existing upload") === true
|
|
389
432
|
? `\nRun toloka dataset fields ${JSON.stringify(fieldsHint)} to inspect the inferred schema stored from the first successful upload.`
|
|
390
433
|
: "";
|
|
391
|
-
|
|
434
|
+
assertCli(
|
|
392
435
|
completed.status === "succeeded",
|
|
393
436
|
`${completed.error ?? `Dataset upload ${completed.id} failed.`}${compatibilityHint}`,
|
|
437
|
+
{ hint: { disabled: true } },
|
|
394
438
|
);
|
|
395
439
|
|
|
396
440
|
const uploadedOutput = `Uploaded ${completed.result.itemsIngested} rows to dataset ${resolved.name}.`;
|
|
@@ -455,20 +499,32 @@ const downloadExportRows = async (options: {
|
|
|
455
499
|
readonly fetch: typeof globalThis.fetch;
|
|
456
500
|
}): Promise<Record<string, unknown>[]> => {
|
|
457
501
|
const download = await options.fetch(options.downloadUrl);
|
|
458
|
-
|
|
502
|
+
assertCli(
|
|
503
|
+
download.ok,
|
|
504
|
+
`Dataset export download failed with HTTP ${download.status}.`,
|
|
505
|
+
{ hint: RETRY_OR_REPORT_HINT },
|
|
506
|
+
);
|
|
459
507
|
const text = await download.text();
|
|
460
508
|
let parsed: unknown;
|
|
461
509
|
try {
|
|
462
510
|
parsed = JSON.parse(text);
|
|
463
511
|
} catch (error) {
|
|
464
|
-
throw new
|
|
512
|
+
throw new TolokaCliError(`Dataset export ${options.exportId} is not valid JSON.`, {
|
|
513
|
+
cause: error,
|
|
514
|
+
hint: REPORT_HINT,
|
|
515
|
+
});
|
|
465
516
|
}
|
|
466
|
-
|
|
517
|
+
assertCli(
|
|
518
|
+
Array.isArray(parsed),
|
|
519
|
+
`Dataset export ${options.exportId} is not a JSON array.`,
|
|
520
|
+
{ hint: REPORT_HINT },
|
|
521
|
+
);
|
|
467
522
|
const rows: Record<string, unknown>[] = [];
|
|
468
523
|
for (const [index, element] of parsed.entries()) {
|
|
469
|
-
|
|
524
|
+
assertCli(
|
|
470
525
|
isJsonObject(element),
|
|
471
526
|
`Dataset export ${options.exportId} row ${index} is not a JSON object.`,
|
|
527
|
+
{ hint: REPORT_HINT },
|
|
472
528
|
);
|
|
473
529
|
rows.push(element);
|
|
474
530
|
}
|
|
@@ -485,7 +541,11 @@ export async function loadDataset(
|
|
|
485
541
|
if (field.internal || field.name === RAW_UPLOAD_FIELD_NAME) continue;
|
|
486
542
|
fieldNames.push(field.name);
|
|
487
543
|
}
|
|
488
|
-
|
|
544
|
+
assertCli(
|
|
545
|
+
fieldNames.length > 0,
|
|
546
|
+
`Dataset ${resolved.name} has no fields to export.`,
|
|
547
|
+
{ hint: { description: "Upload rows first with toloka dataset upload <name> <file>." } },
|
|
548
|
+
);
|
|
489
549
|
|
|
490
550
|
const started = await client.datasets.startExport({
|
|
491
551
|
datasetId: resolved.datasetId,
|
|
@@ -498,16 +558,18 @@ export async function loadDataset(
|
|
|
498
558
|
options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs },
|
|
499
559
|
);
|
|
500
560
|
} catch (error) {
|
|
501
|
-
|
|
561
|
+
assertCli(
|
|
502
562
|
!(error instanceof PotatoApiError && error.code === "wait_timeout"),
|
|
503
563
|
`Stopped waiting for dataset export ${started.id}; raise --timeout to wait longer.`,
|
|
564
|
+
{ hint: { disabled: true } },
|
|
504
565
|
);
|
|
505
566
|
throw error;
|
|
506
567
|
}
|
|
507
568
|
|
|
508
|
-
|
|
569
|
+
assertCli(
|
|
509
570
|
completed.result.downloadUrl,
|
|
510
571
|
`Dataset export ${completed.id} succeeded without a download URL.`,
|
|
572
|
+
{ hint: REPORT_HINT },
|
|
511
573
|
);
|
|
512
574
|
|
|
513
575
|
const rows = await downloadExportRows({
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { PotatoApiError } from "../errors";
|
|
2
|
+
import { TolokaCliError } from "./cli-error";
|
|
3
|
+
|
|
1
4
|
export type DeliveryOrderRecord = {
|
|
2
5
|
readonly id: string;
|
|
3
6
|
readonly name: string;
|
|
@@ -15,7 +18,21 @@ export type DeliveryOrderListApi = {
|
|
|
15
18
|
export async function listDeliveryOrders(options: {
|
|
16
19
|
readonly client: DeliveryOrderListApi;
|
|
17
20
|
}): Promise<{ readonly output: string }> {
|
|
18
|
-
|
|
21
|
+
let orders: readonly DeliveryOrderRecord[];
|
|
22
|
+
try {
|
|
23
|
+
orders = await options.client.deliveryOrders.list();
|
|
24
|
+
} catch (error) {
|
|
25
|
+
if (error instanceof PotatoApiError && error.status === 403) {
|
|
26
|
+
throw new TolokaCliError("Delivery orders are available to internal teams only.", {
|
|
27
|
+
cause: error,
|
|
28
|
+
hint: {
|
|
29
|
+
description:
|
|
30
|
+
"Run toloka whoami; when it prints Internal: false, create the project without --delivery-order-id.",
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
19
36
|
if (orders.length === 0) {
|
|
20
37
|
return { output: "No delivery orders visible." };
|
|
21
38
|
}
|
package/src/cli/file-system.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import { lstat } from "node:fs/promises";
|
|
3
2
|
import { join } from "node:path";
|
|
3
|
+
import { assertCli } from "./cli-error";
|
|
4
4
|
|
|
5
5
|
export const hasErrorCode = (error: unknown, ...codes: readonly string[]) =>
|
|
6
6
|
error instanceof Error &&
|
|
@@ -29,6 +29,10 @@ export const assertSafeDestination = async (
|
|
|
29
29
|
currentPath = join(currentPath, segment);
|
|
30
30
|
const entry = await entryStats(currentPath);
|
|
31
31
|
if (entry === undefined) return;
|
|
32
|
-
|
|
32
|
+
assertCli(
|
|
33
|
+
!entry.isSymbolicLink(),
|
|
34
|
+
`Refusing to ${action} through symbolic link ${currentPath}.`,
|
|
35
|
+
{ hint: { description: "Replace the symbolic link with a regular directory, or point the command at the real path." } },
|
|
36
|
+
);
|
|
33
37
|
}
|
|
34
38
|
};
|