@toloka-platform/sdk 0.2.0-preview.2 → 0.2.0-preview.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -4
- package/dist/cli/toloka.js +162 -154
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +18 -1
- package/docs/cli-v2.md +6 -4
- package/docs/cli.md +58 -11
- package/docs/getting-started.md +3 -3
- package/docs/limitations.md +1 -1
- package/docs/testing.md +1 -1
- package/docs/troubleshooting.md +1 -1
- package/examples/face-image-collection/README.md +2 -2
- package/openapi/gen/api.ts +78 -0
- package/openapi/gen/models/baseModel.ts +1 -0
- package/openapi/gen/models/checkReport.ts +2 -1
- package/openapi/gen/models/checkReportAnyOfTwoseven.ts +5 -0
- package/openapi/gen/models/checkReportAnyOfTwosevenResultsItem.ts +6 -0
- package/openapi/gen/models/checkReportAnyOfTwosevenResultsItemVerdict.ts +2 -0
- package/openapi/gen/models/currentUser.ts +3 -0
- package/openapi/gen/models/currentUserTeamsItem.ts +6 -0
- package/openapi/gen/models/deliveryOrder.ts +9 -0
- package/openapi/gen/models/deliveryOrderClientName.ts +2 -0
- package/openapi/gen/models/getTrainingRunMetricHistoryParams.ts +4 -0
- package/openapi/gen/models/gistingTrainingConfiguration.ts +6 -2
- package/openapi/gen/models/gistingTrainingConfigurationBatchSize.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationGistTokens.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationInput.ts +6 -2
- package/openapi/gen/models/gistingTrainingConfigurationInputBatchSize.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationInputGistTokens.ts +2 -0
- package/openapi/gen/models/index.ts +28 -4
- package/openapi/gen/models/listDeliveryOrders200.ts +5 -0
- package/openapi/gen/models/sftTrainingConfiguration.ts +6 -3
- package/openapi/gen/models/sftTrainingConfigurationBatchSize.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationInput.ts +6 -3
- package/openapi/gen/models/sftTrainingConfigurationInputBatchSize.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationInputLoraRank.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationLoraRank.ts +2 -0
- package/openapi/gen/models/trainingRunMetricHistory.ts +9 -0
- package/openapi/gen/models/trainingRunMetricHistoryMetric.ts +2 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItem.ts +9 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItemNonFinite.ts +2 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItemValue.ts +2 -0
- package/openapi/gen/models/trainingRunMetrics.ts +9 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItem.ts +8 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatest.ts +9 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatestNonFinite.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatestValue.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemMetric.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsUnavailableReason.ts +2 -0
- package/package.json +2 -2
- package/skills/catalog.json +55 -7
- package/skills/toloka-build/SKILL.md +35 -21
- package/skills/toloka-build/references/node-semantics.md +13 -11
- package/skills/toloka-build/references/workflow-and-cli.md +31 -20
- package/skills/toloka-fine-tuning/SKILL.md +155 -0
- package/skills/toloka-get-started/SKILL.md +70 -0
- package/skills/toloka-gisting/SKILL.md +166 -0
- package/skills/toloka-report-issue/references/report-template.md +1 -1
- package/src/authoring/index.ts +14 -0
- package/src/checks/task-ui-typecheck.ts +12 -10
- package/src/cli/analytics.ts +88 -15
- package/src/cli/auth.ts +93 -21
- package/src/cli/auto-update.ts +36 -39
- package/src/cli/check.ts +88 -39
- package/src/cli/cli-entry.ts +3 -2
- package/src/cli/cli-error.ts +43 -0
- package/src/cli/commands/auth.ts +89 -0
- package/src/cli/commands/config.ts +28 -0
- package/src/cli/commands/dataset.ts +85 -0
- package/src/cli/commands/delivery-order.ts +18 -0
- package/src/cli/commands/models.ts +121 -0
- package/src/cli/commands/pipeline.ts +131 -0
- package/src/cli/commands/project.ts +168 -0
- package/src/cli/commands/report.ts +43 -0
- package/src/cli/commands/secret.ts +40 -0
- package/src/cli/commands/skill.ts +36 -0
- package/src/cli/commands/source.ts +204 -0
- package/src/cli/commands/status.ts +17 -0
- package/src/cli/commands/training.ts +261 -0
- package/src/cli/commands/version.ts +14 -0
- package/src/cli/config.ts +7 -8
- package/src/cli/context.ts +251 -0
- package/src/cli/credentials.ts +12 -4
- package/src/cli/data.ts +96 -34
- package/src/cli/delivery-orders.ts +47 -0
- package/src/cli/environment.ts +26 -0
- package/src/cli/file-system.ts +6 -2
- package/src/cli/format-error.ts +45 -2
- package/src/cli/init.ts +14 -11
- package/src/cli/models.ts +43 -12
- package/src/cli/program.ts +120 -1347
- package/src/cli/project-context.ts +126 -43
- package/src/cli/project.ts +80 -28
- package/src/cli/report.ts +24 -9
- package/src/cli/runs.ts +33 -8
- package/src/cli/sdk-version.ts +38 -0
- package/src/cli/settings.ts +13 -5
- package/src/cli/skill.ts +67 -24
- package/src/cli/source.ts +3 -2
- package/src/cli/terminal-io.ts +110 -0
- package/src/cli/trainings.ts +242 -57
- package/src/cli/version.ts +3 -3
- package/src/client.ts +21 -9
- package/src/importer.ts +32 -2
- package/src/transport.ts +13 -1
- package/openapi/gen/models/gistingTrainingConfigurationInputRegime.ts +0 -2
- package/openapi/gen/models/gistingTrainingConfigurationRegime.ts +0 -2
- package/openapi/gen/models/sftTrainingConfigurationInputRegime.ts +0 -2
- package/openapi/gen/models/sftTrainingConfigurationRegime.ts +0 -2
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { InvalidArgumentError, type Command } from "commander";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { createClient } from "../client";
|
|
4
|
+
import type { PotatoClient } from "../client";
|
|
5
|
+
import { readTolokaState } from "../state";
|
|
6
|
+
import type { TolokaState } from "../state";
|
|
7
|
+
import type { AuthPrompt } from "./auth";
|
|
8
|
+
import type { CliAnalytics } from "./analytics";
|
|
9
|
+
import type { checkProject } from "./check";
|
|
10
|
+
import { PRODUCTION_BASE_URL, resolveTolokaConfig } from "./config";
|
|
11
|
+
import type { TolokaEnvironment } from "./config";
|
|
12
|
+
import { findCommandCredentials } from "./credentials";
|
|
13
|
+
import type { StoredCredentials } from "./credentials";
|
|
14
|
+
import { resolveProjectContext } from "./project-context";
|
|
15
|
+
import { findSourceRoot } from "./source";
|
|
16
|
+
import type { TrainingDraftOptions } from "./trainings";
|
|
17
|
+
import type { RunOutcome } from "./runs";
|
|
18
|
+
|
|
19
|
+
export type CliIo = {
|
|
20
|
+
readonly stdout: (text: string) => void;
|
|
21
|
+
readonly stderr: (text: string) => void;
|
|
22
|
+
readonly readStdin: () => Promise<string>;
|
|
23
|
+
readonly prompt?: AuthPrompt;
|
|
24
|
+
readonly select?: (choices: readonly { name: string; value: string }[]) => Promise<string>;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type ProgramContext = {
|
|
28
|
+
readonly environment: TolokaEnvironment;
|
|
29
|
+
readonly io: CliIo;
|
|
30
|
+
readonly injectedClient?: PotatoClient;
|
|
31
|
+
readonly runChecks?: typeof checkProject;
|
|
32
|
+
readonly analytics?: CliAnalytics;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
export type GlobalOptions = {
|
|
36
|
+
readonly sourceRoot?: string;
|
|
37
|
+
readonly apiKey?: string;
|
|
38
|
+
readonly baseUrl?: string;
|
|
39
|
+
readonly projectId?: string;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export type CreatedProgram = {
|
|
43
|
+
readonly program: Command;
|
|
44
|
+
readonly exitCode: { value: number };
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export type Resolved = {
|
|
48
|
+
readonly sourceRoot: string;
|
|
49
|
+
readonly client: PotatoClient;
|
|
50
|
+
readonly baseUrl: string;
|
|
51
|
+
readonly state: TolokaState | undefined;
|
|
52
|
+
readonly stored: StoredCredentials | undefined;
|
|
53
|
+
readonly projectId?: string;
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
export const CREDENTIAL_PRECEDENCE = [
|
|
57
|
+
"Credentials resolve in this order, highest first:",
|
|
58
|
+
" 1. --api-key / --base-url",
|
|
59
|
+
" 2. TOLOKA_API_KEY / TOLOKA_BASE_URL",
|
|
60
|
+
" 3. the file written by toloka login",
|
|
61
|
+
`The platform defaults to ${PRODUCTION_BASE_URL}.`,
|
|
62
|
+
].join("\n");
|
|
63
|
+
|
|
64
|
+
export const authoredByOption = (raw: string): "human" | "agent" => {
|
|
65
|
+
if (raw !== "human" && raw !== "agent")
|
|
66
|
+
throw new InvalidArgumentError('Expected "human" or "agent".');
|
|
67
|
+
return raw;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
export const positiveInteger = (raw: string): number => {
|
|
71
|
+
const value = Number(raw);
|
|
72
|
+
if (!Number.isInteger(value) || value <= 0)
|
|
73
|
+
throw new InvalidArgumentError("Expected a positive integer.");
|
|
74
|
+
return value;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
export const nonNegativeInteger = (raw: string): number => {
|
|
78
|
+
const value = Number(raw);
|
|
79
|
+
if (!Number.isInteger(value) || value < 0)
|
|
80
|
+
throw new InvalidArgumentError("Expected a non-negative integer.");
|
|
81
|
+
return value;
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
export const trainingMethod = (raw: string): "sft" | "gisting" => {
|
|
85
|
+
if (raw !== "sft" && raw !== "gisting") {
|
|
86
|
+
throw new InvalidArgumentError('Expected training method "sft" or "gisting".');
|
|
87
|
+
}
|
|
88
|
+
return raw;
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
export const trainingLoraRank = (raw: string): 8 | 16 | 32 => {
|
|
92
|
+
const value = Number(raw);
|
|
93
|
+
if (value !== 8 && value !== 16 && value !== 32) {
|
|
94
|
+
throw new InvalidArgumentError("Expected a LoRA rank of 8, 16, or 32.");
|
|
95
|
+
}
|
|
96
|
+
return value;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
export const trainingGistTokens = (raw: string): 8 | 32 | 64 => {
|
|
100
|
+
const value = Number(raw);
|
|
101
|
+
if (value !== 8 && value !== 32 && value !== 64) {
|
|
102
|
+
throw new InvalidArgumentError("Expected 8, 32, or 64 gist tokens.");
|
|
103
|
+
}
|
|
104
|
+
return value;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
export const trainingEpochs = (raw: string): number => {
|
|
108
|
+
const value = Number(raw);
|
|
109
|
+
if (!Number.isInteger(value) || value < 1 || value > 5) {
|
|
110
|
+
throw new InvalidArgumentError("Expected epochs from 1 through 5.");
|
|
111
|
+
}
|
|
112
|
+
return value;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
export const trainingLearningRate = (raw: string): number => {
|
|
116
|
+
const value = Number(raw);
|
|
117
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
118
|
+
throw new InvalidArgumentError("Expected a positive learning rate, for example 0.00015.");
|
|
119
|
+
}
|
|
120
|
+
return value;
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
export const trainingBatchSize = (raw: string): 8 | 16 | 32 | 64 => {
|
|
124
|
+
const value = Number(raw);
|
|
125
|
+
if (value !== 8 && value !== 16 && value !== 32 && value !== 64) {
|
|
126
|
+
throw new InvalidArgumentError("Expected a batch size of 8, 16, 32, or 64.");
|
|
127
|
+
}
|
|
128
|
+
return value;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
export const trainingMaxSequenceLength = (raw: string): number => {
|
|
132
|
+
const value = Number(raw);
|
|
133
|
+
if (!Number.isInteger(value) || value < 64 || value > 16384) {
|
|
134
|
+
throw new InvalidArgumentError("Expected a maximum sequence length from 64 through 16384 tokens.");
|
|
135
|
+
}
|
|
136
|
+
return value;
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
export type RawTrainingDraftOptions = {
|
|
140
|
+
method?: TrainingDraftOptions["method"];
|
|
141
|
+
baseModel?: TrainingDraftOptions["baseModelId"];
|
|
142
|
+
modelVersion?: string;
|
|
143
|
+
dataset?: string;
|
|
144
|
+
validationDataset?: string;
|
|
145
|
+
trainFilter?: string;
|
|
146
|
+
validationFilter?: string;
|
|
147
|
+
loraRank?: TrainingDraftOptions["loraRank"];
|
|
148
|
+
gistTokens?: TrainingDraftOptions["gistTokens"];
|
|
149
|
+
epochs?: number;
|
|
150
|
+
learningRate?: number;
|
|
151
|
+
batchSize?: TrainingDraftOptions["batchSize"];
|
|
152
|
+
maxSequenceLength?: number;
|
|
153
|
+
promptFile?: string;
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
export const trainingDraftOptions = (options: RawTrainingDraftOptions): TrainingDraftOptions => ({
|
|
157
|
+
method: options.method,
|
|
158
|
+
baseModelId: options.baseModel,
|
|
159
|
+
modelVersionId: options.modelVersion,
|
|
160
|
+
datasetId: options.dataset,
|
|
161
|
+
validationDatasetId: options.validationDataset,
|
|
162
|
+
trainingFilter: options.trainFilter,
|
|
163
|
+
validationFilter: options.validationFilter,
|
|
164
|
+
loraRank: options.loraRank,
|
|
165
|
+
gistTokens: options.gistTokens,
|
|
166
|
+
epochs: options.epochs,
|
|
167
|
+
learningRate: options.learningRate,
|
|
168
|
+
batchSize: options.batchSize,
|
|
169
|
+
maxSequenceLength: options.maxSequenceLength,
|
|
170
|
+
promptFilePath: options.promptFile,
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
export const exitCodeForOutcome = (outcome: RunOutcome): number => {
|
|
174
|
+
if (outcome === "failed") return 1;
|
|
175
|
+
if (outcome === "timedOut") return 2;
|
|
176
|
+
return 0;
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
export const resolveSourceRoot = async (globals: GlobalOptions, io: CliIo): Promise<string> => {
|
|
180
|
+
if (globals.sourceRoot !== undefined) return resolve(globals.sourceRoot);
|
|
181
|
+
|
|
182
|
+
const workingDirectory = process.cwd();
|
|
183
|
+
const discovered = await findSourceRoot(workingDirectory);
|
|
184
|
+
if (discovered === undefined || discovered === workingDirectory) return workingDirectory;
|
|
185
|
+
|
|
186
|
+
io.stderr(`Using project root ${discovered} (found from ${workingDirectory}).`);
|
|
187
|
+
return discovered;
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
export const resolveInitRoot = (globals: GlobalOptions): string =>
|
|
191
|
+
globals.sourceRoot === undefined ? process.cwd() : resolve(globals.sourceRoot);
|
|
192
|
+
|
|
193
|
+
export const readCommandCredentials = async (globals: GlobalOptions, context: ProgramContext) => {
|
|
194
|
+
const { found, warning } = await findCommandCredentials(
|
|
195
|
+
context.environment,
|
|
196
|
+
globals.apiKey ?? context.environment.TOLOKA_API_KEY,
|
|
197
|
+
);
|
|
198
|
+
if (warning !== undefined) context.io.stderr(warning);
|
|
199
|
+
return found?.credentials;
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
export const identifyClientUser = async (
|
|
203
|
+
client: PotatoClient,
|
|
204
|
+
analytics: CliAnalytics | undefined,
|
|
205
|
+
) => {
|
|
206
|
+
if (analytics?.shouldIdentify !== true) return;
|
|
207
|
+
try {
|
|
208
|
+
analytics.identifyCurrentUser(await client.me());
|
|
209
|
+
} catch {}
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
export const resolveContext = async (
|
|
213
|
+
globals: GlobalOptions,
|
|
214
|
+
context: ProgramContext,
|
|
215
|
+
): Promise<Resolved> => {
|
|
216
|
+
const sourceRoot = await resolveSourceRoot(globals, context.io);
|
|
217
|
+
const state = await readTolokaState(sourceRoot);
|
|
218
|
+
const stored = await readCommandCredentials(globals, context);
|
|
219
|
+
const config = resolveTolokaConfig(globals, state, context.environment, stored);
|
|
220
|
+
const client =
|
|
221
|
+
context.injectedClient ?? createClient({ apiKey: config.apiKey, baseUrl: config.baseUrl });
|
|
222
|
+
await identifyClientUser(client, context.analytics);
|
|
223
|
+
|
|
224
|
+
return {
|
|
225
|
+
sourceRoot,
|
|
226
|
+
client,
|
|
227
|
+
baseUrl: config.baseUrl,
|
|
228
|
+
state,
|
|
229
|
+
stored,
|
|
230
|
+
...(config.projectId === undefined ? {} : { projectId: config.projectId }),
|
|
231
|
+
};
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
export const resolveSelectedProject = (
|
|
235
|
+
globals: GlobalOptions,
|
|
236
|
+
context: ProgramContext,
|
|
237
|
+
stored: StoredCredentials | undefined,
|
|
238
|
+
state: TolokaState | undefined,
|
|
239
|
+
) =>
|
|
240
|
+
resolveProjectContext({
|
|
241
|
+
flags: globals,
|
|
242
|
+
environment: context.environment,
|
|
243
|
+
stored,
|
|
244
|
+
...(state === undefined ? {} : { state }),
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
export type CommandHost = {
|
|
248
|
+
readonly context: ProgramContext;
|
|
249
|
+
readonly exitCode: { value: number };
|
|
250
|
+
readonly globals: () => GlobalOptions;
|
|
251
|
+
};
|
package/src/cli/credentials.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import { chmod, lstat, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
3
2
|
import { homedir } from "node:os";
|
|
4
3
|
import { dirname, join } from "node:path";
|
|
4
|
+
import { assertCli, TolokaCliError } from "./cli-error";
|
|
5
|
+
import type { TolokaCliErrorOptions } from "./cli-error";
|
|
5
6
|
|
|
6
7
|
export type StoredProjectAlias = {
|
|
7
8
|
readonly id: string;
|
|
@@ -28,8 +29,8 @@ export type CommandCredentialLookup = {
|
|
|
28
29
|
readonly warning?: string;
|
|
29
30
|
};
|
|
30
31
|
|
|
31
|
-
export class CredentialsError extends
|
|
32
|
-
constructor(message: string, options
|
|
32
|
+
export class CredentialsError extends TolokaCliError {
|
|
33
|
+
constructor(message: string, options: TolokaCliErrorOptions) {
|
|
33
34
|
super(message, options);
|
|
34
35
|
this.name = "CredentialsError";
|
|
35
36
|
}
|
|
@@ -168,6 +169,7 @@ export async function findStoredCredentials(
|
|
|
168
169
|
if (primary === "invalid") {
|
|
169
170
|
throw new CredentialsError(
|
|
170
171
|
`Invalid credentials file at ${path}. Run toloka logout, then toloka login.`,
|
|
172
|
+
{ hint: { disabled: true } },
|
|
171
173
|
);
|
|
172
174
|
}
|
|
173
175
|
return { credentials: primary.credentials, path, legacy: false };
|
|
@@ -181,6 +183,7 @@ export async function findStoredCredentials(
|
|
|
181
183
|
if (legacy === "invalid") {
|
|
182
184
|
throw new CredentialsError(
|
|
183
185
|
`Invalid credentials file at ${legacyPath}. Run toloka logout, then toloka login.`,
|
|
186
|
+
{ hint: { disabled: true } },
|
|
184
187
|
);
|
|
185
188
|
}
|
|
186
189
|
return { credentials: legacy.credentials, path: legacyPath, legacy: true };
|
|
@@ -211,6 +214,7 @@ export async function requireStoredCredentials(
|
|
|
211
214
|
if (found !== undefined) return found;
|
|
212
215
|
throw new CredentialsError(
|
|
213
216
|
`No stored credentials found at ${credentialsPath(environment)}. Run toloka login.`,
|
|
217
|
+
{ hint: { disabled: true } },
|
|
214
218
|
);
|
|
215
219
|
}
|
|
216
220
|
|
|
@@ -228,7 +232,11 @@ const serializedCredentials = (credentials: StoredCredentials): StoredCredential
|
|
|
228
232
|
const assertNotSymbolicLink = async (path: string): Promise<void> => {
|
|
229
233
|
try {
|
|
230
234
|
const entry = await lstat(path);
|
|
231
|
-
|
|
235
|
+
assertCli(
|
|
236
|
+
!entry.isSymbolicLink(),
|
|
237
|
+
`Refusing to write through symbolic link ${path}.`,
|
|
238
|
+
{ hint: { description: "Replace the symbolic link with a regular file or directory, then retry." } },
|
|
239
|
+
);
|
|
232
240
|
} catch (error) {
|
|
233
241
|
if (error instanceof Error && "code" in error && error.code === "ENOENT") return;
|
|
234
242
|
throw error;
|
package/src/cli/data.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
2
|
import { basename, dirname } from "node:path";
|
|
4
3
|
import type { Dataset, InferredField, PotatoClient, UploadId } from "../client";
|
|
@@ -7,18 +6,20 @@ import { generateFieldDeclarationsSource } from "../importer";
|
|
|
7
6
|
import { readTolokaState } from "../state";
|
|
8
7
|
import type { WaitForOptions } from "../wait-for";
|
|
9
8
|
import { desiredDatasets } from "./project";
|
|
10
|
-
import {
|
|
9
|
+
import { assertProjectSelected } from "./project-context";
|
|
11
10
|
import { loadProjectSource } from "./source";
|
|
12
11
|
import type { ProjectSource } from "./source";
|
|
12
|
+
import { assertCli, REPORT_HINT, RETRY_OR_REPORT_HINT, TolokaCliError } from "./cli-error";
|
|
13
|
+
import type { TolokaCliErrorOptions } from "./cli-error";
|
|
13
14
|
|
|
14
15
|
const RAW_UPLOAD_FIELD_NAME = "uploaded";
|
|
15
16
|
const DATASET_ID_PATTERN =
|
|
16
17
|
/^ds\.[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
17
18
|
const FINALIZE_UPLOAD_ATTEMPTS = 3;
|
|
18
19
|
|
|
19
|
-
class DatasetUploadError extends
|
|
20
|
-
constructor(message: string,
|
|
21
|
-
super(message,
|
|
20
|
+
class DatasetUploadError extends TolokaCliError {
|
|
21
|
+
constructor(message: string, options: TolokaCliErrorOptions) {
|
|
22
|
+
super(message, options);
|
|
22
23
|
this.name = "DatasetUploadError";
|
|
23
24
|
}
|
|
24
25
|
}
|
|
@@ -171,7 +172,7 @@ const resolveLiveDataset = async (
|
|
|
171
172
|
): Promise<DatasetRef> => {
|
|
172
173
|
const state = await readTolokaState(options.sourceRoot);
|
|
173
174
|
const projectId = options.projectId ?? state?.projectId;
|
|
174
|
-
|
|
175
|
+
assertProjectSelected(projectId);
|
|
175
176
|
const token = options.datasetName;
|
|
176
177
|
const live = await collectLiveDatasets(client, projectId);
|
|
177
178
|
const nameIsUnique = (name: string) =>
|
|
@@ -186,13 +187,14 @@ const resolveLiveDataset = async (
|
|
|
186
187
|
}
|
|
187
188
|
|
|
188
189
|
const byName = live.filter((dataset) => dataset.name === token);
|
|
189
|
-
|
|
190
|
+
assertCli(
|
|
190
191
|
byName.length <= 1,
|
|
191
192
|
`Project ${projectId} has multiple datasets named "${token}": ${byName.map(formatDatasetChoice).join(", ")}.`,
|
|
193
|
+
{ hint: { description: "Pass the dataset id (ds.…) instead of the name." } },
|
|
192
194
|
);
|
|
193
195
|
if (byName.length === 1) {
|
|
194
196
|
const [dataset] = byName;
|
|
195
|
-
|
|
197
|
+
assertCli(dataset, `Dataset "${token}" has no resolved ID.`, { hint: REPORT_HINT });
|
|
196
198
|
return { datasetId: dataset.id, name: dataset.name, nameIsUnique: true };
|
|
197
199
|
}
|
|
198
200
|
|
|
@@ -204,16 +206,18 @@ const resolveLiveDataset = async (
|
|
|
204
206
|
}
|
|
205
207
|
}
|
|
206
208
|
|
|
207
|
-
|
|
209
|
+
assertCli(
|
|
208
210
|
createWhenMissing && datasetId === undefined,
|
|
209
211
|
live.length === 0
|
|
210
|
-
? `Dataset "${token}" does not exist in project ${projectId}. Run toloka apply
|
|
212
|
+
? `Dataset "${token}" does not exist in project ${projectId}. Run toloka dataset create ${JSON.stringify(token)} first, or toloka apply for a dataset a pipeline declares.`
|
|
211
213
|
: `Dataset "${token}" does not exist in project ${projectId}. This project has: ${live.map(formatDatasetChoice).join(", ")}.`,
|
|
214
|
+
{ hint: { disabled: true } },
|
|
212
215
|
);
|
|
213
216
|
|
|
214
|
-
|
|
217
|
+
assertCli(
|
|
215
218
|
(await declaredDatasetFields(options)).length > 0,
|
|
216
|
-
`Dataset "${token}" does not exist in project ${projectId} and no pipeline in source declares it.
|
|
219
|
+
`Dataset "${token}" does not exist in project ${projectId} and no pipeline in source declares it. Run toloka dataset create ${JSON.stringify(token)} to create it empty, bind a pipeline to it in source so its columns are known, or upload to an existing dataset.`,
|
|
220
|
+
{ hint: { disabled: true } },
|
|
217
221
|
);
|
|
218
222
|
const created = await client.datasets.create({ projectId, name: token });
|
|
219
223
|
return { datasetId: created.id, name: created.name, nameIsUnique: true };
|
|
@@ -226,7 +230,11 @@ const resolveDatasetUploadFormat = (filename: string): DatasetUploadFormat => {
|
|
|
226
230
|
if (normalizedFilename.endsWith(".jsonl")) return "jsonl";
|
|
227
231
|
if (normalizedFilename.endsWith(".csv")) return "csv";
|
|
228
232
|
|
|
229
|
-
|
|
233
|
+
assertCli(
|
|
234
|
+
false,
|
|
235
|
+
`${filename} must have a .json, .jsonl, or .csv extension`,
|
|
236
|
+
{ hint: { description: "Rename the file with the extension that matches its format." } },
|
|
237
|
+
);
|
|
230
238
|
};
|
|
231
239
|
|
|
232
240
|
const readDatasetUpload = async (filePath: string) => {
|
|
@@ -243,9 +251,17 @@ const readDatasetUpload = async (filePath: string) => {
|
|
|
243
251
|
try {
|
|
244
252
|
value = JSON.parse(line);
|
|
245
253
|
} catch {
|
|
246
|
-
|
|
254
|
+
assertCli(
|
|
255
|
+
false,
|
|
256
|
+
`${filename}:${lineNumber} is not valid JSON`,
|
|
257
|
+
{ hint: { description: "Fix the JSON on that line; every line of a JSONL file must be one complete object." } },
|
|
258
|
+
);
|
|
247
259
|
}
|
|
248
|
-
|
|
260
|
+
assertCli(
|
|
261
|
+
isJsonObject(value),
|
|
262
|
+
`${filename}:${lineNumber} must contain a JSON object`,
|
|
263
|
+
{ hint: { description: "Wrap the value in an object with named fields, one object per line." } },
|
|
264
|
+
);
|
|
249
265
|
}
|
|
250
266
|
}
|
|
251
267
|
|
|
@@ -265,16 +281,22 @@ const cancelStartedUpload = async (
|
|
|
265
281
|
await client.datasets.cancelUpload({ uploadId });
|
|
266
282
|
} catch (cancelError) {
|
|
267
283
|
const cancelReason = cancelError instanceof Error ? cancelError.message : String(cancelError);
|
|
268
|
-
throw new
|
|
269
|
-
[error, cancelError],
|
|
284
|
+
throw new DatasetUploadError(
|
|
270
285
|
`Dataset upload ${uploadId} failed and could not be canceled. Upload error: ${reason}. Cancellation error: ${cancelReason}.`,
|
|
286
|
+
{
|
|
287
|
+
cause: new AggregateError([error, cancelError]),
|
|
288
|
+
hint: {
|
|
289
|
+
description:
|
|
290
|
+
"Check toloka dataset list before re-running toloka dataset upload; an upload that never finalized is cleaned up automatically.",
|
|
291
|
+
},
|
|
292
|
+
},
|
|
271
293
|
);
|
|
272
294
|
}
|
|
273
295
|
|
|
274
|
-
throw new DatasetUploadError(
|
|
275
|
-
|
|
276
|
-
error,
|
|
277
|
-
);
|
|
296
|
+
throw new DatasetUploadError(`Dataset upload ${uploadId} failed and was canceled: ${reason}`, {
|
|
297
|
+
cause: error,
|
|
298
|
+
hint: { description: "Fix the upload error above, then re-run toloka dataset upload." },
|
|
299
|
+
});
|
|
278
300
|
};
|
|
279
301
|
|
|
280
302
|
const transferAndFinalizeUpload = async ({
|
|
@@ -310,10 +332,9 @@ const transferAndFinalizeUpload = async ({
|
|
|
310
332
|
return cancelStartedUpload(
|
|
311
333
|
client,
|
|
312
334
|
started.job.id,
|
|
313
|
-
new DatasetUploadError(
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
),
|
|
335
|
+
new DatasetUploadError(`Dataset upload failed with HTTP ${uploadResponse.status}.`, {
|
|
336
|
+
hint: RETRY_OR_REPORT_HINT,
|
|
337
|
+
}),
|
|
317
338
|
);
|
|
318
339
|
}
|
|
319
340
|
|
|
@@ -334,7 +355,7 @@ const transferAndFinalizeUpload = async ({
|
|
|
334
355
|
started.job.id,
|
|
335
356
|
new DatasetUploadError(
|
|
336
357
|
`Storage remained unavailable after ${FINALIZE_UPLOAD_ATTEMPTS} finalization attempts.`,
|
|
337
|
-
error,
|
|
358
|
+
{ cause: error, hint: RETRY_OR_REPORT_HINT },
|
|
338
359
|
),
|
|
339
360
|
);
|
|
340
361
|
}
|
|
@@ -346,6 +367,28 @@ const transferAndFinalizeUpload = async ({
|
|
|
346
367
|
}
|
|
347
368
|
};
|
|
348
369
|
|
|
370
|
+
export type CreateDatasetResult = {
|
|
371
|
+
readonly datasetId: string;
|
|
372
|
+
readonly output: string;
|
|
373
|
+
};
|
|
374
|
+
|
|
375
|
+
export async function createDataset(
|
|
376
|
+
client: Pick<PotatoClient, "datasets">,
|
|
377
|
+
options: { readonly projectId: string; readonly name: string },
|
|
378
|
+
): Promise<CreateDatasetResult> {
|
|
379
|
+
const name = options.name.trim();
|
|
380
|
+
assertCli(
|
|
381
|
+
name.length > 0,
|
|
382
|
+
"Dataset name cannot be empty.",
|
|
383
|
+
{ hint: { description: "Pass a non-empty dataset name." } },
|
|
384
|
+
);
|
|
385
|
+
const created = await client.datasets.create({ projectId: options.projectId, name });
|
|
386
|
+
return {
|
|
387
|
+
datasetId: created.id,
|
|
388
|
+
output: `Created dataset ${created.name} (${created.id}). Upload rows with toloka dataset upload ${created.id} <file>.`,
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
|
|
349
392
|
export async function uploadDataset(
|
|
350
393
|
client: PotatoClient,
|
|
351
394
|
options: UploadDatasetOptions,
|
|
@@ -388,9 +431,10 @@ export async function uploadDataset(
|
|
|
388
431
|
completed.error?.startsWith("Schema incompatible with existing upload") === true
|
|
389
432
|
? `\nRun toloka dataset fields ${JSON.stringify(fieldsHint)} to inspect the inferred schema stored from the first successful upload.`
|
|
390
433
|
: "";
|
|
391
|
-
|
|
434
|
+
assertCli(
|
|
392
435
|
completed.status === "succeeded",
|
|
393
436
|
`${completed.error ?? `Dataset upload ${completed.id} failed.`}${compatibilityHint}`,
|
|
437
|
+
{ hint: { disabled: true } },
|
|
394
438
|
);
|
|
395
439
|
|
|
396
440
|
const uploadedOutput = `Uploaded ${completed.result.itemsIngested} rows to dataset ${resolved.name}.`;
|
|
@@ -455,20 +499,32 @@ const downloadExportRows = async (options: {
|
|
|
455
499
|
readonly fetch: typeof globalThis.fetch;
|
|
456
500
|
}): Promise<Record<string, unknown>[]> => {
|
|
457
501
|
const download = await options.fetch(options.downloadUrl);
|
|
458
|
-
|
|
502
|
+
assertCli(
|
|
503
|
+
download.ok,
|
|
504
|
+
`Dataset export download failed with HTTP ${download.status}.`,
|
|
505
|
+
{ hint: RETRY_OR_REPORT_HINT },
|
|
506
|
+
);
|
|
459
507
|
const text = await download.text();
|
|
460
508
|
let parsed: unknown;
|
|
461
509
|
try {
|
|
462
510
|
parsed = JSON.parse(text);
|
|
463
511
|
} catch (error) {
|
|
464
|
-
throw new
|
|
512
|
+
throw new TolokaCliError(`Dataset export ${options.exportId} is not valid JSON.`, {
|
|
513
|
+
cause: error,
|
|
514
|
+
hint: REPORT_HINT,
|
|
515
|
+
});
|
|
465
516
|
}
|
|
466
|
-
|
|
517
|
+
assertCli(
|
|
518
|
+
Array.isArray(parsed),
|
|
519
|
+
`Dataset export ${options.exportId} is not a JSON array.`,
|
|
520
|
+
{ hint: REPORT_HINT },
|
|
521
|
+
);
|
|
467
522
|
const rows: Record<string, unknown>[] = [];
|
|
468
523
|
for (const [index, element] of parsed.entries()) {
|
|
469
|
-
|
|
524
|
+
assertCli(
|
|
470
525
|
isJsonObject(element),
|
|
471
526
|
`Dataset export ${options.exportId} row ${index} is not a JSON object.`,
|
|
527
|
+
{ hint: REPORT_HINT },
|
|
472
528
|
);
|
|
473
529
|
rows.push(element);
|
|
474
530
|
}
|
|
@@ -485,7 +541,11 @@ export async function loadDataset(
|
|
|
485
541
|
if (field.internal || field.name === RAW_UPLOAD_FIELD_NAME) continue;
|
|
486
542
|
fieldNames.push(field.name);
|
|
487
543
|
}
|
|
488
|
-
|
|
544
|
+
assertCli(
|
|
545
|
+
fieldNames.length > 0,
|
|
546
|
+
`Dataset ${resolved.name} has no fields to export.`,
|
|
547
|
+
{ hint: { description: "Upload rows first with toloka dataset upload <name> <file>." } },
|
|
548
|
+
);
|
|
489
549
|
|
|
490
550
|
const started = await client.datasets.startExport({
|
|
491
551
|
datasetId: resolved.datasetId,
|
|
@@ -498,16 +558,18 @@ export async function loadDataset(
|
|
|
498
558
|
options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs },
|
|
499
559
|
);
|
|
500
560
|
} catch (error) {
|
|
501
|
-
|
|
561
|
+
assertCli(
|
|
502
562
|
!(error instanceof PotatoApiError && error.code === "wait_timeout"),
|
|
503
563
|
`Stopped waiting for dataset export ${started.id}; raise --timeout to wait longer.`,
|
|
564
|
+
{ hint: { disabled: true } },
|
|
504
565
|
);
|
|
505
566
|
throw error;
|
|
506
567
|
}
|
|
507
568
|
|
|
508
|
-
|
|
569
|
+
assertCli(
|
|
509
570
|
completed.result.downloadUrl,
|
|
510
571
|
`Dataset export ${completed.id} succeeded without a download URL.`,
|
|
572
|
+
{ hint: REPORT_HINT },
|
|
511
573
|
);
|
|
512
574
|
|
|
513
575
|
const rows = await downloadExportRows({
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { PotatoApiError } from "../errors";
|
|
2
|
+
import { TolokaCliError } from "./cli-error";
|
|
3
|
+
|
|
4
|
+
export type DeliveryOrderRecord = {
|
|
5
|
+
readonly id: string;
|
|
6
|
+
readonly name: string;
|
|
7
|
+
readonly mondayId: string;
|
|
8
|
+
readonly status: string;
|
|
9
|
+
readonly clientName: string | null;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export type DeliveryOrderListApi = {
|
|
13
|
+
readonly deliveryOrders: {
|
|
14
|
+
readonly list: () => Promise<readonly DeliveryOrderRecord[]>;
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export async function listDeliveryOrders(options: {
|
|
19
|
+
readonly client: DeliveryOrderListApi;
|
|
20
|
+
}): Promise<{ readonly output: string }> {
|
|
21
|
+
let orders: readonly DeliveryOrderRecord[];
|
|
22
|
+
try {
|
|
23
|
+
orders = await options.client.deliveryOrders.list();
|
|
24
|
+
} catch (error) {
|
|
25
|
+
if (error instanceof PotatoApiError && error.status === 403) {
|
|
26
|
+
throw new TolokaCliError("Delivery orders are available to internal teams only.", {
|
|
27
|
+
cause: error,
|
|
28
|
+
hint: {
|
|
29
|
+
description:
|
|
30
|
+
"Run toloka whoami; when it prints Internal: false, create the project without --delivery-order-id.",
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
if (orders.length === 0) {
|
|
37
|
+
return { output: "No delivery orders visible." };
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
output: [
|
|
41
|
+
"ID\tNAME\tSTATUS\tCLIENT",
|
|
42
|
+
...orders.map(
|
|
43
|
+
(order) => `${order.id}\t${order.name}\t${order.status}\t${order.clientName ?? "-"}`,
|
|
44
|
+
),
|
|
45
|
+
].join("\n"),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { TolokaEnvironment } from "./config";
|
|
2
|
+
|
|
3
|
+
const LEGACY_ENVIRONMENT_NAMES: Readonly<Record<string, string>> = {
|
|
4
|
+
POTATO_API_KEY: "TOLOKA_API_KEY",
|
|
5
|
+
POTATO_BASE_URL: "TOLOKA_BASE_URL",
|
|
6
|
+
POTATO_PROJECT_ID: "TOLOKA_PROJECT_ID",
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export const API_KEY_ENVIRONMENT_NAMES = ["TOLOKA_API_KEY", "POTATO_API_KEY"];
|
|
10
|
+
|
|
11
|
+
export type NormalizedEnvironment = {
|
|
12
|
+
readonly environment: TolokaEnvironment;
|
|
13
|
+
readonly warnings: readonly string[];
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export const normalizeEnvironment = (environment: TolokaEnvironment): NormalizedEnvironment => {
|
|
17
|
+
const normalized: Record<string, string | undefined> = { ...environment };
|
|
18
|
+
const warnings: string[] = [];
|
|
19
|
+
for (const [legacy, current] of Object.entries(LEGACY_ENVIRONMENT_NAMES)) {
|
|
20
|
+
const value = environment[legacy];
|
|
21
|
+
if (value === undefined) continue;
|
|
22
|
+
normalized[current] ??= value;
|
|
23
|
+
warnings.push(`${legacy} is deprecated and will stop being read; set ${current} instead.`);
|
|
24
|
+
}
|
|
25
|
+
return { environment: normalized, warnings };
|
|
26
|
+
};
|
package/src/cli/file-system.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import { lstat } from "node:fs/promises";
|
|
3
2
|
import { join } from "node:path";
|
|
3
|
+
import { assertCli } from "./cli-error";
|
|
4
4
|
|
|
5
5
|
export const hasErrorCode = (error: unknown, ...codes: readonly string[]) =>
|
|
6
6
|
error instanceof Error &&
|
|
@@ -29,6 +29,10 @@ export const assertSafeDestination = async (
|
|
|
29
29
|
currentPath = join(currentPath, segment);
|
|
30
30
|
const entry = await entryStats(currentPath);
|
|
31
31
|
if (entry === undefined) return;
|
|
32
|
-
|
|
32
|
+
assertCli(
|
|
33
|
+
!entry.isSymbolicLink(),
|
|
34
|
+
`Refusing to ${action} through symbolic link ${currentPath}.`,
|
|
35
|
+
{ hint: { description: "Replace the symbolic link with a regular directory, or point the command at the real path." } },
|
|
36
|
+
);
|
|
33
37
|
}
|
|
34
38
|
};
|