@toloka-platform/sdk 0.2.0-preview.2 → 0.2.0-preview.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -4
- package/dist/cli/toloka.js +162 -154
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +18 -1
- package/docs/cli-v2.md +6 -4
- package/docs/cli.md +58 -11
- package/docs/getting-started.md +3 -3
- package/docs/limitations.md +1 -1
- package/docs/testing.md +1 -1
- package/docs/troubleshooting.md +1 -1
- package/examples/face-image-collection/README.md +2 -2
- package/openapi/gen/api.ts +78 -0
- package/openapi/gen/models/baseModel.ts +1 -0
- package/openapi/gen/models/checkReport.ts +2 -1
- package/openapi/gen/models/checkReportAnyOfTwoseven.ts +5 -0
- package/openapi/gen/models/checkReportAnyOfTwosevenResultsItem.ts +6 -0
- package/openapi/gen/models/checkReportAnyOfTwosevenResultsItemVerdict.ts +2 -0
- package/openapi/gen/models/currentUser.ts +3 -0
- package/openapi/gen/models/currentUserTeamsItem.ts +6 -0
- package/openapi/gen/models/deliveryOrder.ts +9 -0
- package/openapi/gen/models/deliveryOrderClientName.ts +2 -0
- package/openapi/gen/models/getTrainingRunMetricHistoryParams.ts +4 -0
- package/openapi/gen/models/gistingTrainingConfiguration.ts +6 -2
- package/openapi/gen/models/gistingTrainingConfigurationBatchSize.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationGistTokens.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationInput.ts +6 -2
- package/openapi/gen/models/gistingTrainingConfigurationInputBatchSize.ts +2 -0
- package/openapi/gen/models/gistingTrainingConfigurationInputGistTokens.ts +2 -0
- package/openapi/gen/models/index.ts +28 -4
- package/openapi/gen/models/listDeliveryOrders200.ts +5 -0
- package/openapi/gen/models/sftTrainingConfiguration.ts +6 -3
- package/openapi/gen/models/sftTrainingConfigurationBatchSize.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationInput.ts +6 -3
- package/openapi/gen/models/sftTrainingConfigurationInputBatchSize.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationInputLoraRank.ts +2 -0
- package/openapi/gen/models/sftTrainingConfigurationLoraRank.ts +2 -0
- package/openapi/gen/models/trainingRunMetricHistory.ts +9 -0
- package/openapi/gen/models/trainingRunMetricHistoryMetric.ts +2 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItem.ts +9 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItemNonFinite.ts +2 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItemValue.ts +2 -0
- package/openapi/gen/models/trainingRunMetrics.ts +9 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItem.ts +8 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatest.ts +9 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatestNonFinite.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatestValue.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemMetric.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsUnavailableReason.ts +2 -0
- package/package.json +2 -2
- package/skills/catalog.json +55 -7
- package/skills/toloka-build/SKILL.md +35 -21
- package/skills/toloka-build/references/node-semantics.md +13 -11
- package/skills/toloka-build/references/workflow-and-cli.md +31 -20
- package/skills/toloka-fine-tuning/SKILL.md +155 -0
- package/skills/toloka-get-started/SKILL.md +70 -0
- package/skills/toloka-gisting/SKILL.md +166 -0
- package/skills/toloka-report-issue/references/report-template.md +1 -1
- package/src/authoring/index.ts +14 -0
- package/src/checks/task-ui-typecheck.ts +12 -10
- package/src/cli/analytics.ts +88 -15
- package/src/cli/auth.ts +93 -21
- package/src/cli/auto-update.ts +36 -39
- package/src/cli/check.ts +88 -39
- package/src/cli/cli-entry.ts +3 -2
- package/src/cli/cli-error.ts +43 -0
- package/src/cli/commands/auth.ts +89 -0
- package/src/cli/commands/config.ts +28 -0
- package/src/cli/commands/dataset.ts +85 -0
- package/src/cli/commands/delivery-order.ts +18 -0
- package/src/cli/commands/models.ts +121 -0
- package/src/cli/commands/pipeline.ts +131 -0
- package/src/cli/commands/project.ts +168 -0
- package/src/cli/commands/report.ts +43 -0
- package/src/cli/commands/secret.ts +40 -0
- package/src/cli/commands/skill.ts +36 -0
- package/src/cli/commands/source.ts +204 -0
- package/src/cli/commands/status.ts +17 -0
- package/src/cli/commands/training.ts +261 -0
- package/src/cli/commands/version.ts +14 -0
- package/src/cli/config.ts +7 -8
- package/src/cli/context.ts +251 -0
- package/src/cli/credentials.ts +12 -4
- package/src/cli/data.ts +96 -34
- package/src/cli/delivery-orders.ts +47 -0
- package/src/cli/environment.ts +26 -0
- package/src/cli/file-system.ts +6 -2
- package/src/cli/format-error.ts +45 -2
- package/src/cli/init.ts +14 -11
- package/src/cli/models.ts +43 -12
- package/src/cli/program.ts +120 -1347
- package/src/cli/project-context.ts +126 -43
- package/src/cli/project.ts +80 -28
- package/src/cli/report.ts +24 -9
- package/src/cli/runs.ts +33 -8
- package/src/cli/sdk-version.ts +38 -0
- package/src/cli/settings.ts +13 -5
- package/src/cli/skill.ts +67 -24
- package/src/cli/source.ts +3 -2
- package/src/cli/terminal-io.ts +110 -0
- package/src/cli/trainings.ts +242 -57
- package/src/cli/version.ts +3 -3
- package/src/client.ts +21 -9
- package/src/importer.ts +32 -2
- package/src/transport.ts +13 -1
- package/openapi/gen/models/gistingTrainingConfigurationInputRegime.ts +0 -2
- package/openapi/gen/models/gistingTrainingConfigurationRegime.ts +0 -2
- package/openapi/gen/models/sftTrainingConfigurationInputRegime.ts +0 -2
- package/openapi/gen/models/sftTrainingConfigurationRegime.ts +0 -2
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { PotatoApiError } from "../errors";
|
|
2
2
|
import { readTolokaState, TOLOKA_STATE_FILENAME, writeTolokaState } from "../state";
|
|
3
3
|
import type { TolokaState } from "../state";
|
|
4
4
|
import type { AuthPrompt } from "./auth";
|
|
@@ -6,6 +6,8 @@ import type { TolokaCliFlags, TolokaEnvironment } from "./config";
|
|
|
6
6
|
import { resolveInternalBaseUrl } from "./config";
|
|
7
7
|
import { normalizeBaseUrl, requireStoredCredentials, writeStoredCredentials } from "./credentials";
|
|
8
8
|
import type { CliEnvironment, StoredCredentials, StoredProjectAlias } from "./credentials";
|
|
9
|
+
import { apiErrorDetails } from "./format-error";
|
|
10
|
+
import { assertCli, TolokaCliError } from "./cli-error";
|
|
9
11
|
|
|
10
12
|
type ProjectRecord = { readonly id: string; readonly name: string; readonly teamId: string };
|
|
11
13
|
|
|
@@ -24,33 +26,35 @@ export type ProjectGetApi = {
|
|
|
24
26
|
};
|
|
25
27
|
};
|
|
26
28
|
|
|
29
|
+
export type AccountTeam = { readonly id: string; readonly writable: boolean; readonly internal?: boolean };
|
|
30
|
+
|
|
27
31
|
export type ProjectCreateApi = {
|
|
32
|
+
readonly me: () => Promise<{ readonly teams?: readonly AccountTeam[] }>;
|
|
28
33
|
readonly projects: {
|
|
29
34
|
readonly create: (input: {
|
|
30
35
|
readonly teamId: string;
|
|
31
36
|
readonly name?: string;
|
|
37
|
+
readonly deliveryOrderId?: string;
|
|
32
38
|
}) => Promise<ProjectRecord>;
|
|
33
|
-
readonly listAll: (params?: object) => AsyncIterable<ProjectRecord>;
|
|
34
39
|
};
|
|
35
40
|
};
|
|
36
41
|
|
|
37
42
|
export const MISSING_PROJECT_ID_MESSAGE =
|
|
38
43
|
"Missing project ID. Pass --project-id, set TOLOKA_PROJECT_ID, use a bound toloka.json, run toloka project create for a new project, or run toloka project add for an existing project.";
|
|
39
44
|
|
|
45
|
+
export function assertProjectSelected(projectId: string | undefined): asserts projectId is string {
|
|
46
|
+
assertCli(projectId, MISSING_PROJECT_ID_MESSAGE, { hint: { disabled: true } });
|
|
47
|
+
}
|
|
48
|
+
|
|
40
49
|
export const NO_PROJECT_SELECTED_MESSAGE =
|
|
41
50
|
"No project selected. Run toloka project create for a new project, toloka project add for an existing project, or cd into a checkout.";
|
|
42
51
|
|
|
43
52
|
const NETWORK_MISMATCH_MESSAGE =
|
|
44
|
-
"project add writes the machine registry; unset --api-key / --base-url /
|
|
53
|
+
"project add writes the machine registry; unset --api-key / --base-url / TOLOKA_API_KEY / TOLOKA_BASE_URL, or run login against that platform first.";
|
|
45
54
|
|
|
46
55
|
const ALIAS_PATTERN = /^[^/\s]+$/u;
|
|
47
56
|
|
|
48
|
-
export type ProjectContextSource =
|
|
49
|
-
| "flag"
|
|
50
|
-
| "toloka-project-id"
|
|
51
|
-
| "potato-project-id"
|
|
52
|
-
| "toloka.json"
|
|
53
|
-
| "registry";
|
|
57
|
+
export type ProjectContextSource = "flag" | "toloka-project-id" | "toloka.json" | "registry";
|
|
54
58
|
|
|
55
59
|
export type ProjectContext = {
|
|
56
60
|
readonly id: string;
|
|
@@ -74,9 +78,10 @@ export type ProjectIo = {
|
|
|
74
78
|
|
|
75
79
|
export const validateProjectAlias = (alias: string): string => {
|
|
76
80
|
const trimmed = alias.trim();
|
|
77
|
-
|
|
81
|
+
assertCli(
|
|
78
82
|
trimmed.length > 0 && ALIAS_PATTERN.test(trimmed),
|
|
79
83
|
"Project alias must be non-empty and contain no / or whitespace.",
|
|
84
|
+
{ hint: { description: "Use letters, digits, dashes, or underscores, for example my-project." } },
|
|
80
85
|
);
|
|
81
86
|
return trimmed;
|
|
82
87
|
};
|
|
@@ -93,7 +98,7 @@ const effectiveNetworkBaseUrl = (
|
|
|
93
98
|
stored: StoredCredentials | undefined,
|
|
94
99
|
): string | undefined =>
|
|
95
100
|
firstNonEmpty(flags?.baseUrl) ??
|
|
96
|
-
firstNonEmpty(environment?.
|
|
101
|
+
firstNonEmpty(environment?.TOLOKA_BASE_URL) ??
|
|
97
102
|
resolveInternalBaseUrl(environment ?? {}) ??
|
|
98
103
|
firstNonEmpty(stored?.baseUrl);
|
|
99
104
|
|
|
@@ -144,7 +149,6 @@ export function resolveProjectContext(
|
|
|
144
149
|
}> = [
|
|
145
150
|
{ id: firstNonEmpty(flags.projectId), source: "flag" },
|
|
146
151
|
{ id: firstNonEmpty(environment.TOLOKA_PROJECT_ID), source: "toloka-project-id" },
|
|
147
|
-
{ id: firstNonEmpty(environment.POTATO_PROJECT_ID), source: "potato-project-id" },
|
|
148
152
|
{ id: firstNonEmpty(options.state?.projectId), source: "toloka.json" },
|
|
149
153
|
{
|
|
150
154
|
id:
|
|
@@ -183,16 +187,21 @@ const assertStoredNetworkMatches = (
|
|
|
183
187
|
environment: TolokaEnvironment,
|
|
184
188
|
stored: StoredCredentials,
|
|
185
189
|
) => {
|
|
186
|
-
const flagOrEnvKey = firstNonEmpty(flags.apiKey) ?? firstNonEmpty(environment.
|
|
190
|
+
const flagOrEnvKey = firstNonEmpty(flags.apiKey) ?? firstNonEmpty(environment.TOLOKA_API_KEY);
|
|
187
191
|
const flagOrEnvBase =
|
|
188
192
|
firstNonEmpty(flags.baseUrl) ??
|
|
189
|
-
firstNonEmpty(environment.
|
|
193
|
+
firstNonEmpty(environment.TOLOKA_BASE_URL) ??
|
|
190
194
|
resolveInternalBaseUrl(environment);
|
|
191
|
-
|
|
192
|
-
|
|
195
|
+
assertCli(
|
|
196
|
+
flagOrEnvKey === undefined || flagOrEnvKey === stored.apiKey,
|
|
197
|
+
NETWORK_MISMATCH_MESSAGE,
|
|
198
|
+
{ hint: { disabled: true } },
|
|
199
|
+
);
|
|
200
|
+
assertCli(
|
|
193
201
|
flagOrEnvBase === undefined ||
|
|
194
202
|
normalizeBaseUrl(flagOrEnvBase) === normalizeBaseUrl(stored.baseUrl),
|
|
195
203
|
NETWORK_MISMATCH_MESSAGE,
|
|
204
|
+
{ hint: { disabled: true } },
|
|
196
205
|
);
|
|
197
206
|
};
|
|
198
207
|
|
|
@@ -219,9 +228,9 @@ const promptOrFail = async (
|
|
|
219
228
|
question: string,
|
|
220
229
|
missing: string,
|
|
221
230
|
): Promise<string> => {
|
|
222
|
-
|
|
231
|
+
assertCli(io?.prompt, missing, { hint: { disabled: true } });
|
|
223
232
|
const value = (await io.prompt(question)).trim();
|
|
224
|
-
|
|
233
|
+
assertCli(value.length > 0, missing, { hint: { disabled: true } });
|
|
225
234
|
return value;
|
|
226
235
|
};
|
|
227
236
|
|
|
@@ -259,27 +268,37 @@ export async function untrackProjectResource(options: {
|
|
|
259
268
|
readonly io: ProjectIo;
|
|
260
269
|
}): Promise<{ readonly output: string }> {
|
|
261
270
|
const state = await readTolokaState(options.sourceRoot);
|
|
262
|
-
|
|
271
|
+
assertCli(
|
|
272
|
+
state,
|
|
273
|
+
`No ${TOLOKA_STATE_FILENAME} found in ${options.sourceRoot}.`,
|
|
274
|
+
{ hint: { description: "Run toloka init in the source root, or cd into the checkout that holds toloka.json." } },
|
|
275
|
+
);
|
|
263
276
|
const bindings = options.type === "pipeline" ? state.pipelines : state.datasets;
|
|
264
277
|
const binding = bindings[options.name];
|
|
265
|
-
|
|
278
|
+
assertCli(
|
|
266
279
|
binding,
|
|
267
280
|
`${options.type === "pipeline" ? "Pipeline" : "Dataset"} ${options.name} is not tracked.`,
|
|
281
|
+
{ hint: { description: "Open toloka.json to see the tracked pipelines and datasets." } },
|
|
268
282
|
);
|
|
269
283
|
const id = "pipelineId" in binding ? binding.pipelineId : binding.datasetId;
|
|
270
284
|
const label = `${options.type} ${options.name} (${id})`;
|
|
271
285
|
|
|
272
286
|
if (!options.confirm) {
|
|
273
|
-
|
|
287
|
+
assertCli(
|
|
274
288
|
options.io.prompt,
|
|
275
289
|
`Refusing to untrack ${label} without confirmation. Pass --confirm after reviewing ${TOLOKA_STATE_FILENAME}.`,
|
|
290
|
+
{ hint: { disabled: true } },
|
|
276
291
|
);
|
|
277
292
|
const answer = (
|
|
278
293
|
await options.io.prompt(`Untrack ${label}? The live resource will not be deleted. [y/N] `)
|
|
279
294
|
)
|
|
280
295
|
.trim()
|
|
281
296
|
.toLowerCase();
|
|
282
|
-
|
|
297
|
+
assertCli(
|
|
298
|
+
answer === "y" || answer === "yes",
|
|
299
|
+
"Untrack canceled.",
|
|
300
|
+
{ hint: { disabled: true } },
|
|
301
|
+
);
|
|
283
302
|
}
|
|
284
303
|
|
|
285
304
|
const remaining = Object.fromEntries(
|
|
@@ -324,9 +343,10 @@ export async function addProject(options: {
|
|
|
324
343
|
|
|
325
344
|
const existing = ownProject(found.credentials.projects, alias);
|
|
326
345
|
|
|
327
|
-
|
|
346
|
+
assertCli(
|
|
328
347
|
existing === undefined || existing.id === projectId,
|
|
329
348
|
`Alias ${alias} already points at ${existing?.id}. Run toloka project remove ${alias} first.`,
|
|
349
|
+
{ hint: { disabled: true } },
|
|
330
350
|
);
|
|
331
351
|
|
|
332
352
|
await options.afterPreflight?.();
|
|
@@ -351,7 +371,11 @@ export async function switchProject(options: {
|
|
|
351
371
|
assertStoredNetworkMatches(options.flags ?? {}, options.environment, found.credentials);
|
|
352
372
|
const alias = validateProjectAlias(options.alias);
|
|
353
373
|
const project = ownProject(found.credentials.projects, alias);
|
|
354
|
-
|
|
374
|
+
assertCli(
|
|
375
|
+
project,
|
|
376
|
+
`Unknown project alias ${alias}.`,
|
|
377
|
+
{ hint: { description: "Run toloka project add <alias> <project-id> to register it, or check the alias spelling." } },
|
|
378
|
+
);
|
|
355
379
|
|
|
356
380
|
await writeRegistry({ ...found.credentials, current: alias }, options.environment);
|
|
357
381
|
const output = `Using project ${project.id} as ${alias}.`;
|
|
@@ -372,7 +396,11 @@ export async function removeProject(options: {
|
|
|
372
396
|
string,
|
|
373
397
|
StoredProjectAlias
|
|
374
398
|
>;
|
|
375
|
-
|
|
399
|
+
assertCli(
|
|
400
|
+
ownProject(projects, alias),
|
|
401
|
+
`Unknown project alias ${alias}.`,
|
|
402
|
+
{ hint: { description: "Run toloka project add <alias> <project-id> to register it, or check the alias spelling." } },
|
|
403
|
+
);
|
|
376
404
|
delete projects[alias];
|
|
377
405
|
|
|
378
406
|
const current = found.credentials.current === alias ? undefined : found.credentials.current;
|
|
@@ -406,25 +434,71 @@ export function currentProject(options: {
|
|
|
406
434
|
return project === undefined ? { output } : { output, project };
|
|
407
435
|
}
|
|
408
436
|
|
|
409
|
-
const uniqueTeamId = (projects: ReadonlyArray<{ readonly teamId: string }>): string | undefined => {
|
|
410
|
-
const teams = [...new Set(projects.map((project) => project.teamId))];
|
|
411
|
-
return teams.length === 1 ? teams[0] : undefined;
|
|
412
|
-
};
|
|
413
|
-
|
|
414
437
|
const resolveTeamId = async (
|
|
415
438
|
client: ProjectCreateApi,
|
|
416
439
|
teamId: string | undefined,
|
|
417
440
|
): Promise<string> => {
|
|
418
441
|
const explicit = firstNonEmpty(teamId);
|
|
419
442
|
if (explicit !== undefined) return explicit;
|
|
420
|
-
const
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
443
|
+
const { teams } = await client.me();
|
|
444
|
+
assertCli(
|
|
445
|
+
teams !== undefined,
|
|
446
|
+
"Missing team id. Pass --team-id. This platform does not report team memberships, so the team cannot be inferred.",
|
|
447
|
+
{
|
|
448
|
+
hint: {
|
|
449
|
+
description: "Copy the team id from the platform URL after /teams/, or update the platform.",
|
|
450
|
+
},
|
|
451
|
+
},
|
|
452
|
+
);
|
|
453
|
+
const writable = teams.filter((team) => team.writable).map((team) => team.id);
|
|
454
|
+
assertCli(
|
|
455
|
+
writable.length > 0,
|
|
456
|
+
"This account has no writable team, so it cannot create a project. Ask a team admin for write access or use toloka project add for an existing project.",
|
|
457
|
+
{ hint: { disabled: true } },
|
|
426
458
|
);
|
|
427
|
-
|
|
459
|
+
const [single] = writable;
|
|
460
|
+
assertCli(
|
|
461
|
+
single !== undefined && writable.length === 1,
|
|
462
|
+
`Missing team id. Pass --team-id. This account can create projects in several teams: ${writable.join(", ")}.`,
|
|
463
|
+
{ hint: { disabled: true } },
|
|
464
|
+
);
|
|
465
|
+
return single;
|
|
466
|
+
};
|
|
467
|
+
|
|
468
|
+
const DELIVERY_ORDER_HINT =
|
|
469
|
+
"Internal teams attach a Sunday delivery order to every project: run toloka delivery-order list, then re-run toloka project create with --delivery-order-id <id>.";
|
|
470
|
+
|
|
471
|
+
const DELIVERY_ORDER_NOT_ALLOWED_HINT =
|
|
472
|
+
"The owning team is not internal, so re-run toloka project create without --delivery-order-id. toloka whoami marks internal teams (internal) on its Teams: line.";
|
|
473
|
+
|
|
474
|
+
const deliveryOrderHint = (
|
|
475
|
+
input: Parameters<ProjectCreateApi["projects"]["create"]>[0],
|
|
476
|
+
details: string,
|
|
477
|
+
): string | undefined => {
|
|
478
|
+
if (!details.includes("deliveryOrderId")) return undefined;
|
|
479
|
+
if (input.deliveryOrderId === undefined && details.includes("required")) return DELIVERY_ORDER_HINT;
|
|
480
|
+
if (input.deliveryOrderId !== undefined && details.includes("only be set")) {
|
|
481
|
+
return DELIVERY_ORDER_NOT_ALLOWED_HINT;
|
|
482
|
+
}
|
|
483
|
+
return undefined;
|
|
484
|
+
};
|
|
485
|
+
|
|
486
|
+
const createLiveProject = async (
|
|
487
|
+
client: ProjectCreateApi,
|
|
488
|
+
input: Parameters<ProjectCreateApi["projects"]["create"]>[0],
|
|
489
|
+
): Promise<ProjectRecord> => {
|
|
490
|
+
try {
|
|
491
|
+
return await client.projects.create(input);
|
|
492
|
+
} catch (error) {
|
|
493
|
+
if (error instanceof PotatoApiError && error.status === 400) {
|
|
494
|
+
const details = apiErrorDetails(error);
|
|
495
|
+
const hint = details === undefined ? undefined : deliveryOrderHint(input, details);
|
|
496
|
+
if (hint !== undefined) {
|
|
497
|
+
throw new TolokaCliError(error.message, { cause: error, hint: { description: hint } });
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
throw error;
|
|
501
|
+
}
|
|
428
502
|
};
|
|
429
503
|
|
|
430
504
|
const bindCheckout = async (
|
|
@@ -442,12 +516,17 @@ const bindCheckout = async (
|
|
|
442
516
|
});
|
|
443
517
|
} catch (error) {
|
|
444
518
|
const reason = error instanceof Error ? error.message : String(error);
|
|
445
|
-
throw new
|
|
519
|
+
throw new TolokaCliError(
|
|
446
520
|
[
|
|
447
521
|
`Created project ${created.id} as ${alias}.`,
|
|
448
522
|
`Failed to write toloka.json: ${reason}`,
|
|
449
523
|
].join("\n"),
|
|
450
|
-
{
|
|
524
|
+
{
|
|
525
|
+
cause: error,
|
|
526
|
+
hint: {
|
|
527
|
+
description: `The project exists. Make ${sourceRoot} writable, then run toloka --project-id ${created.id} init to bind it.`,
|
|
528
|
+
},
|
|
529
|
+
},
|
|
451
530
|
);
|
|
452
531
|
}
|
|
453
532
|
return undefined;
|
|
@@ -464,6 +543,7 @@ export async function createProject(options: {
|
|
|
464
543
|
readonly alias?: string;
|
|
465
544
|
readonly name?: string;
|
|
466
545
|
readonly teamId?: string;
|
|
546
|
+
readonly deliveryOrderId?: string;
|
|
467
547
|
readonly sourceRoot?: string;
|
|
468
548
|
readonly afterPreflight?: () => Promise<void>;
|
|
469
549
|
}): Promise<{ readonly output: string; readonly projectId: string; readonly alias: string }> {
|
|
@@ -478,9 +558,10 @@ export async function createProject(options: {
|
|
|
478
558
|
"Missing project alias. Pass an alias or run this in a terminal.",
|
|
479
559
|
)),
|
|
480
560
|
);
|
|
481
|
-
|
|
561
|
+
assertCli(
|
|
482
562
|
found.credentials.projects?.[alias] === undefined,
|
|
483
563
|
`Alias ${alias} already exists. Run toloka project remove ${alias} first.`,
|
|
564
|
+
{ hint: { disabled: true } },
|
|
484
565
|
);
|
|
485
566
|
|
|
486
567
|
const existingState =
|
|
@@ -489,9 +570,11 @@ export async function createProject(options: {
|
|
|
489
570
|
await options.afterPreflight?.();
|
|
490
571
|
|
|
491
572
|
const teamId = await resolveTeamId(options.client, options.teamId);
|
|
492
|
-
const
|
|
573
|
+
const deliveryOrderId = firstNonEmpty(options.deliveryOrderId);
|
|
574
|
+
const created = await createLiveProject(options.client, {
|
|
493
575
|
teamId,
|
|
494
576
|
name: firstNonEmpty(options.name) ?? alias,
|
|
577
|
+
...(deliveryOrderId === undefined ? {} : { deliveryOrderId }),
|
|
495
578
|
});
|
|
496
579
|
|
|
497
580
|
try {
|
|
@@ -501,12 +584,12 @@ export async function createProject(options: {
|
|
|
501
584
|
);
|
|
502
585
|
} catch (error) {
|
|
503
586
|
const reason = error instanceof Error ? error.message : String(error);
|
|
504
|
-
throw new
|
|
587
|
+
throw new TolokaCliError(
|
|
505
588
|
[
|
|
506
589
|
`Created ${created.id} but failed to update the local registry: ${reason}`,
|
|
507
590
|
`Run: toloka project add ${alias} ${created.id}`,
|
|
508
591
|
].join("\n"),
|
|
509
|
-
{ cause: error },
|
|
592
|
+
{ cause: error, hint: { disabled: true } },
|
|
510
593
|
);
|
|
511
594
|
}
|
|
512
595
|
|
package/src/cli/project.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { assert } from "es-toolkit";
|
|
2
1
|
import { mkdir, rm, stat, writeFile } from "node:fs/promises";
|
|
3
2
|
import { dirname, join, resolve } from "node:path";
|
|
4
3
|
import type {
|
|
@@ -12,10 +11,11 @@ import type { Dataset, DatasetField, Pipeline, PotatoClient } from "../client";
|
|
|
12
11
|
import { generatePipelineSourceTree } from "../importer";
|
|
13
12
|
import { readTolokaState, TOLOKA_STATE_FILENAME, writeTolokaState } from "../state";
|
|
14
13
|
import type { TolokaState } from "../state";
|
|
15
|
-
import {
|
|
14
|
+
import { assertProjectSelected } from "./project-context";
|
|
16
15
|
import { collectGraphSecretNames } from "./secrets";
|
|
17
16
|
import { loadProjectSource } from "./source";
|
|
18
17
|
import type { ProjectPipelineSource, ProjectSource } from "./source";
|
|
18
|
+
import { assertCli, REPORT_HINT } from "./cli-error";
|
|
19
19
|
|
|
20
20
|
export type PipelinePlanChange = {
|
|
21
21
|
readonly kind: "node" | "connection" | "field" | "layout";
|
|
@@ -296,9 +296,10 @@ export const desiredDatasets = (source: ProjectSource): readonly DesiredDataset[
|
|
|
296
296
|
}
|
|
297
297
|
for (const field of pipeline.definition.graph.fields) {
|
|
298
298
|
const existing = fields.get(field.name);
|
|
299
|
-
|
|
299
|
+
assertCli(
|
|
300
300
|
existing === undefined || valueKey(existing.jsonSchema) === valueKey(field.jsonSchema),
|
|
301
301
|
`Dataset "${pipeline.definition.dataset}" has conflicting schemas for field "${field.name}".`,
|
|
302
|
+
{ hint: { description: "Give the field one JSON schema across all pipelines that share the dataset." } },
|
|
302
303
|
);
|
|
303
304
|
if (existing === undefined) fields.set(field.name, field);
|
|
304
305
|
}
|
|
@@ -314,13 +315,15 @@ export const desiredDatasets = (source: ProjectSource): readonly DesiredDataset[
|
|
|
314
315
|
const validateSource = (source: ProjectSource) => {
|
|
315
316
|
const sourceDirectories = new Set<string>();
|
|
316
317
|
for (const pipeline of source.pipelines) {
|
|
317
|
-
|
|
318
|
+
assertCli(
|
|
318
319
|
sourceDirectoryPattern.test(pipeline.sourceDirectory),
|
|
319
320
|
`Invalid pipeline source directory ${pipeline.sourceDirectory}`,
|
|
321
|
+
{ hint: { description: "Use a relative path without backslashes, // or . and .. segments." } },
|
|
320
322
|
);
|
|
321
|
-
|
|
323
|
+
assertCli(
|
|
322
324
|
!sourceDirectories.has(pipeline.sourceDirectory),
|
|
323
325
|
`Pipeline source directory ${pipeline.sourceDirectory} is declared more than once`,
|
|
326
|
+
{ hint: { description: "Give each pipeline its own source directory." } },
|
|
324
327
|
);
|
|
325
328
|
sourceDirectories.add(pipeline.sourceDirectory);
|
|
326
329
|
validatePipelineDefinition(pipeline.definition);
|
|
@@ -338,14 +341,19 @@ const liveDatasetFor = (
|
|
|
338
341
|
? undefined
|
|
339
342
|
: datasets.find((dataset) => dataset.id === binding.datasetId);
|
|
340
343
|
if (bound !== undefined) {
|
|
341
|
-
|
|
344
|
+
assertCli(
|
|
342
345
|
bound.name === name,
|
|
343
346
|
`Dataset ${bound.id} is bound as "${name}" but is named "${bound.name}".`,
|
|
347
|
+
{ hint: { description: "Rename the dataset in source to match the platform, or run toloka project untrack dataset <name> and bind it again with toloka apply." } },
|
|
344
348
|
);
|
|
345
349
|
return bound;
|
|
346
350
|
}
|
|
347
351
|
const matching = datasets.filter((dataset) => dataset.name === name);
|
|
348
|
-
|
|
352
|
+
assertCli(
|
|
353
|
+
matching.length <= 1,
|
|
354
|
+
`Project has multiple datasets named "${name}".`,
|
|
355
|
+
{ hint: { description: "Rename the duplicate datasets in the platform, or reference the dataset by id." } },
|
|
356
|
+
);
|
|
349
357
|
return matching[0];
|
|
350
358
|
};
|
|
351
359
|
|
|
@@ -360,7 +368,11 @@ const livePipelineFor = (
|
|
|
360
368
|
: pipelines.find((pipeline) => pipeline.id === binding.pipelineId);
|
|
361
369
|
if (bound !== undefined) return bound;
|
|
362
370
|
const matching = pipelines.filter((pipeline) => pipeline.name === source.name);
|
|
363
|
-
|
|
371
|
+
assertCli(
|
|
372
|
+
matching.length <= 1,
|
|
373
|
+
`Project has multiple pipelines named "${source.name}".`,
|
|
374
|
+
{ hint: { description: "Rename the duplicate pipelines in the platform, then re-run the command." } },
|
|
375
|
+
);
|
|
364
376
|
return matching[0];
|
|
365
377
|
};
|
|
366
378
|
|
|
@@ -542,7 +554,7 @@ const inspectProject = async (
|
|
|
542
554
|
validateSource(source);
|
|
543
555
|
const state = await readTolokaState(options.sourceRoot);
|
|
544
556
|
const projectId = options.projectId ?? state?.projectId;
|
|
545
|
-
|
|
557
|
+
assertProjectSelected(projectId);
|
|
546
558
|
|
|
547
559
|
const bindings = state?.projectId === projectId ? state : undefined;
|
|
548
560
|
const requiredSecrets = new Set(
|
|
@@ -627,7 +639,11 @@ export async function importProject(
|
|
|
627
639
|
const datasetIds = new Set<string>();
|
|
628
640
|
for (const snapshot of snapshots) {
|
|
629
641
|
const rootNode = snapshot.graph.nodes.find((node) => node.type === "root");
|
|
630
|
-
|
|
642
|
+
assertCli(
|
|
643
|
+
rootNode?.settings.datasetId,
|
|
644
|
+
"Every imported pipeline must have a bound dataset.",
|
|
645
|
+
{ hint: { description: "Bind a dataset to the pipeline in the platform, then re-run toloka import." } },
|
|
646
|
+
);
|
|
631
647
|
datasetIds.add(rootNode.settings.datasetId);
|
|
632
648
|
}
|
|
633
649
|
const projectDatasets = await collectAsync(
|
|
@@ -636,7 +652,11 @@ export async function importProject(
|
|
|
636
652
|
const datasets: Dataset[] = [];
|
|
637
653
|
for (const datasetId of [...datasetIds].toSorted()) {
|
|
638
654
|
const dataset = projectDatasets.find((candidate) => candidate.id === datasetId);
|
|
639
|
-
|
|
655
|
+
assertCli(
|
|
656
|
+
dataset,
|
|
657
|
+
`Project ${options.projectId} has no dataset ${datasetId}.`,
|
|
658
|
+
{ hint: { description: "Check toloka dataset list; the pipeline's dataset binding must point at a dataset of this project." } },
|
|
659
|
+
);
|
|
640
660
|
datasets.push(dataset);
|
|
641
661
|
}
|
|
642
662
|
const datasetsById = new Map<string, Dataset>(datasets.map((dataset) => [dataset.id, dataset]));
|
|
@@ -646,22 +666,32 @@ export async function importProject(
|
|
|
646
666
|
const stateDatasets: Record<string, { readonly datasetId: string }> = {};
|
|
647
667
|
for (const [index, pipeline] of pipelines.entries()) {
|
|
648
668
|
const snapshot = snapshots[index];
|
|
649
|
-
|
|
669
|
+
assertCli(snapshot, `Pipeline ${pipeline.id} has no graph snapshot.`, { hint: REPORT_HINT });
|
|
650
670
|
const rootNode = snapshot.graph.nodes.find((node) => node.type === "root");
|
|
651
|
-
|
|
671
|
+
assertCli(
|
|
672
|
+
rootNode?.settings.datasetId,
|
|
673
|
+
`Pipeline ${pipeline.id} has no dataset binding.`,
|
|
674
|
+
{ hint: { description: "Bind a dataset to the pipeline in the platform, then re-run toloka import." } },
|
|
675
|
+
);
|
|
652
676
|
const dataset = datasetsById.get(rootNode.settings.datasetId);
|
|
653
|
-
|
|
677
|
+
assertCli(
|
|
654
678
|
dataset,
|
|
655
679
|
`Pipeline ${pipeline.id} references unknown dataset ${rootNode.settings.datasetId}.`,
|
|
680
|
+
{ hint: { description: "Check toloka dataset list; the pipeline's dataset binding must point at a dataset of this project." } },
|
|
656
681
|
);
|
|
657
682
|
const existingDatasetBinding = stateDatasets[dataset.name];
|
|
658
|
-
|
|
683
|
+
assertCli(
|
|
659
684
|
existingDatasetBinding === undefined || existingDatasetBinding.datasetId === dataset.id,
|
|
660
685
|
`Project has multiple referenced datasets named "${dataset.name}".`,
|
|
686
|
+
{ hint: { description: "Rename the duplicate datasets in the platform, then re-run toloka import." } },
|
|
661
687
|
);
|
|
662
688
|
stateDatasets[dataset.name] = { datasetId: dataset.id };
|
|
663
689
|
const sourceDirectory = directories.get(pipeline.id);
|
|
664
|
-
|
|
690
|
+
assertCli(
|
|
691
|
+
sourceDirectory,
|
|
692
|
+
`Pipeline ${pipeline.id} has no source directory.`,
|
|
693
|
+
{ hint: REPORT_HINT },
|
|
694
|
+
);
|
|
665
695
|
statePipelines[sourceDirectory] = { pipelineId: pipeline.id };
|
|
666
696
|
for (const [relativePath, source] of Object.entries(
|
|
667
697
|
generatePipelineSourceTree(snapshot.graph, { dataset: dataset.name }),
|
|
@@ -678,9 +708,10 @@ export async function importProject(
|
|
|
678
708
|
} catch (error) {
|
|
679
709
|
if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error;
|
|
680
710
|
}
|
|
681
|
-
|
|
711
|
+
assertCli(
|
|
682
712
|
!pipelinesRootExists || options.force === true,
|
|
683
713
|
`Folder already exists: ${pipelinesRoot}. Pass --force to replace it.`,
|
|
714
|
+
{ hint: { disabled: true } },
|
|
684
715
|
);
|
|
685
716
|
if (pipelinesRootExists) await rm(pipelinesRoot, { recursive: true, force: true });
|
|
686
717
|
|
|
@@ -725,7 +756,7 @@ export async function projectStatus(
|
|
|
725
756
|
options: PlanProjectOptions,
|
|
726
757
|
): Promise<ProjectStatusResult> {
|
|
727
758
|
const projectId = options.projectId;
|
|
728
|
-
|
|
759
|
+
assertProjectSelected(projectId);
|
|
729
760
|
const live = await client.projects.get({ projectId });
|
|
730
761
|
const inspection = await inspectProject(client, { ...options, listAllSecrets: true });
|
|
731
762
|
const pending = inspection.changes.length;
|
|
@@ -751,24 +782,28 @@ export async function applyProject(
|
|
|
751
782
|
options: ApplyProjectOptions,
|
|
752
783
|
): Promise<ApplyProjectResult> {
|
|
753
784
|
const inspection = await inspectProject(client, options);
|
|
754
|
-
|
|
785
|
+
assertCli(
|
|
755
786
|
inspection.removedPipelines.length === 0,
|
|
756
787
|
`Refusing to delete pipelines absent from source: ${inspection.removedPipelines.join(", ")}.`,
|
|
788
|
+
{ hint: { description: "Restore the pipeline in source, or run toloka project untrack pipeline <name> to stop managing it." } },
|
|
757
789
|
);
|
|
758
|
-
|
|
790
|
+
assertCli(
|
|
759
791
|
inspection.removedDatasets.length === 0,
|
|
760
792
|
`Refusing to delete datasets absent from source: ${inspection.removedDatasets.join(", ")}. Dataset items are never deleted.`,
|
|
793
|
+
{ hint: { description: "Restore the dataset in source, or run toloka project untrack dataset <name> to stop managing it." } },
|
|
761
794
|
);
|
|
762
|
-
|
|
795
|
+
assertCli(
|
|
763
796
|
inspection.missingSecrets.length === 0,
|
|
764
797
|
`Required secrets are missing: ${inspection.missingSecrets.join(", ")}. Run toloka secret set <name> before apply.`,
|
|
798
|
+
{ hint: { disabled: true } },
|
|
765
799
|
);
|
|
766
800
|
const conflictingFields = inspection.datasets.flatMap((dataset) =>
|
|
767
801
|
dataset.conflictingFields.map((field) => `${dataset.name}.${field.name}`),
|
|
768
802
|
);
|
|
769
|
-
|
|
803
|
+
assertCli(
|
|
770
804
|
conflictingFields.length === 0,
|
|
771
805
|
`Dataset field schemas conflict with source: ${conflictingFields.join(", ")}. Existing fields are never rewritten.`,
|
|
806
|
+
{ hint: { description: "Add the changed columns under new field names, or create a new dataset for the new schema." } },
|
|
772
807
|
);
|
|
773
808
|
|
|
774
809
|
let writes = 0;
|
|
@@ -798,9 +833,10 @@ export async function applyProject(
|
|
|
798
833
|
const pipelineIds = new Map<string, string>();
|
|
799
834
|
for (const pipelineInspection of inspection.pipelines) {
|
|
800
835
|
const datasetId = datasetIds.get(pipelineInspection.source.definition.dataset);
|
|
801
|
-
|
|
836
|
+
assertCli(
|
|
802
837
|
datasetId,
|
|
803
838
|
`Dataset ${pipelineInspection.source.definition.dataset} has no resolved ID.`,
|
|
839
|
+
{ hint: REPORT_HINT },
|
|
804
840
|
);
|
|
805
841
|
const desiredGraph = resolvePipelineDefinition(pipelineInspection.source.definition, {
|
|
806
842
|
datasetId,
|
|
@@ -819,7 +855,11 @@ export async function applyProject(
|
|
|
819
855
|
pipelineIds.set(pipelineInspection.source.sourceDirectory, pipelineInspection.pipeline.id);
|
|
820
856
|
if (pipelineInspection.graphChanges.length === 0) continue;
|
|
821
857
|
const snapshot = pipelineInspection.snapshot;
|
|
822
|
-
|
|
858
|
+
assertCli(
|
|
859
|
+
snapshot,
|
|
860
|
+
`Pipeline ${pipelineInspection.pipeline.id} has no graph snapshot.`,
|
|
861
|
+
{ hint: REPORT_HINT },
|
|
862
|
+
);
|
|
823
863
|
await client.pipelines.replaceGraph({
|
|
824
864
|
pipelineId: pipelineInspection.pipeline.id,
|
|
825
865
|
graph: graphWithLiveState(desiredGraph, snapshot.graph),
|
|
@@ -833,14 +873,18 @@ export async function applyProject(
|
|
|
833
873
|
pipelines: Object.fromEntries(
|
|
834
874
|
inspection.source.pipelines.map((pipeline) => {
|
|
835
875
|
const pipelineId = pipelineIds.get(pipeline.sourceDirectory);
|
|
836
|
-
|
|
876
|
+
assertCli(
|
|
877
|
+
pipelineId,
|
|
878
|
+
`Pipeline ${pipeline.sourceDirectory} has no resolved ID.`,
|
|
879
|
+
{ hint: REPORT_HINT },
|
|
880
|
+
);
|
|
837
881
|
return [pipeline.sourceDirectory, { pipelineId }];
|
|
838
882
|
}),
|
|
839
883
|
),
|
|
840
884
|
datasets: Object.fromEntries(
|
|
841
885
|
inspection.datasets.map((dataset) => {
|
|
842
886
|
const datasetId = datasetIds.get(dataset.name);
|
|
843
|
-
|
|
887
|
+
assertCli(datasetId, `Dataset ${dataset.name} has no resolved ID.`, { hint: REPORT_HINT });
|
|
844
888
|
return [dataset.name, { datasetId }];
|
|
845
889
|
}),
|
|
846
890
|
),
|
|
@@ -957,8 +1001,16 @@ export async function validateProject(
|
|
|
957
1001
|
const lines = results.flatMap((result) => {
|
|
958
1002
|
if (result.skipped !== undefined)
|
|
959
1003
|
return [`- ${result.sourceDirectory}: skipped (${result.skipped})`];
|
|
960
|
-
|
|
961
|
-
|
|
1004
|
+
assertCli(
|
|
1005
|
+
result.report,
|
|
1006
|
+
`Pipeline ${result.sourceDirectory} is missing a validation report`,
|
|
1007
|
+
{ hint: REPORT_HINT },
|
|
1008
|
+
);
|
|
1009
|
+
assertCli(
|
|
1010
|
+
result.status,
|
|
1011
|
+
`Pipeline ${result.sourceDirectory} is missing a validation status`,
|
|
1012
|
+
{ hint: REPORT_HINT },
|
|
1013
|
+
);
|
|
962
1014
|
const header = `- ${result.sourceDirectory}: ${result.status.replace("_", "-")}`;
|
|
963
1015
|
return [header, ...formatValidationIssues(result.report)];
|
|
964
1016
|
});
|