@toloka-platform/sdk 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/toloka.js +156 -137
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +2 -0
- package/docs/cli.md +13 -3
- package/openapi/gen/api.ts +30 -0
- package/openapi/gen/models/gistingTrainingConfigurationGistTokens.ts +1 -1
- package/openapi/gen/models/gistingTrainingConfigurationInputGistTokens.ts +1 -1
- package/openapi/gen/models/index.ts +3 -0
- package/openapi/gen/models/listAudienceSpecializations200.ts +7 -0
- package/openapi/gen/models/listAudienceSpecializationsParams.ts +5 -0
- package/openapi/gen/models/specialization.ts +6 -0
- package/openapi/gen/models/trainingCostEstimateOneOfTwoseven.ts +0 -2
- package/openapi/gen/models/trainingRunQuote.ts +0 -2
- package/package.json +1 -1
- package/skills/catalog.json +8 -8
- package/skills/toloka-build/references/node-semantics.md +2 -2
- package/skills/toloka-build/references/workflow-and-cli.md +6 -3
- package/skills/toloka-fine-tuning/SKILL.md +2 -0
- package/skills/toloka-gisting/SKILL.md +3 -1
- package/src/checks/task-ui-typecheck.ts +4 -1
- package/src/cli/analytics.ts +1 -0
- package/src/cli/audience.ts +31 -0
- package/src/cli/auto-update.ts +363 -24
- package/src/cli/check.ts +51 -52
- package/src/cli/cli-entry.ts +31 -0
- package/src/cli/commands/audience.ts +22 -0
- package/src/cli/commands/training.ts +10 -2
- package/src/cli/context.ts +7 -2
- package/src/cli/init.ts +7 -2
- package/src/cli/models.ts +1 -21
- package/src/cli/native-runtime.ts +7 -0
- package/src/cli/program.ts +2 -0
- package/src/cli/skill.ts +34 -26
- package/src/cli/table.ts +21 -0
- package/src/cli/trainings.ts +1 -22
- package/src/cli/version.ts +12 -1
- package/src/client.ts +17 -0
- package/src/index.ts +1 -0
package/src/cli/check.ts
CHANGED
|
@@ -23,7 +23,8 @@ import {
|
|
|
23
23
|
type TestEvidence,
|
|
24
24
|
type TestEvidenceKind,
|
|
25
25
|
} from "../testing/test-evidence";
|
|
26
|
-
import {
|
|
26
|
+
import { cliRelaunchCommand } from "./cli-entry";
|
|
27
|
+
import { bunSubprocessEnvironment } from "./native-runtime";
|
|
27
28
|
import { API_KEY_ENVIRONMENT_NAMES } from "./environment";
|
|
28
29
|
import { loadProjectSource, type ProjectSource } from "./source";
|
|
29
30
|
import { assertCli, REPORT_HINT, TolokaCliError } from "./cli-error";
|
|
@@ -160,16 +161,15 @@ export const collectOutput = async (stream: NodeJS.ReadableStream): Promise<stri
|
|
|
160
161
|
return truncated ? `${output}\n…output truncated` : output;
|
|
161
162
|
};
|
|
162
163
|
|
|
163
|
-
const
|
|
164
|
+
const runCommand = async (
|
|
164
165
|
sourceRoot: string,
|
|
165
|
-
|
|
166
|
+
executable: string,
|
|
166
167
|
args: readonly string[],
|
|
167
|
-
environment: NodeJS.ProcessEnv
|
|
168
|
+
environment: NodeJS.ProcessEnv,
|
|
168
169
|
signal?: AbortSignal,
|
|
169
|
-
executable: string = process.execPath,
|
|
170
170
|
): Promise<ToolRunResult> => {
|
|
171
171
|
signal?.throwIfAborted();
|
|
172
|
-
const child = spawn(executable,
|
|
172
|
+
const child = spawn(executable, args, {
|
|
173
173
|
cwd: sourceRoot,
|
|
174
174
|
env: environment,
|
|
175
175
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -208,11 +208,11 @@ const runTypeScript = async (
|
|
|
208
208
|
signal?: AbortSignal,
|
|
209
209
|
): Promise<ToolRunResult> => {
|
|
210
210
|
try {
|
|
211
|
-
return await
|
|
211
|
+
return await runCommand(
|
|
212
212
|
sourceRoot,
|
|
213
|
-
|
|
214
|
-
["--noEmit"],
|
|
215
|
-
environment,
|
|
213
|
+
process.execPath,
|
|
214
|
+
[resolveProjectModule(sourceRoot, "typescript", "bin/tsc"), "--noEmit"],
|
|
215
|
+
bunSubprocessEnvironment(environment),
|
|
216
216
|
signal,
|
|
217
217
|
);
|
|
218
218
|
} catch (error) {
|
|
@@ -247,11 +247,15 @@ const runVitest = async ({
|
|
|
247
247
|
? typedTestNamePattern(fingerprint, environment[TEST_NODE_FILTER_ENV])
|
|
248
248
|
: undefined;
|
|
249
249
|
return classifyVitestRun(
|
|
250
|
-
await
|
|
250
|
+
await runCommand(
|
|
251
251
|
sourceRoot,
|
|
252
|
-
|
|
253
|
-
[
|
|
254
|
-
|
|
252
|
+
process.execPath,
|
|
253
|
+
[
|
|
254
|
+
resolveProjectModule(sourceRoot, "vitest", "vitest.mjs"),
|
|
255
|
+
"run",
|
|
256
|
+
...(focusedPattern === undefined ? [] : ["--testNamePattern", focusedPattern]),
|
|
257
|
+
],
|
|
258
|
+
bunSubprocessEnvironment(environment),
|
|
255
259
|
signal,
|
|
256
260
|
),
|
|
257
261
|
);
|
|
@@ -804,11 +808,9 @@ const parseFreshCheckDiagnostic = (value: unknown): CheckDiagnostic => {
|
|
|
804
808
|
const line = Reflect.get(value, "line");
|
|
805
809
|
const column = Reflect.get(value, "column");
|
|
806
810
|
const code = Reflect.get(value, "code");
|
|
807
|
-
assertCli(
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
{ hint: REPORT_HINT },
|
|
811
|
-
);
|
|
811
|
+
assertCli(typeof message === "string", "Fresh check diagnostic has no message", {
|
|
812
|
+
hint: REPORT_HINT,
|
|
813
|
+
});
|
|
812
814
|
assertCli(
|
|
813
815
|
severity === undefined || severity === "error" || severity === "warning",
|
|
814
816
|
"Fresh check diagnostic has invalid severity",
|
|
@@ -845,11 +847,9 @@ const parseFreshCheckDiagnostic = (value: unknown): CheckDiagnostic => {
|
|
|
845
847
|
};
|
|
846
848
|
|
|
847
849
|
const parseFreshCheckResultItem = (value: unknown): CheckResult => {
|
|
848
|
-
assertCli(
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
{ hint: REPORT_HINT },
|
|
852
|
-
);
|
|
850
|
+
assertCli(typeof value === "object" && value !== null, "Fresh check returned an invalid result", {
|
|
851
|
+
hint: REPORT_HINT,
|
|
852
|
+
});
|
|
853
853
|
const scope = Reflect.get(value, "scope");
|
|
854
854
|
const kind = Reflect.get(value, "kind");
|
|
855
855
|
const status = Reflect.get(value, "status");
|
|
@@ -861,11 +861,9 @@ const parseFreshCheckResultItem = (value: unknown): CheckResult => {
|
|
|
861
861
|
assertCli(isCheckScope(scope), "Fresh check returned an invalid scope", { hint: REPORT_HINT });
|
|
862
862
|
assertCli(isCheckKind(kind), "Fresh check returned an invalid kind", { hint: REPORT_HINT });
|
|
863
863
|
assertCli(isCheckStatus(status), "Fresh check returned an invalid status", { hint: REPORT_HINT });
|
|
864
|
-
assertCli(
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
{ hint: REPORT_HINT },
|
|
868
|
-
);
|
|
864
|
+
assertCli(Array.isArray(diagnostics), "Fresh check returned invalid diagnostics", {
|
|
865
|
+
hint: REPORT_HINT,
|
|
866
|
+
});
|
|
869
867
|
assertCli(
|
|
870
868
|
sourceDirectory === undefined || typeof sourceDirectory === "string",
|
|
871
869
|
"Fresh check returned an invalid source directory",
|
|
@@ -899,11 +897,9 @@ const parseFreshCheckResultItem = (value: unknown): CheckResult => {
|
|
|
899
897
|
};
|
|
900
898
|
|
|
901
899
|
const parseFreshCheckResult = (value: unknown, json: boolean): CheckProjectResult => {
|
|
902
|
-
assertCli(
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
{ hint: REPORT_HINT },
|
|
906
|
-
);
|
|
900
|
+
assertCli(typeof value === "object" && value !== null, "Fresh check returned invalid JSON", {
|
|
901
|
+
hint: REPORT_HINT,
|
|
902
|
+
});
|
|
907
903
|
const exitCode = Reflect.get(value, "exitCode");
|
|
908
904
|
const rawResults = Reflect.get(value, "results");
|
|
909
905
|
assertCli(
|
|
@@ -973,30 +969,33 @@ export const readFreshCheckResult = async (
|
|
|
973
969
|
}
|
|
974
970
|
};
|
|
975
971
|
export const runFreshCheck = async (options: CheckProjectOptions): Promise<CheckProjectResult> => {
|
|
976
|
-
const entry = resolveCliEntry();
|
|
977
972
|
const resultDirectory = await mkdtemp(join(tmpdir(), "toloka-check-watch-result-"));
|
|
978
973
|
const resultPath = join(resultDirectory, "result.json");
|
|
974
|
+
const environment = {
|
|
975
|
+
...(process.env.PATH === undefined ? {} : { PATH: process.env.PATH }),
|
|
976
|
+
...(options.environment ?? process.env),
|
|
977
|
+
[CHECK_RESULT_FILE_ENV]: resultPath,
|
|
978
|
+
};
|
|
979
|
+
const command = cliRelaunchCommand(
|
|
980
|
+
[
|
|
981
|
+
"--source-root",
|
|
982
|
+
options.sourceRoot,
|
|
983
|
+
"check",
|
|
984
|
+
...(options.pipeline === undefined ? [] : [options.pipeline]),
|
|
985
|
+
...(options.node === undefined ? [] : ["--node", options.node]),
|
|
986
|
+
"--json",
|
|
987
|
+
],
|
|
988
|
+
environment,
|
|
989
|
+
);
|
|
979
990
|
try {
|
|
980
991
|
let run: ToolRunResult;
|
|
981
992
|
try {
|
|
982
|
-
run = await
|
|
993
|
+
run = await runCommand(
|
|
983
994
|
options.sourceRoot,
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
options.sourceRoot,
|
|
988
|
-
"check",
|
|
989
|
-
...(options.pipeline === undefined ? [] : [options.pipeline]),
|
|
990
|
-
...(options.node === undefined ? [] : ["--node", options.node]),
|
|
991
|
-
"--json",
|
|
992
|
-
],
|
|
993
|
-
{
|
|
994
|
-
...(process.env.PATH === undefined ? {} : { PATH: process.env.PATH }),
|
|
995
|
-
...(options.environment ?? process.env),
|
|
996
|
-
[CHECK_RESULT_FILE_ENV]: resultPath,
|
|
997
|
-
},
|
|
995
|
+
command.executable,
|
|
996
|
+
command.args,
|
|
997
|
+
command.environment,
|
|
998
998
|
options.signal,
|
|
999
|
-
"bun",
|
|
1000
999
|
);
|
|
1001
1000
|
} catch (error) {
|
|
1002
1001
|
throw new FreshCheckProcessError(
|
|
@@ -1004,7 +1003,7 @@ export const runFreshCheck = async (options: CheckProjectOptions): Promise<Check
|
|
|
1004
1003
|
`Unable to start the local check process: ${error instanceof Error ? error.message : String(error)}`,
|
|
1005
1004
|
{
|
|
1006
1005
|
cause: error,
|
|
1007
|
-
hint: { description: "
|
|
1006
|
+
hint: { description: "Reinstall the Toloka CLI, then re-run toloka check." },
|
|
1008
1007
|
},
|
|
1009
1008
|
);
|
|
1010
1009
|
}
|
package/src/cli/cli-entry.ts
CHANGED
|
@@ -1,8 +1,20 @@
|
|
|
1
1
|
import { existsSync, realpathSync } from "node:fs";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
3
|
import { assertCli } from "./cli-error";
|
|
4
|
+
import { isNativeCli } from "./native-runtime";
|
|
4
5
|
|
|
5
6
|
const CLI_ENTRY = join("bin", "toloka.js");
|
|
7
|
+
export type CliRelaunchCommand = {
|
|
8
|
+
readonly executable: string;
|
|
9
|
+
readonly args: readonly string[];
|
|
10
|
+
readonly environment: NodeJS.ProcessEnv;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export type CliRelaunchOptions = {
|
|
14
|
+
readonly native?: boolean;
|
|
15
|
+
readonly executable?: string;
|
|
16
|
+
readonly runtimeEntry?: string;
|
|
17
|
+
};
|
|
6
18
|
|
|
7
19
|
const runtimePath = (entry: string): string => {
|
|
8
20
|
try {
|
|
@@ -40,3 +52,22 @@ export const resolveCliEntry = (runtimeEntry: string | undefined = process.argv[
|
|
|
40
52
|
);
|
|
41
53
|
return workingDirectoryEntry;
|
|
42
54
|
};
|
|
55
|
+
|
|
56
|
+
export const cliRelaunchCommand = (
|
|
57
|
+
argv: readonly string[],
|
|
58
|
+
environment: NodeJS.ProcessEnv,
|
|
59
|
+
options: CliRelaunchOptions = {},
|
|
60
|
+
): CliRelaunchCommand => {
|
|
61
|
+
const executable = options.executable ?? process.execPath;
|
|
62
|
+
if (!(options.native ?? isNativeCli())) {
|
|
63
|
+
return {
|
|
64
|
+
executable:
|
|
65
|
+
options.executable ?? (Reflect.get(process.versions, "bun") ? process.execPath : "bun"),
|
|
66
|
+
args: [resolveCliEntry(options.runtimeEntry), ...argv],
|
|
67
|
+
environment,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
const reexecEnvironment = { ...environment };
|
|
71
|
+
delete reexecEnvironment.BUN_BE_BUN;
|
|
72
|
+
return { executable, args: [...argv], environment: reexecEnvironment };
|
|
73
|
+
};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Command } from "commander";
|
|
2
|
+
import { listSpecializations } from "../audience";
|
|
3
|
+
import { resolveContext, type CommandHost } from "../context";
|
|
4
|
+
|
|
5
|
+
export const registerAudienceCommands = (program: Command, host: CommandHost): void => {
|
|
6
|
+
const { context, globals } = host;
|
|
7
|
+
|
|
8
|
+
const audience = program
|
|
9
|
+
.command("audience")
|
|
10
|
+
.description("inspect the expert audience catalog used by node audience filters");
|
|
11
|
+
const specialization = audience
|
|
12
|
+
.command("specialization")
|
|
13
|
+
.description("expert specializations (platform qualifications)");
|
|
14
|
+
specialization
|
|
15
|
+
.command("list")
|
|
16
|
+
.option("--json", "print JSON", false)
|
|
17
|
+
.description("list specializations with their id, name, and expert count")
|
|
18
|
+
.action(async (options: { json: boolean }) => {
|
|
19
|
+
const { client } = await resolveContext(globals(), context);
|
|
20
|
+
await listSpecializations(client, { json: options.json }, context.io);
|
|
21
|
+
});
|
|
22
|
+
};
|
|
@@ -64,7 +64,11 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
64
64
|
.option("--train-filter <field=value>", "training Dataset field equality filter")
|
|
65
65
|
.option("--validation-filter <field=value>", "validation Dataset field equality filter")
|
|
66
66
|
.option("--lora-rank <rank>", "SFT LoRA adapter rank: 8, 16, or 32 (default 16)", trainingLoraRank)
|
|
67
|
-
.option(
|
|
67
|
+
.option(
|
|
68
|
+
"--gist-tokens <count>",
|
|
69
|
+
'gisting soft tokens: 8, 32, or 64, or a fraction of the prompt "1/4", "1/8", "1/16", "1/32" (default 32)',
|
|
70
|
+
trainingGistTokens,
|
|
71
|
+
)
|
|
68
72
|
.option("--epochs <count>", "training epochs, 1 through 5 (default 3)", trainingEpochs)
|
|
69
73
|
.option(
|
|
70
74
|
"--learning-rate <rate>",
|
|
@@ -111,7 +115,11 @@ export const registerTrainingCommands = (program: Command, host: CommandHost): v
|
|
|
111
115
|
"replacement validation Dataset field equality filter",
|
|
112
116
|
)
|
|
113
117
|
.option("--lora-rank <rank>", "SFT LoRA adapter rank: 8, 16, or 32 (default 16)", trainingLoraRank)
|
|
114
|
-
.option(
|
|
118
|
+
.option(
|
|
119
|
+
"--gist-tokens <count>",
|
|
120
|
+
'gisting soft tokens: 8, 32, or 64, or a fraction of the prompt "1/4", "1/8", "1/16", "1/32" (default 32)',
|
|
121
|
+
trainingGistTokens,
|
|
122
|
+
)
|
|
115
123
|
.option("--epochs <count>", "training epochs, 1 through 5 (default 3)", trainingEpochs)
|
|
116
124
|
.option(
|
|
117
125
|
"--learning-rate <rate>",
|
package/src/cli/context.ts
CHANGED
|
@@ -96,10 +96,15 @@ export const trainingLoraRank = (raw: string): 8 | 16 | 32 => {
|
|
|
96
96
|
return value;
|
|
97
97
|
};
|
|
98
98
|
|
|
99
|
-
export const trainingGistTokens = (raw: string): 8 | 32 | 64 => {
|
|
99
|
+
export const trainingGistTokens = (raw: string): 8 | 32 | 64 | "1/4" | "1/8" | "1/16" | "1/32" => {
|
|
100
|
+
if (raw === "1/4" || raw === "1/8" || raw === "1/16" || raw === "1/32") {
|
|
101
|
+
return raw;
|
|
102
|
+
}
|
|
100
103
|
const value = Number(raw);
|
|
101
104
|
if (value !== 8 && value !== 32 && value !== 64) {
|
|
102
|
-
throw new InvalidArgumentError(
|
|
105
|
+
throw new InvalidArgumentError(
|
|
106
|
+
'Expected 8, 32, or 64 gist tokens, or a fraction of the prompt: "1/4", "1/8", "1/16", or "1/32".',
|
|
107
|
+
);
|
|
103
108
|
}
|
|
104
109
|
return value;
|
|
105
110
|
};
|
package/src/cli/init.ts
CHANGED
|
@@ -223,7 +223,12 @@ const bindCheckout = async (
|
|
|
223
223
|
assertCli(
|
|
224
224
|
existing === undefined || existing.projectId === projectId,
|
|
225
225
|
`This directory is already bound to ${existing?.projectId}.`,
|
|
226
|
-
{
|
|
226
|
+
{
|
|
227
|
+
hint: {
|
|
228
|
+
description:
|
|
229
|
+
"Run toloka init without --project-id to keep the current binding, or remove toloka.json to bind another project.",
|
|
230
|
+
},
|
|
231
|
+
},
|
|
227
232
|
);
|
|
228
233
|
if (existing !== undefined) return undefined;
|
|
229
234
|
await writeTolokaState(targetRoot, { projectId, pipelines: {}, datasets: {} });
|
|
@@ -275,7 +280,7 @@ export async function initProject(
|
|
|
275
280
|
];
|
|
276
281
|
const warning = dependencyReady
|
|
277
282
|
? undefined
|
|
278
|
-
: `Warning: ${SDK_PACKAGE_NAME} is not listed in dependencies or devDependencies in ${join(targetRoot, "package.json")}.
|
|
283
|
+
: `Warning: ${SDK_PACKAGE_NAME} is not listed in dependencies or devDependencies in ${join(targetRoot, "package.json")}. Install it with npm or Bun before authoring pipelines, then re-run \`toloka init\`. Training commands do not require it.`;
|
|
279
284
|
return {
|
|
280
285
|
targetDirectory: targetRoot,
|
|
281
286
|
files,
|
package/src/cli/models.ts
CHANGED
|
@@ -8,10 +8,9 @@ import { pipeline } from "node:stream/promises";
|
|
|
8
8
|
|
|
9
9
|
import type { BaseModel, BaseModelId, Model, ModelVersion, PotatoClient } from "../client";
|
|
10
10
|
import { assertCli, REPORT_HINT, RETRY_OR_REPORT_HINT } from "./cli-error";
|
|
11
|
+
import { formatTable, type TableIo as ModelIo, writeOutput } from "./table";
|
|
11
12
|
|
|
12
13
|
type ModelApi = Pick<PotatoClient, "baseModels" | "models">;
|
|
13
|
-
type ModelIo = { readonly stdout: (text: string) => void };
|
|
14
|
-
type TableRow = readonly string[];
|
|
15
14
|
|
|
16
15
|
const BASE_MODEL_IDS = [
|
|
17
16
|
"qwen3-4b",
|
|
@@ -20,25 +19,6 @@ const BASE_MODEL_IDS = [
|
|
|
20
19
|
"qwen3-235b",
|
|
21
20
|
] satisfies readonly BaseModelId[];
|
|
22
21
|
|
|
23
|
-
const formatTable = (rows: readonly TableRow[]): string => {
|
|
24
|
-
const widths = rows.reduce<number[]>(
|
|
25
|
-
(current, row) => row.map((cell, index) => Math.max(current[index] ?? 0, cell.length)),
|
|
26
|
-
[],
|
|
27
|
-
);
|
|
28
|
-
return rows
|
|
29
|
-
.map((row) =>
|
|
30
|
-
row
|
|
31
|
-
.map((cell, index) => cell.padEnd(widths[index] ?? cell.length))
|
|
32
|
-
.join(" ")
|
|
33
|
-
.trimEnd(),
|
|
34
|
-
)
|
|
35
|
-
.join("\n");
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
const writeOutput = (io: ModelIo, json: boolean, value: unknown, formatted: string): void => {
|
|
39
|
-
io.stdout(json ? JSON.stringify(value, null, 2) : formatted);
|
|
40
|
-
};
|
|
41
|
-
|
|
42
22
|
const formatBaseModel = (model: BaseModel): string =>
|
|
43
23
|
formatTable([
|
|
44
24
|
["ID", model.id],
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export const isNativeCli = (): boolean =>
|
|
2
|
+
Reflect.get(globalThis, "Bun")?.isStandaloneExecutable === true;
|
|
3
|
+
|
|
4
|
+
export const bunSubprocessEnvironment = (
|
|
5
|
+
environment: NodeJS.ProcessEnv,
|
|
6
|
+
native = isNativeCli(),
|
|
7
|
+
): NodeJS.ProcessEnv => (native ? { ...environment, BUN_BE_BUN: "1" } : environment);
|
package/src/cli/program.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type { CliAnalytics, CliAnalyticsCommandOutcome } from "./analytics";
|
|
|
5
5
|
import type { AutoUpdateHooks } from "./auto-update";
|
|
6
6
|
import { maybeAutoUpdate } from "./auto-update";
|
|
7
7
|
import { CHECK_RESULT_FILE_ENV, type checkProject } from "./check";
|
|
8
|
+
import { registerAudienceCommands } from "./commands/audience";
|
|
8
9
|
import { registerAuthCommands } from "./commands/auth";
|
|
9
10
|
import { registerConfigCommands } from "./commands/config";
|
|
10
11
|
import { registerDatasetCommands } from "./commands/dataset";
|
|
@@ -91,6 +92,7 @@ export function createProgram(context: ProgramContext): CreatedProgram {
|
|
|
91
92
|
registerAuthCommands(program, host);
|
|
92
93
|
registerProjectCommands(program, host);
|
|
93
94
|
registerDeliveryOrderCommands(program, host);
|
|
95
|
+
registerAudienceCommands(program, host);
|
|
94
96
|
registerSourceCommands(program, host);
|
|
95
97
|
registerSecretCommands(program, host);
|
|
96
98
|
registerModelCommands(program, host);
|
package/src/cli/skill.ts
CHANGED
|
@@ -1,9 +1,19 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
copyFile,
|
|
4
|
+
mkdir,
|
|
5
|
+
mkdtemp,
|
|
6
|
+
readFile,
|
|
7
|
+
realpath,
|
|
8
|
+
rename,
|
|
9
|
+
rm,
|
|
10
|
+
stat,
|
|
11
|
+
writeFile,
|
|
12
|
+
} from "node:fs/promises";
|
|
3
13
|
import { dirname, join } from "node:path";
|
|
4
14
|
import { assertSafeDestination, entryStats, hasErrorCode } from "./file-system";
|
|
5
15
|
import { formatCliError } from "./format-error";
|
|
6
|
-
import {
|
|
16
|
+
import { SDK_PACKAGE_NAME, SDK_VERSION } from "./sdk-version";
|
|
7
17
|
import { assertCli, REINSTALL_HINT, TolokaCliError } from "./cli-error";
|
|
8
18
|
|
|
9
19
|
export const SKILL_CATALOG_MANIFEST = "catalog.json";
|
|
@@ -16,9 +26,7 @@ const MAX_MANIFEST_BYTES = 1024 * 1024;
|
|
|
16
26
|
const MAX_SKILL_ID_LENGTH = 64;
|
|
17
27
|
const SKILL_ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u;
|
|
18
28
|
const DIGEST_PATTERN = /^[a-f0-9]{64}$/u;
|
|
19
|
-
const UPGRADE_HINT =
|
|
20
|
-
releaseChannelFor(SDK_VERSION) ?? "latest"
|
|
21
|
-
}`;
|
|
29
|
+
const UPGRADE_HINT = "rerun the CLI installer to update";
|
|
22
30
|
|
|
23
31
|
export type SkillDescriptor = {
|
|
24
32
|
readonly id: string;
|
|
@@ -86,21 +94,15 @@ const parseDescriptor = (value: unknown, invalid: (detail: string) => string): S
|
|
|
86
94
|
{ hint: REINSTALL_HINT },
|
|
87
95
|
);
|
|
88
96
|
assertCli(isNonEmptyString(name), invalid(`skill ${id} needs a name.`), { hint: REINSTALL_HINT });
|
|
89
|
-
assertCli(
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
);
|
|
99
|
-
assertCli(
|
|
100
|
-
isNonEmptyString(version),
|
|
101
|
-
invalid(`skill ${id} needs a version.`),
|
|
102
|
-
{ hint: REINSTALL_HINT },
|
|
103
|
-
);
|
|
97
|
+
assertCli(isNonEmptyString(description), invalid(`skill ${id} needs a description.`), {
|
|
98
|
+
hint: REINSTALL_HINT,
|
|
99
|
+
});
|
|
100
|
+
assertCli(isStringArray(tags), invalid(`skill ${id} tags must be strings.`), {
|
|
101
|
+
hint: REINSTALL_HINT,
|
|
102
|
+
});
|
|
103
|
+
assertCli(isNonEmptyString(version), invalid(`skill ${id} needs a version.`), {
|
|
104
|
+
hint: REINSTALL_HINT,
|
|
105
|
+
});
|
|
104
106
|
assertCli(
|
|
105
107
|
typeof digest === "string" && DIGEST_PATTERN.test(digest),
|
|
106
108
|
invalid(`skill ${id} digest must be the sha256 hex digest of its files.`),
|
|
@@ -154,6 +156,9 @@ const findSkillCatalogRootAbove = async (directory: string): Promise<string | un
|
|
|
154
156
|
export async function resolveBundledSkillCatalogRoot(
|
|
155
157
|
runtimeEntry: string | undefined = process.argv[1],
|
|
156
158
|
): Promise<string> {
|
|
159
|
+
if (Reflect.get(globalThis, "Bun")?.isStandaloneExecutable === true) {
|
|
160
|
+
return join(dirname(process.argv[1]!), "skills");
|
|
161
|
+
}
|
|
157
162
|
const entry = runtimeEntry === undefined ? undefined : await realpath(runtimeEntry);
|
|
158
163
|
const runtimeRoot =
|
|
159
164
|
entry === undefined ? undefined : await findSkillCatalogRootAbove(dirname(entry));
|
|
@@ -197,11 +202,9 @@ export async function loadSkillCatalog(catalogRoot: string): Promise<SkillCatalo
|
|
|
197
202
|
const skills = manifest.skills.map((skill) => parseDescriptor(skill, invalid));
|
|
198
203
|
const ids = new Set<string>();
|
|
199
204
|
for (const skill of skills) {
|
|
200
|
-
assertCli(
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
{ hint: REINSTALL_HINT },
|
|
204
|
-
);
|
|
205
|
+
assertCli(!ids.has(skill.id), invalid(`skill id ${skill.id} appears more than once.`), {
|
|
206
|
+
hint: REINSTALL_HINT,
|
|
207
|
+
});
|
|
205
208
|
ids.add(skill.id);
|
|
206
209
|
}
|
|
207
210
|
return { root: catalogRoot, manifestPath, skills };
|
|
@@ -329,7 +332,12 @@ export async function installSkill({
|
|
|
329
332
|
for (const path of skill.files) {
|
|
330
333
|
const target = join(temporary, path);
|
|
331
334
|
await mkdir(dirname(target), { recursive: true });
|
|
332
|
-
|
|
335
|
+
const source = join(catalog.root, skill.id, path);
|
|
336
|
+
if (Reflect.get(globalThis, "Bun")?.isStandaloneExecutable === true) {
|
|
337
|
+
await writeFile(target, await readFile(source));
|
|
338
|
+
} else {
|
|
339
|
+
await copyFile(source, target);
|
|
340
|
+
}
|
|
333
341
|
}
|
|
334
342
|
assertCli(
|
|
335
343
|
(await computeSkillDigest(temporary, skill.files)) === skill.digest,
|
package/src/cli/table.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export type TableRow = readonly string[];
|
|
2
|
+
export type TableIo = { readonly stdout: (text: string) => void };
|
|
3
|
+
|
|
4
|
+
export const formatTable = (rows: readonly TableRow[]): string => {
|
|
5
|
+
const widths = rows.reduce<number[]>(
|
|
6
|
+
(current, row) => row.map((cell, index) => Math.max(current[index] ?? 0, cell.length)),
|
|
7
|
+
[],
|
|
8
|
+
);
|
|
9
|
+
return rows
|
|
10
|
+
.map((row) =>
|
|
11
|
+
row
|
|
12
|
+
.map((cell, index) => cell.padEnd(widths[index] ?? cell.length))
|
|
13
|
+
.join(" ")
|
|
14
|
+
.trimEnd(),
|
|
15
|
+
)
|
|
16
|
+
.join("\n");
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export const writeOutput = (io: TableIo, json: boolean, value: unknown, formatted: string): void => {
|
|
20
|
+
io.stdout(json ? JSON.stringify(value, null, 2) : formatted);
|
|
21
|
+
};
|
package/src/cli/trainings.ts
CHANGED
|
@@ -14,6 +14,7 @@ import type {
|
|
|
14
14
|
import type { BaseModelId, TrainingConfigurationInput } from "../../openapi/gen/models";
|
|
15
15
|
import { assertCli, RETRY_OR_REPORT_HINT, TolokaCliError } from "./cli-error";
|
|
16
16
|
import type { CliErrorHint } from "./cli-error";
|
|
17
|
+
import { formatTable, type TableIo as TrainingIo, writeOutput } from "./table";
|
|
17
18
|
|
|
18
19
|
const ESTIMATE_FILE_HINT: CliErrorHint = {
|
|
19
20
|
description: "Regenerate it with toloka training cost <training> --json > <file>.",
|
|
@@ -49,28 +50,6 @@ type TrainingDraftApi = Pick<PotatoClient, "trainings" | "files" | "datasets">;
|
|
|
49
50
|
type TrainingCostApi = Pick<PotatoClient, "trainings" | "waitFor">;
|
|
50
51
|
type TrainingRunApi = Pick<PotatoClient, "trainings" | "waitFor">;
|
|
51
52
|
const GISTING_PROMPT_MAX_BYTES = 256 * 1024;
|
|
52
|
-
type TrainingIo = { readonly stdout: (text: string) => void };
|
|
53
|
-
type TableRow = readonly string[];
|
|
54
|
-
|
|
55
|
-
const formatTable = (rows: readonly TableRow[]): string => {
|
|
56
|
-
const widths = rows.reduce<number[]>(
|
|
57
|
-
(current, row) => row.map((cell, index) => Math.max(current[index] ?? 0, cell.length)),
|
|
58
|
-
[],
|
|
59
|
-
);
|
|
60
|
-
return rows
|
|
61
|
-
.map((row) =>
|
|
62
|
-
row
|
|
63
|
-
.map((cell, index) => cell.padEnd(widths[index] ?? cell.length))
|
|
64
|
-
.join(" ")
|
|
65
|
-
.trimEnd(),
|
|
66
|
-
)
|
|
67
|
-
.join("\n");
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
const writeOutput = (io: TrainingIo, json: boolean, value: unknown, formatted: string): void => {
|
|
71
|
-
io.stdout(json ? JSON.stringify(value, null, 2) : formatted);
|
|
72
|
-
};
|
|
73
|
-
|
|
74
53
|
const formatTraining = (training: Training): string =>
|
|
75
54
|
formatTable([
|
|
76
55
|
["ID", training.id],
|
package/src/cli/version.ts
CHANGED
|
@@ -4,10 +4,13 @@ import { dirname, join, resolve, sep } from "node:path";
|
|
|
4
4
|
|
|
5
5
|
import { SDK_VERSION } from "./sdk-version";
|
|
6
6
|
import { assertCli, REPORT_HINT } from "./cli-error";
|
|
7
|
+
import { isNativeCli } from "./native-runtime";
|
|
7
8
|
|
|
8
|
-
export type CliInstallChannel = "global-registry" | "linked" | "project-local";
|
|
9
|
+
export type CliInstallChannel = "native" | "global-registry" | "linked" | "project-local";
|
|
9
10
|
|
|
10
11
|
type VersionInfoOptions = {
|
|
12
|
+
readonly native?: boolean;
|
|
13
|
+
readonly executablePath?: string;
|
|
11
14
|
readonly globalBinDirectory?: string;
|
|
12
15
|
readonly globalPackageDirectory?: string;
|
|
13
16
|
};
|
|
@@ -36,6 +39,14 @@ const classifyInstallChannel = (
|
|
|
36
39
|
};
|
|
37
40
|
|
|
38
41
|
export const cliVersionInfo = (entryPath = process.argv[1], options: VersionInfoOptions = {}) => {
|
|
42
|
+
if (options.native ?? isNativeCli()) {
|
|
43
|
+
return {
|
|
44
|
+
version: SDK_VERSION,
|
|
45
|
+
commit: process.env.TOLOKA_BUILD_COMMIT ?? null,
|
|
46
|
+
channel: "native",
|
|
47
|
+
path: realpathOrSelf(options.executablePath ?? process.execPath),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
39
50
|
assertCli(entryPath, "CLI entry path is unavailable", { hint: REPORT_HINT });
|
|
40
51
|
const globalBinDirectory =
|
|
41
52
|
options.globalBinDirectory ??
|
package/src/client.ts
CHANGED
|
@@ -33,6 +33,7 @@ import type {
|
|
|
33
33
|
ListPipelineRunsParams,
|
|
34
34
|
ListProjectDatasetsParams,
|
|
35
35
|
ListProjectPipelinesParams,
|
|
36
|
+
ListAudienceSpecializationsParams,
|
|
36
37
|
ListProjectModelsParams,
|
|
37
38
|
ListProjectsParams,
|
|
38
39
|
ListTrainingRunsParams,
|
|
@@ -635,8 +636,24 @@ const createClientImplementation = (config: ClientConfig) => {
|
|
|
635
636
|
list: async () => (await wire.listDeliveryOrders(requestOptions)).deliveryOrders,
|
|
636
637
|
};
|
|
637
638
|
|
|
639
|
+
const audience = {
|
|
640
|
+
specializations: {
|
|
641
|
+
listPage: (params: ListAudienceSpecializationsParams = {}) =>
|
|
642
|
+
wire.listAudienceSpecializations(params, requestOptions),
|
|
643
|
+
listAll: (params: ListAudienceSpecializationsParams = {}) =>
|
|
644
|
+
listAllPages(async (pageOptions) => {
|
|
645
|
+
const page = await wire.listAudienceSpecializations(
|
|
646
|
+
{ ...params, ...pageOptions },
|
|
647
|
+
requestOptions,
|
|
648
|
+
);
|
|
649
|
+
return { items: page.specializations, pagination: page.pagination };
|
|
650
|
+
}, params),
|
|
651
|
+
},
|
|
652
|
+
};
|
|
653
|
+
|
|
638
654
|
return {
|
|
639
655
|
me: () => wire.getCurrentUser(requestOptions),
|
|
656
|
+
audience,
|
|
640
657
|
projects,
|
|
641
658
|
pipelines,
|
|
642
659
|
datasets,
|
package/src/index.ts
CHANGED