@toloka-platform/sdk 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/toloka.js +156 -137
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +2 -0
- package/docs/cli.md +13 -3
- package/openapi/gen/api.ts +30 -0
- package/openapi/gen/models/gistingTrainingConfigurationGistTokens.ts +1 -1
- package/openapi/gen/models/gistingTrainingConfigurationInputGistTokens.ts +1 -1
- package/openapi/gen/models/index.ts +3 -0
- package/openapi/gen/models/listAudienceSpecializations200.ts +7 -0
- package/openapi/gen/models/listAudienceSpecializationsParams.ts +5 -0
- package/openapi/gen/models/specialization.ts +6 -0
- package/openapi/gen/models/trainingCostEstimateOneOfTwoseven.ts +0 -2
- package/openapi/gen/models/trainingRunQuote.ts +0 -2
- package/package.json +1 -1
- package/skills/catalog.json +8 -8
- package/skills/toloka-build/references/node-semantics.md +2 -2
- package/skills/toloka-build/references/workflow-and-cli.md +6 -3
- package/skills/toloka-fine-tuning/SKILL.md +2 -0
- package/skills/toloka-gisting/SKILL.md +3 -1
- package/src/checks/task-ui-typecheck.ts +4 -1
- package/src/cli/analytics.ts +1 -0
- package/src/cli/audience.ts +31 -0
- package/src/cli/auto-update.ts +363 -24
- package/src/cli/check.ts +51 -52
- package/src/cli/cli-entry.ts +31 -0
- package/src/cli/commands/audience.ts +22 -0
- package/src/cli/commands/training.ts +10 -2
- package/src/cli/context.ts +7 -2
- package/src/cli/init.ts +7 -2
- package/src/cli/models.ts +1 -21
- package/src/cli/native-runtime.ts +7 -0
- package/src/cli/program.ts +2 -0
- package/src/cli/skill.ts +34 -26
- package/src/cli/table.ts +21 -0
- package/src/cli/trainings.ts +1 -22
- package/src/cli/version.ts +12 -1
- package/src/client.ts +17 -0
- package/src/index.ts +1 -0
package/dist/cli/toloka.js.jsc
CHANGED
|
Binary file
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
5c6e6ba38b31803da60d3f8371a159ce99d5e156595e002cd7a4f6e6c9432cf8 toloka.js.jsc
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
88ba4c829ee5f68917b216cfdaefcbf44528cb059013afd242f499d15caf506d toloka.js
|
package/docs/authoring.md
CHANGED
|
@@ -152,6 +152,8 @@ export const annotateFaces = generation("annotate_faces", {
|
|
|
152
152
|
});
|
|
153
153
|
```
|
|
154
154
|
|
|
155
|
+
To require a specialization, add `{ value, qualificationId }` entries to `audience.specializations.values`. Both come from the platform catalog — `toloka audience specialization list` — where `NAME` is the `value`, `ID` is the `qualificationId`, and `EXPERTS` tells you how large the pool is. Specializations require `verifiedExperts: true`.
|
|
156
|
+
|
|
155
157
|
Test the interface locally before applying it. The SDK's Vitest fixture renders this exact `ui` block in Chromium, interacts with Fields by `path`, runs postprocess, validates output entities, and saves screenshots. See [Test pipelines locally](./testing.md).
|
|
156
158
|
|
|
157
159
|
### quorum
|
package/docs/cli.md
CHANGED
|
@@ -108,6 +108,16 @@ toloka delivery-order list
|
|
|
108
108
|
|
|
109
109
|
Prints `ID`, `NAME`, `STATUS`, and `CLIENT` columns. Empty output is `No delivery orders visible.`
|
|
110
110
|
|
|
111
|
+
## audience
|
|
112
|
+
|
|
113
|
+
Inspect the global, read-only expert audience catalog: the values a node's `audienceAndPricing.audience` filter can use.
|
|
114
|
+
|
|
115
|
+
```sh
|
|
116
|
+
toloka audience specialization list [--json]
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Lists every expert specialization (platform qualification) with `ID`, `NAME`, and `EXPERTS` columns, sorted by name. Copy `ID` into `qualificationId` and `NAME` into `value` of `audience.specializations.values[]`; never invent a `qualificationId`. `EXPERTS` is the number of ID-verified experts currently eligible for tasks who hold the specialization; it can be `0` while the id is still valid. Pass `--json` for the full array. Empty output is `No specializations available.`
|
|
120
|
+
|
|
111
121
|
## login
|
|
112
122
|
|
|
113
123
|
Stores an API key for this machine after verifying it.
|
|
@@ -294,8 +304,8 @@ Create a reusable Training draft, price it, and run it.
|
|
|
294
304
|
```sh
|
|
295
305
|
toloka training list [--json]
|
|
296
306
|
toloka training get <name-or-id> [--json]
|
|
297
|
-
toloka training create <name> [--method sft|gisting] (--base-model <id> | --model-version <id>) --dataset <id> [--train-filter field=value] [--validation-dataset <id> [--validation-filter field=value]] [--prompt-file <path>] [--lora-rank 8|16|32 | --gist-tokens 8|32|64] [--epochs 1..5] [--learning-rate <rate>] [--batch-size 8|16|32|64] [--max-sequence-length <tokens>] [--json]
|
|
298
|
-
toloka training update <name-or-id> [--base-model <id> | --model-version <id>] [--dataset <id>] [--train-filter field=value] [--validation-dataset <id> [--validation-filter field=value]] [--prompt-file <path>] [--lora-rank 8|16|32 | --gist-tokens 8|32|64] [--epochs 1..5] [--learning-rate <rate>] [--batch-size 8|16|32|64] [--max-sequence-length <tokens>] [--json]
|
|
307
|
+
toloka training create <name> [--method sft|gisting] (--base-model <id> | --model-version <id>) --dataset <id> [--train-filter field=value] [--validation-dataset <id> [--validation-filter field=value]] [--prompt-file <path>] [--lora-rank 8|16|32 | --gist-tokens 8|32|64|1/4|1/8|1/16|1/32] [--epochs 1..5] [--learning-rate <rate>] [--batch-size 8|16|32|64] [--max-sequence-length <tokens>] [--json]
|
|
308
|
+
toloka training update <name-or-id> [--base-model <id> | --model-version <id>] [--dataset <id>] [--train-filter field=value] [--validation-dataset <id> [--validation-filter field=value]] [--prompt-file <path>] [--lora-rank 8|16|32 | --gist-tokens 8|32|64|1/4|1/8|1/16|1/32] [--epochs 1..5] [--learning-rate <rate>] [--batch-size 8|16|32|64] [--max-sequence-length <tokens>] [--json]
|
|
299
309
|
toloka training cost <name-or-id> [--idempotency-key <key>] [--json]
|
|
300
310
|
toloka training run <name-or-id> [--estimate-file <path>] [--version-name <name>] [--idempotency-key <key>] [--json]
|
|
301
311
|
toloka training runs <name-or-id> [--json]
|
|
@@ -306,7 +316,7 @@ toloka training logs <training-run-id>
|
|
|
306
316
|
toloka training metrics <training-run-id> [--key <metric>] [--resolution <n>] [--json]
|
|
307
317
|
```
|
|
308
318
|
|
|
309
|
-
Hyperparameters are optional flags on `training create` and `training update`: `--epochs 1..5` (default 3), `--learning-rate` (SFT 0.00001–0.001, default 0.00015; gisting 0.001–0.3, default 0.03), `--batch-size 8|16|32|64` (default 8; sequences per optimizer step across all GPUs) and `--max-sequence-length 64..16384` (default 12288 for SFT, 3072 for gisting) apply to both methods; `--lora-rank 8|16|32` (default 16) is SFT-only and `--gist-tokens
|
|
319
|
+
Hyperparameters are optional flags on `training create` and `training update`: `--epochs 1..5` (default 3), `--learning-rate` (SFT 0.00001–0.001, default 0.00015; gisting 0.001–0.3, default 0.03), `--batch-size 8|16|32|64` (default 8; sequences per optimizer step across all GPUs) and `--max-sequence-length 64..16384` (default 12288 for SFT, 3072 for gisting) apply to both methods; `--lora-rank 8|16|32` (default 16) is SFT-only and `--gist-tokens` (default 32) is gisting-only — passing one to the other method fails before any request. `--gist-tokens` takes an absolute count `8|32|64` or a fraction of the prompt being compressed (`1/4`, `1/8`, `1/16`, `1/32`), which the run resolves to `ceil(prompt tokens / N)` clamped to 8..256. `--regime` and `--eval-steps` no longer exist. `training update` has no `--method` flag: the method stays as created, and every flag you omit keeps its current value. `training cost` is blocked when the longest example exceeds `--max-sequence-length` or when the training selection leaves fewer rows than `--batch-size` after the 10% holdout. `training get` prints the resolved recipe, so omitted flags are visible as their defaults.
|
|
310
320
|
|
|
311
321
|
## dataset
|
|
312
322
|
|
package/openapi/gen/api.ts
CHANGED
|
@@ -32,6 +32,8 @@ import type {
|
|
|
32
32
|
GetTrainingRunLogsParams,
|
|
33
33
|
GetTrainingRunMetricHistoryParams,
|
|
34
34
|
ImportPipelineBody,
|
|
35
|
+
ListAudienceSpecializations200,
|
|
36
|
+
ListAudienceSpecializationsParams,
|
|
35
37
|
ListBaseModels200,
|
|
36
38
|
ListDatasetFields200,
|
|
37
39
|
ListDatasetFieldsParams,
|
|
@@ -686,6 +688,34 @@ export const cancelTrainingRun = async (trainingRunId: TrainingRunId, options?:
|
|
|
686
688
|
|
|
687
689
|
|
|
688
690
|
|
|
691
|
+
export const getListAudienceSpecializationsUrl = (params?: ListAudienceSpecializationsParams,) => {
|
|
692
|
+
const normalizedParams = new URLSearchParams();
|
|
693
|
+
|
|
694
|
+
Object.entries(params || {}).forEach(([key, value]) => {
|
|
695
|
+
|
|
696
|
+
if (value !== undefined) {
|
|
697
|
+
normalizedParams.append(key, value === null ? 'null' : value.toString())
|
|
698
|
+
}
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
const stringifiedParams = normalizedParams.toString();
|
|
702
|
+
|
|
703
|
+
return stringifiedParams.length > 0 ? `/api/v2-beta/audience/specializations?${stringifiedParams}` : `/api/v2-beta/audience/specializations`
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
export const listAudienceSpecializations = async (params?: ListAudienceSpecializationsParams, options?: RequestInit): Promise<ListAudienceSpecializations200> => {
|
|
707
|
+
|
|
708
|
+
return potatoFetch<ListAudienceSpecializations200>(getListAudienceSpecializationsUrl(params),
|
|
709
|
+
{
|
|
710
|
+
...options,
|
|
711
|
+
method: 'GET'
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
}
|
|
715
|
+
);}
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
|
|
689
719
|
export const getGetCurrentUserUrl = () => {
|
|
690
720
|
|
|
691
721
|
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
|
|
2
|
-
export type GistingTrainingConfigurationGistTokens = 8 | 32 | 64;
|
|
2
|
+
export type GistingTrainingConfigurationGistTokens = 8 | 32 | 64 | '1/4' | '1/8' | '1/16' | '1/32';
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
|
|
2
|
-
export type GistingTrainingConfigurationInputGistTokens = 8 | 32 | 64;
|
|
2
|
+
export type GistingTrainingConfigurationInputGistTokens = 8 | 32 | 64 | '1/4' | '1/8' | '1/16' | '1/32';
|
|
@@ -280,6 +280,8 @@ export * from './jsonSchema';
|
|
|
280
280
|
export * from './jsonSchemaAdditionalProperties';
|
|
281
281
|
export * from './jsonSchemaType';
|
|
282
282
|
export * from './jsonSchemaTypeAnyOfItem';
|
|
283
|
+
export * from './listAudienceSpecializations200';
|
|
284
|
+
export * from './listAudienceSpecializationsParams';
|
|
283
285
|
export * from './listBaseModels200';
|
|
284
286
|
export * from './listDatasetFields200';
|
|
285
287
|
export * from './listDatasetFieldsParams';
|
|
@@ -829,6 +831,7 @@ export * from './sftTrainingConfigurationInput';
|
|
|
829
831
|
export * from './sftTrainingConfigurationInputBatchSize';
|
|
830
832
|
export * from './sftTrainingConfigurationInputLoraRank';
|
|
831
833
|
export * from './sftTrainingConfigurationLoraRank';
|
|
834
|
+
export * from './specialization';
|
|
832
835
|
export * from './training';
|
|
833
836
|
export * from './trainingArchivedAt';
|
|
834
837
|
export * from './trainingConfiguration';
|
|
@@ -15,8 +15,6 @@ export type TrainingCostEstimateOneOfTwoseven = {
|
|
|
15
15
|
status: 'ready';
|
|
16
16
|
issues: TrainingCostEstimateOneOfTwosevenIssuesItem[];
|
|
17
17
|
budget: TrainingCostEstimateOneOfTwosevenBudget;
|
|
18
|
-
basePricePerGpuHour: string;
|
|
19
|
-
marginMultiplier: string;
|
|
20
18
|
maximumCharge: string;
|
|
21
19
|
currency: 'USD';
|
|
22
20
|
fingerprint: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toloka-platform/sdk",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Build, inspect, and operate Toloka pipeline projects as TypeScript source. The public API may change between minor versions before 1.0.",
|
|
6
6
|
"license": "Apache-2.0",
|
package/skills/catalog.json
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
"testing",
|
|
12
12
|
"checks"
|
|
13
13
|
],
|
|
14
|
-
"version": "0.2.
|
|
15
|
-
"digest": "
|
|
14
|
+
"version": "0.2.1",
|
|
15
|
+
"digest": "b7ffd9ccebdbb7cae9cf672cd3e43afa2c353398b8ea89a2558e8b7277b83fb7",
|
|
16
16
|
"files": [
|
|
17
17
|
"SKILL.md",
|
|
18
18
|
"references/node-semantics.md",
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"bug",
|
|
30
30
|
"report"
|
|
31
31
|
],
|
|
32
|
-
"version": "0.2.
|
|
32
|
+
"version": "0.2.1",
|
|
33
33
|
"digest": "56a0c283649052c85ba882f05d4a996bd98bf1bc011bc7a70af234b3664057d6",
|
|
34
34
|
"files": [
|
|
35
35
|
"SKILL.md",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"getting-started",
|
|
47
47
|
"agent"
|
|
48
48
|
],
|
|
49
|
-
"version": "0.2.
|
|
49
|
+
"version": "0.2.1",
|
|
50
50
|
"digest": "61a3cf60f0b7d00e48c9663ed6e0e3d77fd3923e528cee52e16ef6e2ccebae69",
|
|
51
51
|
"files": [
|
|
52
52
|
"SKILL.md"
|
|
@@ -63,8 +63,8 @@
|
|
|
63
63
|
"sft",
|
|
64
64
|
"models"
|
|
65
65
|
],
|
|
66
|
-
"version": "0.2.
|
|
67
|
-
"digest": "
|
|
66
|
+
"version": "0.2.1",
|
|
67
|
+
"digest": "75b7909bfaf8fdc13a91515efdcdbdcad31fd3355fa3401f22ee66e24258098c",
|
|
68
68
|
"files": [
|
|
69
69
|
"SKILL.md"
|
|
70
70
|
]
|
|
@@ -80,8 +80,8 @@
|
|
|
80
80
|
"prompt",
|
|
81
81
|
"models"
|
|
82
82
|
],
|
|
83
|
-
"version": "0.2.
|
|
84
|
-
"digest": "
|
|
83
|
+
"version": "0.2.1",
|
|
84
|
+
"digest": "8bd6fee5baa84e6f94871eebf34e916bf3f5deb202c60fb9e063ebbc68ddb5b3",
|
|
85
85
|
"files": [
|
|
86
86
|
"SKILL.md"
|
|
87
87
|
]
|
|
@@ -164,7 +164,7 @@ Keep `instruction` actionable and in English. Include the workflow, concepts, qu
|
|
|
164
164
|
|
|
165
165
|
#### Audience and pricing
|
|
166
166
|
|
|
167
|
-
For humans, set `audienceType: 'human'`, `verifiedExperts`, all three audience filters, `pricePerTask`, and `timeForTask`. The public shapes differ by filter: languages use `{ type: 'and' | 'or', values: [{ value: 'English' }] }`; specializations use `{ type: 'and' | 'or', values: [{ value, qualificationId }] }`; countries use ISO 3166-1 alpha-2 codes, for example `{ type: 'or', values: ['US'] }`. Verify every country value matches `/^[A-Z]{2}$/`; apply currently accepts arbitrary strings that can match no experts. Derive languages and expertise from the task and its inputs, leaving specialization and country filters unrestricted unless needed. English is a reasonable starting language for an English-language task, not a universal audience default. Ask when an unresolved audience requirement changes who can do the work. Do not put specializations on an unverified audience. A specialization's `qualificationId` is platform identity: preserve an imported value, and
|
|
167
|
+
For humans, set `audienceType: 'human'`, `verifiedExperts`, all three audience filters, `pricePerTask`, and `timeForTask`. The public shapes differ by filter: languages use `{ type: 'and' | 'or', values: [{ value: 'English' }] }`; specializations use `{ type: 'and' | 'or', values: [{ value, qualificationId }] }`; countries use ISO 3166-1 alpha-2 codes, for example `{ type: 'or', values: ['US'] }`. Verify every country value matches `/^[A-Z]{2}$/`; apply currently accepts arbitrary strings that can match no experts. Derive languages and expertise from the task and its inputs, leaving specialization and country filters unrestricted unless needed. English is a reasonable starting language for an English-language task, not a universal audience default. Ask when an unresolved audience requirement changes who can do the work. Do not put specializations on an unverified audience. A specialization's `qualificationId` is platform identity: preserve an imported value, and discover new ones with `toloka audience specialization list`, copying `ID` into `qualificationId` and `NAME` into `value`; never invent one.
|
|
168
168
|
|
|
169
169
|
`$0.20` and 120 seconds are a starting point for a short task, not a universal rate. Estimate effort from the work and required expertise, respect explicit rates, and explain meaningful adjustments in the cost forecast. The per-expert cap is normally off. For collection tasks, choose a cap based on contributor diversity and repetition requirements; 1 fits one-response-per-expert collection. Avoid copying an arbitrary cap into a task that does not need one.
|
|
170
170
|
|
|
@@ -281,7 +281,7 @@ Ordering is semantic: `conditions[0]` routes to `matched(0)`, the first true con
|
|
|
281
281
|
|
|
282
282
|
When the requirement depends on one of these capabilities, state the gap instead of inventing source syntax:
|
|
283
283
|
|
|
284
|
-
- Audience discovery
|
|
284
|
+
- Audience discovery beyond specializations has no SDK or CLI path: valid-language/country suggestions, audience-size statistics for a composed filter, and recommended-rate calculation. Specializations and their expert counts come from `toloka audience specialization list`.
|
|
285
285
|
- Audience-reuse candidate discovery has no SDK or CLI path. Source can store a known cross-pipeline pointer, but cannot search candidates.
|
|
286
286
|
- Smart Search rendering, source-fingerprint computation, and stale-profile diagnosis have no SDK or CLI path. Source can carry the public profile; it cannot generate the server-owned fingerprint or preview rendered sourcing documents.
|
|
287
287
|
- The `@toloka-platform/sdk/testing` entry point provides typed `test.code`, `test.ui`, and `test.pipeline` wrappers. `toloka check` runs their evidence with project TypeScript and automatic Task UI typechecking. Missing Remote Node Checks remain separate from this local gate.
|
|
@@ -105,6 +105,7 @@ toloka project remove <alias>
|
|
|
105
105
|
toloka project create [alias] [--name <name>] [--team-id <id>] [--delivery-order-id <id>]
|
|
106
106
|
toloka project untrack <pipeline|dataset> <name> [--confirm]
|
|
107
107
|
toloka delivery-order list
|
|
108
|
+
toloka audience specialization list [--json]
|
|
108
109
|
toloka init
|
|
109
110
|
toloka import [projectId]
|
|
110
111
|
toloka check [pipeline] [--node <name-or-id>] [--json] [--watch]
|
|
@@ -134,8 +135,8 @@ toloka model create <name> --base <base-model-id> [--description <text>] [--json
|
|
|
134
135
|
toloka model archive <name-or-id> [--json]
|
|
135
136
|
toloka training list [--json]
|
|
136
137
|
toloka training get <name-or-id> [--json]
|
|
137
|
-
toloka training create <name> [--method sft|gisting] (--base-model <id> | --model-version <id>) --dataset <id> [--train-filter field=value] [--validation-dataset <id> [--validation-filter field=value]] [--prompt-file <path>] [--lora-rank 8|16|32 | --gist-tokens 8|32|64] [--epochs 1..5] [--learning-rate <rate>] [--batch-size 8|16|32|64] [--max-sequence-length <tokens>] [--json]
|
|
138
|
-
toloka training update <name-or-id> [--base-model <id> | --model-version <id>] [--dataset <id>] [--train-filter field=value] [--validation-dataset <id> [--validation-filter field=value]] [--prompt-file <path>] [--lora-rank 8|16|32 | --gist-tokens 8|32|64] [--epochs 1..5] [--learning-rate <rate>] [--batch-size 8|16|32|64] [--max-sequence-length <tokens>] [--json]
|
|
138
|
+
toloka training create <name> [--method sft|gisting] (--base-model <id> | --model-version <id>) --dataset <id> [--train-filter field=value] [--validation-dataset <id> [--validation-filter field=value]] [--prompt-file <path>] [--lora-rank 8|16|32 | --gist-tokens 8|32|64|1/4|1/8|1/16|1/32] [--epochs 1..5] [--learning-rate <rate>] [--batch-size 8|16|32|64] [--max-sequence-length <tokens>] [--json]
|
|
139
|
+
toloka training update <name-or-id> [--base-model <id> | --model-version <id>] [--dataset <id>] [--train-filter field=value] [--validation-dataset <id> [--validation-filter field=value]] [--prompt-file <path>] [--lora-rank 8|16|32 | --gist-tokens 8|32|64|1/4|1/8|1/16|1/32] [--epochs 1..5] [--learning-rate <rate>] [--batch-size 8|16|32|64] [--max-sequence-length <tokens>] [--json]
|
|
139
140
|
toloka training cost <name-or-id> [--idempotency-key <key>] [--json]
|
|
140
141
|
toloka training run <name-or-id> [--estimate-file <path>] [--version-name <name>] [--idempotency-key <key>] [--json]
|
|
141
142
|
toloka training runs <name-or-id> [--json]
|
|
@@ -156,6 +157,8 @@ Resolve the intended platform project before `import`, `plan`, or `apply`; use a
|
|
|
156
157
|
|
|
157
158
|
`toloka project url` prints only the selected project's platform UI URL. Use it when the user needs a working link to inspect the live project.
|
|
158
159
|
|
|
160
|
+
`toloka audience specialization list` prints the global specialization catalog: `ID`, `NAME`, and `EXPERTS` columns sorted by name (`--json` for the array). Run it before setting `audience.specializations` on a node, pick entries by name, and copy `ID` into `qualificationId` and `NAME` into `value`; never invent a `qualificationId`. `EXPERTS` counts ID-verified experts currently eligible for tasks; a `0` means the id is valid but nobody currently qualifies, so prefer another specialization or widen the audience.
|
|
161
|
+
|
|
159
162
|
`toloka pipeline cost [name] [--item-limit <n>]` forecasts what a run of a live, validated pipeline would bill. It does not start a run or change the graph. If any billed node is marked `skipped`, the displayed total is incomplete and must not be presented as the pipeline price. Compute and label a labor floor for each skipped billed node as `items × pricePerTask × annotations per item`, using `1` for generation and `overlap` for quorum. Also report `maxEntities`, quality-criteria count, and only exclusions applicable to the authored QA and workflow. Apply accepting a rate does not prove its final billed value.
|
|
160
163
|
|
|
161
164
|
## Manage model checkpoints
|
|
@@ -166,7 +169,7 @@ The global Base Model Catalog is read-only and uses `toloka base-model`. `base-m
|
|
|
166
169
|
|
|
167
170
|
`toloka training create` requires one source (`--base-model` or `--model-version`) and a training Dataset. A `--model-version` source must be a LoRA adapter Model Version (one produced by fine-tuning) and works for both methods — SFT continues the adapter, gisting merges it and trains gist tokens over it; a full-weights version is rejected with "Training from a Model Version requires a LoRA adapter source". The first Training Run creates a new output Model named after the Training; a source Model Version is only the training parent, never the output binding.
|
|
168
171
|
|
|
169
|
-
Hyperparameters are optional flags on `training create` and `training update`: `--epochs 1..5` (default 3), `--learning-rate` (SFT 0.00001–0.001, default 0.00015; gisting 0.001–0.3, default 0.03), `--batch-size 8|16|32|64` (default 8; sequences per optimizer step across all GPUs) and `--max-sequence-length 64..16384` (default 12288 for SFT, 3072 for gisting) apply to both methods; `--lora-rank 8|16|32` (default 16) is SFT-only and `--gist-tokens
|
|
172
|
+
Hyperparameters are optional flags on `training create` and `training update`: `--epochs 1..5` (default 3), `--learning-rate` (SFT 0.00001–0.001, default 0.00015; gisting 0.001–0.3, default 0.03), `--batch-size 8|16|32|64` (default 8; sequences per optimizer step across all GPUs) and `--max-sequence-length 64..16384` (default 12288 for SFT, 3072 for gisting) apply to both methods; `--lora-rank 8|16|32` (default 16) is SFT-only and `--gist-tokens` (default 32) is gisting-only — passing one to the other method fails before any request. `--gist-tokens` takes an absolute count `8|32|64` or a fraction of the prompt being compressed (`1/4`, `1/8`, `1/16`, `1/32`), which the run resolves to `ceil(prompt tokens / N)` clamped to 8..256. `--regime` and `--eval-steps` no longer exist. `training update` has no `--method` flag: the method stays as created, and every flag you omit keeps its current value. `training cost` is blocked when the longest example exceeds `--max-sequence-length` or when the training selection leaves fewer rows than `--batch-size` after the 10% holdout.
|
|
170
173
|
|
|
171
174
|
Use `training cost` to validate inputs and show the maximum charge before `training run`. A blocked estimate exits 1 and lists the issues on stderr instead of printing the estimate; fix them and recalculate. `training run` accepts that ready estimate JSON through `--estimate-file`; without it, the command calculates a fresh ready estimate itself before creating the run. `--version-name` defaults to the Training name; pass it when a Model Version with that name already exists. Use `training status`, `training logs`, `training metrics`, `training resume`, and `training cancel` with the returned Training Run ID. `training metrics` lists each metric's latest value while the run is still training; `--key` prints one metric's point history instead.
|
|
172
175
|
|
|
@@ -145,6 +145,8 @@ toloka training metrics <training-run-id> --json
|
|
|
145
145
|
|
|
146
146
|
Resume only after the user confirms a billing block is resolved. Cancel only on explicit request.
|
|
147
147
|
|
|
148
|
+
A run can train longer than the estimate predicted. It stays `running`, the charge stays capped at the approved `maximumCharge`, and the Model Version publishes when training finishes; treat a slow run as still running, not as failed, and do not start a duplicate run for it. A run that ends `failed` with error code `provider_failed` or `provider_timeout` is a platform-side failure: it is charged nothing and the whole hold returns to the team balance, while its reported GPU seconds stay visible; a run the user cancels during training is charged for the GPU seconds used.
|
|
149
|
+
|
|
148
150
|
For a completed run, verify the observed output and download only when requested:
|
|
149
151
|
|
|
150
152
|
```sh
|
|
@@ -115,7 +115,7 @@ toloka training create "<training-name>" \
|
|
|
115
115
|
|
|
116
116
|
Omit `--validation-dataset` when no separate validation selection exists so the provider uses its 10% holdout. For disjoint selections in a Dataset with top-level fields, add `--train-filter <field>=<value>` and `--validation-filter <field>=<value>`.
|
|
117
117
|
|
|
118
|
-
Preserve requested hyperparameters. The current CLI uses `--gist-tokens
|
|
118
|
+
Preserve requested hyperparameters. The current CLI uses `--gist-tokens` (default 32), `--epochs 1..5` (default 3), `--learning-rate` within 0.001–0.3 (default 0.03), `--batch-size 8|16|32|64` (default 8), and `--max-sequence-length 64..16384` (default 3072), rather than `regime`. Use defaults when suitable. `--gist-tokens` takes an absolute count `8|32|64` or a fraction of the prompt being compressed — `1/4`, `1/8`, `1/16`, `1/32`. An absolute count fixes the soft-prompt size: 8 favors compression for simple prompts, while 64 may help when 32 loses holdout accuracy. A fraction fixes the compression ratio instead — the run resolves it to `ceil(prompt tokens / N)` and clamps it to 8..256 — so prefer one for a long prompt or a prompt whose length changes between drafts, starting at `1/32`. `training get` reports the fraction you set, not the count it resolved to. Choose sequence length from actual examples within the supported limit. `--lora-rank` is rejected for gisting. `training update <name-or-id>` takes the same flags except `--method`; omitted flags retain current values.
|
|
119
119
|
|
|
120
120
|
Verify the saved method, exact source, prompt file, selections, filters, and configuration with `toloka training get <training-id> --json`.
|
|
121
121
|
|
|
@@ -154,6 +154,8 @@ toloka training metrics <training-run-id> --json
|
|
|
154
154
|
|
|
155
155
|
Resume only after the user confirms a billing block is resolved. Cancel only on explicit request.
|
|
156
156
|
|
|
157
|
+
A run can train longer than the estimate predicted. It stays `running`, the charge stays capped at the approved `maximumCharge`, and the Model Version publishes when training finishes; treat a slow run as still running, not as failed, and do not start a duplicate run for it. A run that ends `failed` with error code `provider_failed` or `provider_timeout` is a platform-side failure: it is charged nothing and the whole hold returns to the team balance, while its reported GPU seconds stay visible; a run the user cancels during training is charged for the GPU seconds used.
|
|
158
|
+
|
|
157
159
|
For a completed run, verify the observed output and download only when requested:
|
|
158
160
|
|
|
159
161
|
```sh
|
|
@@ -10,7 +10,10 @@ import {
|
|
|
10
10
|
createVirtualTypeScriptEnvironment,
|
|
11
11
|
} from "./typescript-vfs.js";
|
|
12
12
|
|
|
13
|
-
const tsLibDirectory =
|
|
13
|
+
const tsLibDirectory =
|
|
14
|
+
Reflect.get(globalThis, "Bun")?.isStandaloneExecutable === true
|
|
15
|
+
? path.join(path.dirname(process.argv[1]!), "typescript-lib")
|
|
16
|
+
: path.dirname(ts.getDefaultLibFilePath({ target: ts.ScriptTarget.Latest }));
|
|
14
17
|
|
|
15
18
|
const BLOCK_MARKERS = {
|
|
16
19
|
data: "DATA_INJECTION",
|
package/src/cli/analytics.ts
CHANGED
|
@@ -93,6 +93,7 @@ const COMMAND_GROUPS: Readonly<Record<string, CommandGroup>> = {
|
|
|
93
93
|
},
|
|
94
94
|
dataset: { list: true, create: true, fields: true, load: true, upload: true },
|
|
95
95
|
"delivery-order": { list: true },
|
|
96
|
+
audience: { specialization: { list: true } },
|
|
96
97
|
training: {
|
|
97
98
|
list: true,
|
|
98
99
|
get: true,
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { PotatoClient } from "../client";
|
|
2
|
+
import type { Specialization } from "../../openapi/gen/models";
|
|
3
|
+
import { formatTable, type TableIo, writeOutput } from "./table";
|
|
4
|
+
|
|
5
|
+
type AudienceApi = Pick<PotatoClient, "audience">;
|
|
6
|
+
|
|
7
|
+
export const listSpecializations = async (
|
|
8
|
+
client: AudienceApi,
|
|
9
|
+
input: { json: boolean },
|
|
10
|
+
io: TableIo,
|
|
11
|
+
): Promise<void> => {
|
|
12
|
+
const specializations: Specialization[] = [];
|
|
13
|
+
for await (const specialization of client.audience.specializations.listAll()) {
|
|
14
|
+
specializations.push(specialization);
|
|
15
|
+
}
|
|
16
|
+
writeOutput(
|
|
17
|
+
io,
|
|
18
|
+
input.json,
|
|
19
|
+
specializations,
|
|
20
|
+
specializations.length === 0
|
|
21
|
+
? "No specializations available."
|
|
22
|
+
: formatTable([
|
|
23
|
+
["ID", "NAME", "EXPERTS"],
|
|
24
|
+
...specializations.map((specialization) => [
|
|
25
|
+
specialization.id,
|
|
26
|
+
specialization.name,
|
|
27
|
+
String(specialization.expertsCount),
|
|
28
|
+
]),
|
|
29
|
+
]),
|
|
30
|
+
);
|
|
31
|
+
};
|