@toloka-platform/sdk 0.2.0-preview.2 → 0.2.0-preview.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -4
- package/dist/cli/toloka.js +159 -151
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +18 -1
- package/docs/cli-v2.md +4 -3
- package/docs/cli.md +29 -10
- package/docs/getting-started.md +3 -3
- package/docs/limitations.md +1 -1
- package/docs/testing.md +1 -1
- package/docs/troubleshooting.md +1 -1
- package/examples/face-image-collection/README.md +2 -2
- package/openapi/gen/api.ts +78 -0
- package/openapi/gen/models/checkReport.ts +2 -1
- package/openapi/gen/models/checkReportAnyOfTwoseven.ts +5 -0
- package/openapi/gen/models/checkReportAnyOfTwosevenResultsItem.ts +6 -0
- package/openapi/gen/models/checkReportAnyOfTwosevenResultsItemVerdict.ts +2 -0
- package/openapi/gen/models/currentUser.ts +1 -0
- package/openapi/gen/models/deliveryOrder.ts +9 -0
- package/openapi/gen/models/deliveryOrderClientName.ts +2 -0
- package/openapi/gen/models/getTrainingRunMetricHistoryParams.ts +4 -0
- package/openapi/gen/models/index.ts +19 -0
- package/openapi/gen/models/listDeliveryOrders200.ts +5 -0
- package/openapi/gen/models/trainingRunMetricHistory.ts +9 -0
- package/openapi/gen/models/trainingRunMetricHistoryMetric.ts +2 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItem.ts +9 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItemNonFinite.ts +2 -0
- package/openapi/gen/models/trainingRunMetricHistoryPointsItemValue.ts +2 -0
- package/openapi/gen/models/trainingRunMetrics.ts +9 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItem.ts +8 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatest.ts +9 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatestNonFinite.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemLatestValue.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsMetricsItemMetric.ts +2 -0
- package/openapi/gen/models/trainingRunMetricsUnavailableReason.ts +2 -0
- package/package.json +2 -2
- package/skills/catalog.json +55 -7
- package/skills/toloka-build/SKILL.md +10 -7
- package/skills/toloka-build/references/node-semantics.md +6 -4
- package/skills/toloka-build/references/workflow-and-cli.md +21 -13
- package/skills/toloka-fine-tuning/SKILL.md +72 -0
- package/skills/toloka-get-started/SKILL.md +55 -0
- package/skills/toloka-gisting/SKILL.md +76 -0
- package/skills/toloka-report-issue/references/report-template.md +1 -1
- package/src/authoring/index.ts +14 -0
- package/src/checks/task-ui-typecheck.ts +12 -10
- package/src/cli/analytics.ts +52 -6
- package/src/cli/auth.ts +69 -16
- package/src/cli/auto-update.ts +24 -33
- package/src/cli/check.ts +12 -10
- package/src/cli/commands/auth.ts +89 -0
- package/src/cli/commands/config.ts +28 -0
- package/src/cli/commands/dataset.ts +75 -0
- package/src/cli/commands/delivery-order.ts +18 -0
- package/src/cli/commands/models.ts +122 -0
- package/src/cli/commands/pipeline.ts +132 -0
- package/src/cli/commands/project.ts +175 -0
- package/src/cli/commands/report.ts +39 -0
- package/src/cli/commands/secret.ts +39 -0
- package/src/cli/commands/skill.ts +36 -0
- package/src/cli/commands/source.ts +204 -0
- package/src/cli/commands/status.ts +18 -0
- package/src/cli/commands/training.ts +228 -0
- package/src/cli/commands/version.ts +14 -0
- package/src/cli/config.ts +5 -6
- package/src/cli/context.ts +213 -0
- package/src/cli/delivery-orders.ts +30 -0
- package/src/cli/environment.ts +26 -0
- package/src/cli/init.ts +1 -4
- package/src/cli/program.ts +168 -1344
- package/src/cli/project-context.ts +9 -11
- package/src/cli/sdk-version.ts +38 -0
- package/src/cli/skill.ts +6 -2
- package/src/cli/terminal-io.ts +103 -0
- package/src/cli/trainings.ts +118 -27
- package/src/cli/version.ts +1 -1
- package/src/client.ts +18 -0
- package/src/importer.ts +32 -2
package/skills/catalog.json
CHANGED
|
@@ -3,18 +3,16 @@
|
|
|
3
3
|
{
|
|
4
4
|
"id": "toloka-build",
|
|
5
5
|
"name": "Build Toloka pipelines as code",
|
|
6
|
-
"description": "
|
|
6
|
+
"description": "Build, test, inspect, or change Toloka platform pipeline projects as TypeScript source with @toloka-platform/sdk, toloka check, and toloka apply. Use when the user has selected pipeline authoring or wants to inspect, plan, apply, test, check, or run a pipeline project as code.",
|
|
7
7
|
"tags": [
|
|
8
8
|
"pipelines",
|
|
9
9
|
"authoring",
|
|
10
10
|
"sdk",
|
|
11
|
-
"onboarding",
|
|
12
|
-
"getting-started",
|
|
13
11
|
"testing",
|
|
14
12
|
"checks"
|
|
15
13
|
],
|
|
16
|
-
"version": "0.2.0-preview.
|
|
17
|
-
"digest": "
|
|
14
|
+
"version": "0.2.0-preview.3",
|
|
15
|
+
"digest": "b5819b5f29ed0876850708e3fbe9b370433ddf68d9548260cf24ff7200104bb6",
|
|
18
16
|
"files": [
|
|
19
17
|
"SKILL.md",
|
|
20
18
|
"references/node-semantics.md",
|
|
@@ -31,12 +29,62 @@
|
|
|
31
29
|
"bug",
|
|
32
30
|
"report"
|
|
33
31
|
],
|
|
34
|
-
"version": "0.2.0-preview.
|
|
35
|
-
"digest": "
|
|
32
|
+
"version": "0.2.0-preview.3",
|
|
33
|
+
"digest": "76229bae596f8a1722ce7fc55bea3dd66698df5b61baa31f6cb98ca2b5fa03e8",
|
|
36
34
|
"files": [
|
|
37
35
|
"SKILL.md",
|
|
38
36
|
"references/report-template.md"
|
|
39
37
|
]
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "toloka-get-started",
|
|
41
|
+
"name": "Get started with Toloka",
|
|
42
|
+
"description": "Route a first Toloka agent session to the correct workflow and establish safe CLI, authentication, project, and approval checkpoints. Use when the user is new to Toloka, asks what to do first, or has not selected a pipeline, fine-tuning, or gisting workflow.",
|
|
43
|
+
"tags": [
|
|
44
|
+
"toloka",
|
|
45
|
+
"onboarding",
|
|
46
|
+
"getting-started",
|
|
47
|
+
"agent"
|
|
48
|
+
],
|
|
49
|
+
"version": "0.2.0-preview.3",
|
|
50
|
+
"digest": "7e5e2d4b8daab4abece217ca2a00f76ce4e29da6a2724a44a91ff3dc63aec38a",
|
|
51
|
+
"files": [
|
|
52
|
+
"SKILL.md"
|
|
53
|
+
]
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"id": "toloka-fine-tuning",
|
|
57
|
+
"name": "Fine-tune a model with Toloka",
|
|
58
|
+
"description": "Create, estimate, start, monitor, and verify a project-owned supervised fine-tuning Training through the Toloka CLI. Use when the user wants to adapt a base model or existing Model Version from labeled conversations.",
|
|
59
|
+
"tags": [
|
|
60
|
+
"toloka",
|
|
61
|
+
"training",
|
|
62
|
+
"fine-tuning",
|
|
63
|
+
"sft",
|
|
64
|
+
"models"
|
|
65
|
+
],
|
|
66
|
+
"version": "0.2.0-preview.3",
|
|
67
|
+
"digest": "967a3e862d4a79f6e6462f2ab6d03241b45c24f5ae5550a453259f01289eb85e",
|
|
68
|
+
"files": [
|
|
69
|
+
"SKILL.md"
|
|
70
|
+
]
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"id": "toloka-gisting",
|
|
74
|
+
"name": "Compress a prompt with gisting",
|
|
75
|
+
"description": "Create, estimate, start, monitor, and verify a project-owned prompt-gisting Training through the Toloka CLI. Use when the user wants a compact prompt representation from example conversations and a prompt file.",
|
|
76
|
+
"tags": [
|
|
77
|
+
"toloka",
|
|
78
|
+
"training",
|
|
79
|
+
"gisting",
|
|
80
|
+
"prompt",
|
|
81
|
+
"models"
|
|
82
|
+
],
|
|
83
|
+
"version": "0.2.0-preview.3",
|
|
84
|
+
"digest": "e7f714ec5681648cba092a1ed737f2e6431da1da47f8f3553c4d774777af644a",
|
|
85
|
+
"files": [
|
|
86
|
+
"SKILL.md"
|
|
87
|
+
]
|
|
40
88
|
}
|
|
41
89
|
]
|
|
42
90
|
}
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: toloka-build
|
|
3
|
-
description:
|
|
3
|
+
description: Build, test, inspect, or change Toloka platform pipeline projects as TypeScript source with @toloka-platform/sdk, toloka check, and toloka apply. Use when the user has selected pipeline authoring or wants to inspect, plan, apply, test, check, or run a pipeline project as code.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Build Toloka pipelines as code
|
|
7
7
|
|
|
8
8
|
This SDK is Beta software. The authoring API, the CLI, and generated source can change before general availability. Do not invent helpers or flags that are not in the installed package.
|
|
9
9
|
|
|
10
|
-
Use this skill when the user
|
|
10
|
+
Use this skill when the user has selected pipeline authoring or wants to create, inspect, test, or change a Toloka pipeline project as code. Work in the project source tree; do not translate the request into direct platform API calls.
|
|
11
11
|
|
|
12
|
-
##
|
|
12
|
+
## Enter the pipeline workflow
|
|
13
13
|
|
|
14
|
-
When
|
|
14
|
+
When a user is new to Toloka or has not selected a workflow, run `toloka skill install toloka-get-started`, read `.agents/skills/toloka-get-started/SKILL.md`, and follow the matching public runbook first. This skill starts after pipeline work is selected and owns pipeline authoring semantics, local tests, recovery, and evidence reporting.
|
|
15
15
|
|
|
16
|
-
Do not replace the
|
|
16
|
+
Do not replace the selected runbook's quick demo with another example, and do not run the demo when the user chooses their own task. Never cross its API-key, project-creation, plan, or run-cost checkpoint without the user's required action or approval.
|
|
17
17
|
|
|
18
18
|
## Follow the reconciliation loop
|
|
19
19
|
|
|
@@ -41,7 +41,7 @@ Use the directory containing `toloka.json` as the source root. Commands default
|
|
|
41
41
|
|
|
42
42
|
If `toloka project current` reports `from toloka.json` for a project you did not select in this checkout, stop: project create/add/switch does not rebind an existing checkout. A newly generated state file with empty `pipelines` and `datasets` can be removed before selecting the intended project and rerunning `toloka init`; any state file with live bindings may be someone else's checkout and must not be deleted without user approval.
|
|
43
43
|
|
|
44
|
-
Never assume a project ID from conversation context. Resolve it from `--project-id`, `TOLOKA_PROJECT_ID
|
|
44
|
+
Never assume a project ID from conversation context. Resolve it from `--project-id`, `TOLOKA_PROJECT_ID`, `toloka.json`, or `toloka project current`.
|
|
45
45
|
|
|
46
46
|
`project create`, `project add`, and `project switch` also change the machine-level current-project registry used outside a source root. For temporary or evaluation projects, record the previous alias and restore it before finishing. If create requires `--team-id`, use the team from `toloka status --project-id <authorized-project>` only when that project is known to share the intended owner; state the assumption and confirm the new project's team in `toloka project url`. With no such project, ask the user for the team ID.
|
|
47
47
|
|
|
@@ -50,6 +50,7 @@ Never assume a project ID from conversation context. Resolve it from `--project-
|
|
|
50
50
|
Import authoring helpers from `@toloka-platform/sdk`.
|
|
51
51
|
|
|
52
52
|
- Declare every data contract once with `field(name, jsonSchema)`.
|
|
53
|
+
- Decide entity cardinality from the task before choosing field schemas or Task UI components. One logical output from one labeler is a static entity, even when a component represents it with an array; only multiple independent outputs are dynamic. For dynamic entities, choose a tight task-grounded `maxEntities` instead of defaulting to 50.
|
|
53
54
|
- Do not invent enum, length, range, format, label-precedence, tie-break, default, exact-phrase, or conditional-requiredness rules absent from the request or observed data. Before apply, list every authored rule the request did not fix—including criteria, quiz answers, and UI validation—as an assumption; choose the least surprising option and carry that list into the final report.
|
|
54
55
|
- Use exactly these constructors: `root`, `code`, `generation`, `match`, and `quorum`.
|
|
55
56
|
- Connect typed ports with `connect(from, to)`: `started`, `completed`, `input`, `output`, `matchNode.matched(0)`, and `default`.
|
|
@@ -79,6 +80,8 @@ These are server rules that `definePipeline` does not check. Each one costs an a
|
|
|
79
80
|
|
|
80
81
|
Before choosing Task UI components, call `ui.componentDocumentation()` from a local `@toloka-platform/sdk/testing` scenario. A component's usable contract is the returned `propsType` plus its returned `valueType` and `example`; shared `Field` props such as `path`, `label`, `hint`, and `validation` can be confirmed from any current `Field.*` example, but do not invent any other prop. For structured or array-valued Fields, derive the dataset field schema from `valueType`, then feed the documented example value through `setValue`, `getValue`, and `submit` in the UI scenario so upstream contract drift fails locally. Never translate undocumented property names or replace missing values with defaults. Every identifier in `ui.code` must be a returned component or documented `actions` API, and JSX must have one root element. Register each required UI scenario with `test.ui(pipeline, node, name, callback)` and submit through its bound `render` helper. Run `toloka check <pipeline> --node <name-or-id>` after every Task UI edit; require passing UI types, empty browser validation errors, valid postprocessed entity output, and visual inspection where relevant. Never weaken authored validation merely to make a test pass.
|
|
81
82
|
|
|
83
|
+
`ui.code` accepts TSX, including TypeScript annotations. `preprocess` and `postprocess` are JavaScript function bodies; do not put TypeScript-only syntax in either transform.
|
|
84
|
+
|
|
82
85
|
Register code behavior with `test.code(pipeline, node, name, callback)` and execute the bound `run` helper. Add at least one `test.pipeline(pipeline, name, callback)` scenario. When a local dataset file is available, use its first row as the initial item state; redact sensitive values while preserving field names, nesting, optionality, and media/value shapes. Use synthetic input only when no local row exists. Treat the fixture as `root.started` item state, execute connected nodes in order with the callback's bound code and UI helpers, merge only each node's declared outputs, and assert the final `root.completed` fields. One scenario is appropriate for a linear happy path; use separate scenarios for different match branches, optional paths, and failures.
|
|
83
86
|
|
|
84
87
|
## Keep data and credentials out of source
|
|
@@ -89,7 +92,7 @@ Register code behavior with `test.code(pipeline, node, name, callback)` and exec
|
|
|
89
92
|
## Credentials
|
|
90
93
|
|
|
91
94
|
`toloka login` verifies an API key and stores it with owner-only permissions, so no later command needs
|
|
92
|
-
`--api-key`. Precedence is flag, then environment variable (`
|
|
95
|
+
`--api-key`. Precedence is flag, then environment variable (`TOLOKA_API_KEY`, `TOLOKA_BASE_URL`), then
|
|
93
96
|
that file. `toloka auth-status` reports what is in effect and where it came from; `toloka logout`
|
|
94
97
|
removes it. If a command reports missing credentials, run `toloka login`; never invent a key.
|
|
95
98
|
Anonymous command telemetry is on by default; set `TOLOKA_TELEMETRY=0` or `DO_NOT_TRACK=1` to disable it.
|
|
@@ -137,10 +137,11 @@ Use `about.projectType: 'data_collection'` when experts produce new original con
|
|
|
137
137
|
|
|
138
138
|
#### Entities
|
|
139
139
|
|
|
140
|
-
An authored entity references a declared `field`.
|
|
140
|
+
An authored entity references a declared `field`. Choose its cardinality from the task's meaning before choosing the field schema or Task UI component:
|
|
141
141
|
|
|
142
|
-
-
|
|
143
|
-
-
|
|
142
|
+
- One logical result from one labeler is static. A caption, verdict, rating, recording, or upload remains one entity even when its value is an object or a component holds it in an array. Omit `dynamic` and `maxEntities`, and use a `jsonPath` without `[*]`.
|
|
143
|
+
- Multiple independent results from one labeler are dynamic. Examples include one box per visible face and one defect record per discovered defect. Set `dynamic: true`, use exactly one `[*]` in `jsonPath`, and set `maxEntities` to the tightest upper bound justified by the task. Never use 50 as an unexamined default.
|
|
144
|
+
- Do not choose `dynamic` from a TypeScript or JSON container shape. Ask whether the task expects one logical result or a repeated collection; the answer owns the entity model.
|
|
144
145
|
- Use `allowedLabels: [{ label, value }]` for fixed choices. Use `SINGLE_LABEL` for exactly one stored value and `MULTI_LABEL` for multiple values.
|
|
145
146
|
- Set media duration bounds only when the expert produces the media through an upload or recording output. Input media used for annotation does not require duration bounds.
|
|
146
147
|
- Give every output entity at least one relation to the input it uses: `{ field, rel_type: 'context' | 'references' | 'depends_on' | 'evaluates' }`, or a conditional relation supported by the public type.
|
|
@@ -149,7 +150,7 @@ An authored entity references a declared `field`. Set `entityType`, `optional`,
|
|
|
149
150
|
|
|
150
151
|
Entities enforce format, not meaning. Put types, ranges, cardinality, requiredness, and labels in the field schema and entity. Put accuracy, correctness, completeness, and clarity in quality criteria.
|
|
151
152
|
|
|
152
|
-
For
|
|
153
|
+
For a single-item upload or recording control, the component value is still an array, but the task produces one logical entity. Declare a scalar output field, keep the entity static, and use a non-wildcard path to the completed first value, such as `$.video_file[0].downloadLink`. Do not model the component's array container as `dynamic: true`. Use the matching media or document entity type. For video, also set both entity duration bounds and align them with the UI component.
|
|
153
154
|
|
|
154
155
|
#### Quality and instructions
|
|
155
156
|
|
|
@@ -262,6 +263,7 @@ Set `MatchAuthoringSettings`, derived from `MatchNodeSettings`, through `match(n
|
|
|
262
263
|
- `about`: `name`, optional `description`, optional `projectType`.
|
|
263
264
|
- `input`: every declared `Field` referenced by a predicate, each with `optional`.
|
|
264
265
|
- `conditions`: ordered branches with optional `logic: 'and' | 'or'` and required `predicates`.
|
|
266
|
+
- Each condition `name` is a unique identifier within its Match node, from 2 through 32 characters: start with a lowercase letter, use only lowercase letters, digits, and underscores, and end with a letter or digit. Use `is_disputed`, not `is disputed` or `field_type = general_info`. `match()` throws locally for invalid or duplicate names.
|
|
265
267
|
- Each authored predicate uses `field`, `jsonPath`, `operator`, and scalar `value`.
|
|
266
268
|
|
|
267
269
|
The current public type accepts only `operator: 'text_eq' | 'text_neq'` and `value: string | number | boolean | null`. Do not copy broader internal operator lists into source. Use `jsonPath: '$'` for the whole scalar or a restricted path into an object.
|
|
@@ -51,9 +51,10 @@ It explains where to generate an API key — `console.toloka.ai` → Company →
|
|
|
51
51
|
credentials file with mode `0600`. Nothing is written when the key does not authenticate.
|
|
52
52
|
|
|
53
53
|
Most later commands read that file, so `--api-key` becomes optional. For networked commands that do
|
|
54
|
-
not mutate the machine project registry, key precedence is flag, then `
|
|
54
|
+
not mutate the machine project registry, key precedence is flag, then `TOLOKA_API_KEY`, then the
|
|
55
55
|
stored file; CI can use an environment variable without running login. Non-interactive callers can
|
|
56
|
-
skip the login prompt
|
|
56
|
+
skip the login prompt by piping the key to `toloka login --api-key-stdin`; `toloka login --api-key <key>`
|
|
57
|
+
also works but leaves the key in shell history. The base URL defaults to the platform.
|
|
57
58
|
|
|
58
59
|
```sh
|
|
59
60
|
toloka auth-status # or toloka whoami
|
|
@@ -61,22 +62,25 @@ toloka logout
|
|
|
61
62
|
```
|
|
62
63
|
|
|
63
64
|
`auth-status` reports which platform and key are in effect and where each came from, masks the key
|
|
64
|
-
unless you pass `--show-token`, and exits non-zero
|
|
65
|
-
|
|
65
|
+
unless you pass `--show-token`, and exits non-zero on failure — use it to check a setup rather than
|
|
66
|
+
guessing. Its last lines say which failure it was: `Rejected:` means the platform refused the key
|
|
67
|
+
(run `toloka login` with a working one); `Not reachable:` means the platform did not answer, so the
|
|
68
|
+
key was not evaluated (fix network access, do not ask for a new key); `Platform error:` means the
|
|
69
|
+
platform failed (retry later). `logout` deletes the stored file; it does not revoke the key.
|
|
66
70
|
|
|
67
71
|
A globally installed `toloka` auto-updates by default. It checks npm at most once per day and installs a newer `@toloka-platform/sdk` only after that version has been published for at least 24 hours, then re-runs the original command. `config`, `version`, and `help` never trigger a check. Project-local installs and CI skip it. Turn it off with `toloka config set auto-update false`.
|
|
68
72
|
|
|
69
73
|
## Configure the CLI
|
|
70
74
|
|
|
71
|
-
Provide the API key and base URL on networked workflows through `
|
|
72
|
-
`
|
|
75
|
+
Provide the API key and base URL on networked workflows through `TOLOKA_API_KEY` and
|
|
76
|
+
`TOLOKA_BASE_URL`, or with `--api-key` and `--base-url`. The registry-mutating commands
|
|
73
77
|
`toloka project add`, `toloka project switch`, `toloka project remove`, and `toloka project create`
|
|
74
78
|
require `toloka login` first because they read and write the credentials file. When flags or
|
|
75
|
-
`
|
|
76
|
-
project ID in this order: `--project-id`, `TOLOKA_PROJECT_ID`, `
|
|
79
|
+
`TOLOKA_*` variables are also present, they must identify the same stored key and platform. Resolve
|
|
80
|
+
project ID in this order: `--project-id`, `TOLOKA_PROJECT_ID`, `toloka.json`,
|
|
77
81
|
then the machine registry `current`.
|
|
78
82
|
|
|
79
|
-
Anonymous command telemetry is on by default (`sdk_cli_command_completed` / `sdk_cli_command_failed`). `toloka report` uses the same PostHog client to send explicitly approved Markdown in `sdk_cli_issue_report_submitted`. Set `TOLOKA_TELEMETRY=0` or `DO_NOT_TRACK=1` to disable
|
|
83
|
+
Anonymous command telemetry is on by default (`sdk_cli_command_completed` / `sdk_cli_command_failed`). Failed commands with exceptions also emit a sanitized PostHog Error Tracking `$exception`; API failures include the status code and server request ID when available. Raw arguments, messages, stacks, response bodies, and paths are excluded. `toloka report` uses the same PostHog client to send explicitly approved Markdown in `sdk_cli_issue_report_submitted`. Set `TOLOKA_TELEMETRY=0` or `DO_NOT_TRACK=1` to disable all capture; CI is off unless `TOLOKA_TELEMETRY=1`.
|
|
80
84
|
|
|
81
85
|
All commands accept `--source-root <directory>`. Prefer running from the source root so paths and diffs stay obvious. `init` does not walk up: it binds the working directory or `--source-root` only.
|
|
82
86
|
|
|
@@ -98,8 +102,9 @@ toloka project switch <alias>
|
|
|
98
102
|
toloka project current
|
|
99
103
|
toloka project url
|
|
100
104
|
toloka project remove <alias>
|
|
101
|
-
toloka project create [alias] [--name <name>] [--team-id <id>]
|
|
105
|
+
toloka project create [alias] [--name <name>] [--team-id <id>] [--delivery-order-id <id>]
|
|
102
106
|
toloka project untrack <pipeline|dataset> <name> [--confirm]
|
|
107
|
+
toloka delivery-order list
|
|
103
108
|
toloka init
|
|
104
109
|
toloka import [projectId]
|
|
105
110
|
toloka check [pipeline] [--node <name-or-id>] [--json] [--watch]
|
|
@@ -131,19 +136,22 @@ toloka training get <name-or-id> [--json]
|
|
|
131
136
|
toloka training create <name> (--base-model <id> | --model-version <id>) --dataset <id> [--json]
|
|
132
137
|
toloka training update <name-or-id> [configuration flags] [--json]
|
|
133
138
|
toloka training cost <name-or-id> [--idempotency-key <key>] [--json]
|
|
134
|
-
toloka training run <name-or-id> --estimate-file <path> --version-name <name> [--idempotency-key <key>] [--json]
|
|
139
|
+
toloka training run <name-or-id> [--estimate-file <path>] [--version-name <name>] [--idempotency-key <key>] [--json]
|
|
135
140
|
toloka training runs <name-or-id> [--json]
|
|
136
141
|
toloka training status <training-run-id> [--json]
|
|
137
142
|
toloka training resume <training-run-id> [--json]
|
|
138
143
|
toloka training cancel <training-run-id> [--json]
|
|
139
144
|
toloka training logs <training-run-id> [--follow]
|
|
145
|
+
toloka training metrics <training-run-id> [--key <metric>] [--resolution <n>] [--json]
|
|
140
146
|
toloka skill list [--json]
|
|
141
147
|
toloka skill search <query> [--json]
|
|
142
148
|
toloka skill install <skill-id> [--force]
|
|
143
149
|
toloka report <markdown> [--authored-by human|agent] [--confirm]
|
|
144
150
|
```
|
|
145
151
|
|
|
146
|
-
Select the platform project with `toloka project add` or `toloka project create` before `import`, `plan`, or `apply`. For `init`, `toloka --project-id <id> init
|
|
152
|
+
Select the platform project with `toloka project add` or `toloka project create` before `import`, `plan`, or `apply`. For `init`, `toloka --project-id <id> init` and `TOLOKA_PROJECT_ID` bind an empty checkout directly without changing the machine registry. `project create` also writes `toloka.json`; `project add` changes only the machine registry, so follow it with `import <projectId>` for an existing project or an explicit project ID on `init` for an empty one. Confirm `toloka project current` and `toloka.json` resolve to the intended ID before authoring.
|
|
153
|
+
|
|
154
|
+
`toloka delivery-order list` and `project create --delivery-order-id` appear only when `toloka whoami` prints `Internal: true`. For an internal user, list active Sunday orders first; if the list is empty, stop. Pass a selected id to `project create` — internal teams require it. When `Internal: false`, omit the flag and do not run `delivery-order`.
|
|
147
155
|
|
|
148
156
|
`toloka project url` prints only the selected project's platform UI URL. Use it when the user needs a working link to inspect the live project.
|
|
149
157
|
|
|
@@ -157,7 +165,7 @@ The global Base Model Catalog is read-only and uses `toloka base-model`. `base-m
|
|
|
157
165
|
|
|
158
166
|
`toloka training create` requires one source (`--base-model` or `--model-version`) and a training Dataset. The output Model is created on the first Training Run from the Training name, or bound from an existing Model Version source.
|
|
159
167
|
|
|
160
|
-
Use `training cost` to validate inputs and
|
|
168
|
+
Use `training cost` to validate inputs and show the maximum charge before `training run`. `training run` accepts that ready estimate JSON through `--estimate-file`; without it, the command calculates a fresh ready estimate itself before creating the run. `--version-name` defaults to the Training name; pass it when a Model Version with that name already exists. Use `training status`, `training logs`, `training metrics`, `training resume`, and `training cancel` with the returned Training Run ID. `training metrics` lists each metric's latest value while the run is still training; `--key` prints one metric's point history instead.
|
|
161
169
|
|
|
162
170
|
## Adopt an existing live project
|
|
163
171
|
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: toloka-fine-tuning
|
|
3
|
+
description: Create, estimate, start, monitor, and verify a project-owned supervised fine-tuning Training through the Toloka CLI. Use when the user wants to adapt a base model or existing Model Version from labeled conversations.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Fine-tune a model with Toloka
|
|
7
|
+
|
|
8
|
+
Use this skill for supervised fine-tuning. For the full first-time agent sequence, read `https://platform.toloka.ai/agents/fine-tuning.md`.
|
|
9
|
+
|
|
10
|
+
## Rules
|
|
11
|
+
|
|
12
|
+
- Never ask for an API key in chat; the user enters it through `toloka login`.
|
|
13
|
+
- Never invent project, Dataset, Base Model, Training, Model, Model Version, price, or run IDs.
|
|
14
|
+
- Use authorized project Datasets containing labeled conversations. Keep production data out of source control.
|
|
15
|
+
- Prefer a separate validation Dataset. Without one, the provider holds out 10% of training rows.
|
|
16
|
+
- Do not start a Training Run before the user approves the ready estimate's `maximumCharge`.
|
|
17
|
+
- Use `--model-version` only for an existing Model Version source.
|
|
18
|
+
- Keep polling bounded and report terminal state only when the CLI shows it.
|
|
19
|
+
|
|
20
|
+
## Workflow
|
|
21
|
+
|
|
22
|
+
1. Verify `bun --version`, `toloka --version`, `toloka whoami`, and `toloka project current`.
|
|
23
|
+
2. Inspect the exact training Dataset ID supplied by the user:
|
|
24
|
+
```sh
|
|
25
|
+
toloka dataset fields <training-dataset-id>
|
|
26
|
+
```
|
|
27
|
+
3. Verify the supplied Base Model ID or existing Model Version source:
|
|
28
|
+
```sh
|
|
29
|
+
toloka base-model get <base-model-id>
|
|
30
|
+
```
|
|
31
|
+
4. Ask the user to choose the source, training Dataset, optional validation Dataset, and Training name.
|
|
32
|
+
5. Create the normalized SFT Training:
|
|
33
|
+
```sh
|
|
34
|
+
toloka training create "<training-name>" \
|
|
35
|
+
--base-model <base-model-id> \
|
|
36
|
+
--dataset <training-dataset-id> \
|
|
37
|
+
--validation-dataset <validation-dataset-id> \
|
|
38
|
+
--json
|
|
39
|
+
```
|
|
40
|
+
Omit `--validation-dataset` when the user chooses provider holdout. Use `--model-version <model-version-id>` instead of `--base-model` for an existing Model Version source.
|
|
41
|
+
6. Verify the draft:
|
|
42
|
+
```sh
|
|
43
|
+
toloka training get <training-id> --json
|
|
44
|
+
```
|
|
45
|
+
7. Calculate a ready estimate to an absolute path:
|
|
46
|
+
```sh
|
|
47
|
+
toloka training cost <training-id> --json > <absolute-estimate-file>
|
|
48
|
+
```
|
|
49
|
+
Report rows, tokens, maximum GPU seconds, `maximumCharge`, currency, and fingerprint. Fix every blocked issue and recalculate after changing inputs.
|
|
50
|
+
8. Ask the user to approve the exact `maximumCharge`. After approval, start one run:
|
|
51
|
+
```sh
|
|
52
|
+
toloka training run <training-id> \
|
|
53
|
+
--estimate-file <absolute-estimate-file> \
|
|
54
|
+
--version-name <unique-version-name> \
|
|
55
|
+
--json
|
|
56
|
+
```
|
|
57
|
+
Both flags are optional. Without `--estimate-file`, `training run` calculates a fresh ready estimate itself; without `--version-name`, the Model Version takes the Training name. Passing the approved estimate prevents a fresh quote from replacing the reviewed one.
|
|
58
|
+
9. Monitor with bounded commands:
|
|
59
|
+
```sh
|
|
60
|
+
toloka training status <training-run-id> --json
|
|
61
|
+
toloka training logs <training-run-id>
|
|
62
|
+
toloka training metrics <training-run-id> --json
|
|
63
|
+
```
|
|
64
|
+
Resume only after the user confirms a billing block is resolved. Cancel only on explicit request.
|
|
65
|
+
10. When completed, verify the output Model and Model Version, then download if requested:
|
|
66
|
+
```sh
|
|
67
|
+
toloka training get <training-id> --json
|
|
68
|
+
toloka model get <model-id> --json
|
|
69
|
+
toloka model version download <model-version-id> --out <absolute-output-directory>
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
A completed Training Run produces a durable Model Version. It does not prove deployment or inference readiness.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: toloka-get-started
|
|
3
|
+
description: Route a first Toloka agent session to the correct workflow and establish safe CLI, authentication, project, and approval checkpoints. Use when the user is new to Toloka, asks what to do first, or has not selected a pipeline, fine-tuning, or gisting workflow.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Get started with Toloka
|
|
7
|
+
|
|
8
|
+
Use this skill to orient a new session. Choose one workflow before making a platform mutation:
|
|
9
|
+
|
|
10
|
+
- **Label data or build a pipeline:** install and read `toloka-build`, then follow `https://platform.toloka.ai/agents/data-labeling.md`.
|
|
11
|
+
- **Fine-tune a model:** install and read `toloka-fine-tuning`, then follow `https://platform.toloka.ai/agents/fine-tuning.md`.
|
|
12
|
+
- **Compress a prompt with gisting:** install and read `toloka-gisting`, then follow `https://platform.toloka.ai/agents/gisting.md`.
|
|
13
|
+
|
|
14
|
+
## Prepare the CLI
|
|
15
|
+
|
|
16
|
+
Check the required runtime and CLI:
|
|
17
|
+
|
|
18
|
+
```sh
|
|
19
|
+
bun --version
|
|
20
|
+
toloka --version
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
The runbooks above use the `preview` channel of `@toloka-platform/sdk`, so `toloka --version` must print a `-preview` version. If `toloka` is unavailable or comes from another channel, ask the user to run the installer in their own shell:
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
curl -fsSL https://platform.toloka.ai/cli/install.sh | TOLOKA_CLI_CHANNEL=preview sh
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Do not request an API key in chat or run an unattended installer. Ask the user to run `toloka login`, then verify the intended account with:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
toloka whoami
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
`Rejected:` means the key was refused (ask for `toloka login` again); `Not reachable:` means the platform did not answer and the key was not evaluated (fix network access, do not ask for a new key); `Platform error:` means the platform failed (retry later).
|
|
36
|
+
|
|
37
|
+
Use a source root for pipeline work. Run `toloka init` before searching or installing skills:
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
toloka init
|
|
41
|
+
toloka skill search <workflow>
|
|
42
|
+
toloka skill install <skill-id>
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Read the installed skill before acting. Refresh a stale skill with `--force`.
|
|
46
|
+
|
|
47
|
+
## Shared checkpoints
|
|
48
|
+
|
|
49
|
+
- Resolve the project from CLI output, `toloka.json`, or `toloka project current`. Never reconstruct an ID.
|
|
50
|
+
- Ask the user to choose a project and team before creating or adopting one.
|
|
51
|
+
- Keep credentials and production data out of source control and agent messages.
|
|
52
|
+
- Show the complete plan before `toloka apply`.
|
|
53
|
+
- Show the ready Training Cost Estimate and obtain approval for `maximumCharge` before starting a Training Run.
|
|
54
|
+
- Keep waits bounded. A timeout is still running, not completed or failed.
|
|
55
|
+
- Report only observed IDs, states, prices, and errors.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: toloka-gisting
|
|
3
|
+
description: Create, estimate, start, monitor, and verify a project-owned prompt-gisting Training through the Toloka CLI. Use when the user wants a compact prompt representation from example conversations and a prompt file.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Compress a prompt with gisting
|
|
7
|
+
|
|
8
|
+
Use this skill for prompt gisting. For the full first-time agent sequence, read `https://platform.toloka.ai/agents/gisting.md`.
|
|
9
|
+
|
|
10
|
+
## Rules
|
|
11
|
+
|
|
12
|
+
- Never ask for an API key in chat; the user enters it through `toloka login`.
|
|
13
|
+
- Never invent project, Dataset, Base Model, Training, Model, Model Version, price, prompt, or run IDs.
|
|
14
|
+
- Use authorized project Datasets containing representative examples. Keep production data out of source control.
|
|
15
|
+
- The prompt file must be non-empty UTF-8 text no larger than 256 KiB.
|
|
16
|
+
- Prefer a separate validation Dataset. Without one, the provider holds out 10% of training rows.
|
|
17
|
+
- Do not start a Training Run before the user approves the ready estimate's `maximumCharge`.
|
|
18
|
+
- Use `--model-version` only for an existing Model Version source.
|
|
19
|
+
- Keep polling bounded and report terminal state only when the CLI shows it.
|
|
20
|
+
|
|
21
|
+
## Workflow
|
|
22
|
+
|
|
23
|
+
1. Verify `bun --version`, `toloka --version`, `toloka whoami`, and `toloka project current`.
|
|
24
|
+
2. Inspect the exact training Dataset ID supplied by the user:
|
|
25
|
+
```sh
|
|
26
|
+
toloka dataset fields <training-dataset-id>
|
|
27
|
+
```
|
|
28
|
+
3. Ask the user for an absolute prompt-file path and validate that it is non-empty UTF-8 text within the size limit.
|
|
29
|
+
4. Verify the supplied Base Model ID or existing Model Version source:
|
|
30
|
+
```sh
|
|
31
|
+
toloka base-model get <base-model-id>
|
|
32
|
+
```
|
|
33
|
+
5. Ask the user to choose the source, training Dataset, optional validation Dataset, prompt file, and Training name.
|
|
34
|
+
6. Create the normalized gisting Training:
|
|
35
|
+
```sh
|
|
36
|
+
toloka training create "<training-name>" \
|
|
37
|
+
--method gisting \
|
|
38
|
+
--prompt-file <absolute-prompt-path> \
|
|
39
|
+
--base-model <base-model-id> \
|
|
40
|
+
--dataset <training-dataset-id> \
|
|
41
|
+
--validation-dataset <validation-dataset-id> \
|
|
42
|
+
--json
|
|
43
|
+
```
|
|
44
|
+
Omit `--validation-dataset` when the user chooses provider holdout. Use `--model-version <model-version-id>` instead of `--base-model` for an existing Model Version source.
|
|
45
|
+
7. Verify the draft:
|
|
46
|
+
```sh
|
|
47
|
+
toloka training get <training-id> --json
|
|
48
|
+
```
|
|
49
|
+
8. Calculate a ready estimate to an absolute path:
|
|
50
|
+
```sh
|
|
51
|
+
toloka training cost <training-id> --json > <absolute-estimate-file>
|
|
52
|
+
```
|
|
53
|
+
Report rows, tokens, maximum GPU seconds, `maximumCharge`, currency, and fingerprint. Fix every blocked issue and recalculate after changing inputs.
|
|
54
|
+
9. Ask the user to approve the exact `maximumCharge`. After approval, start one run:
|
|
55
|
+
```sh
|
|
56
|
+
toloka training run <training-id> \
|
|
57
|
+
--estimate-file <absolute-estimate-file> \
|
|
58
|
+
--version-name <unique-version-name> \
|
|
59
|
+
--json
|
|
60
|
+
```
|
|
61
|
+
Both flags are optional. Without `--estimate-file`, `training run` calculates a fresh ready estimate itself; without `--version-name`, the Model Version takes the Training name. Passing the approved estimate prevents a fresh quote from replacing the reviewed one.
|
|
62
|
+
10. Monitor with bounded commands:
|
|
63
|
+
```sh
|
|
64
|
+
toloka training status <training-run-id> --json
|
|
65
|
+
toloka training logs <training-run-id>
|
|
66
|
+
toloka training metrics <training-run-id> --json
|
|
67
|
+
```
|
|
68
|
+
Resume only after the user confirms a billing block is resolved. Cancel only on explicit request.
|
|
69
|
+
11. When completed, verify the output Model and Model Version, then download if requested:
|
|
70
|
+
```sh
|
|
71
|
+
toloka training get <training-id> --json
|
|
72
|
+
toloka model get <model-id> --json
|
|
73
|
+
toloka model version download <model-version-id> --out <absolute-output-directory>
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
A completed Training Run produces a durable Model Version. It does not prove deployment or inference readiness.
|
|
@@ -26,7 +26,7 @@ The CLI exits 1 with `Potato API request failed with status 500`. The same comma
|
|
|
26
26
|
|
|
27
27
|
## Environment
|
|
28
28
|
|
|
29
|
-
- toloka CLI 0.2.0-preview.
|
|
29
|
+
- toloka CLI 0.2.0-preview.3 (`toloka --version`), Bun 1.4.0, macOS 15
|
|
30
30
|
- Platform: https://platform.toloka.ai
|
|
31
31
|
|
|
32
32
|
## Redactions
|
package/src/authoring/index.ts
CHANGED
|
@@ -524,6 +524,20 @@ export function match(
|
|
|
524
524
|
settings: MatchAuthoringSettings,
|
|
525
525
|
options: AuthoringIdOptions = {},
|
|
526
526
|
): AuthoredMatchNode {
|
|
527
|
+
const conditionNames = new Set<string>();
|
|
528
|
+
for (const condition of settings.conditions) {
|
|
529
|
+
assert(condition.name !== "", "Condition must have a name");
|
|
530
|
+
assert(
|
|
531
|
+
condition.name.length >= 2 && condition.name.length <= 32,
|
|
532
|
+
"Condition name must be between 2 and 32 characters long",
|
|
533
|
+
);
|
|
534
|
+
assert(
|
|
535
|
+
/^[a-z]([a-z0-9_]*[a-z0-9])?$/u.test(condition.name),
|
|
536
|
+
"Condition name must start with a letter, contain only lowercase letters, numbers and underscores, and end with a letter or number",
|
|
537
|
+
);
|
|
538
|
+
assert(!conditionNames.has(condition.name), "Condition names must be unique");
|
|
539
|
+
conditionNames.add(condition.name);
|
|
540
|
+
}
|
|
527
541
|
const id = options.id ?? mintedId("match", name);
|
|
528
542
|
const { conditions, input, ...wireSettings } = settings;
|
|
529
543
|
return {
|
|
@@ -265,6 +265,16 @@ const mapParseErrorsToBlocks = (
|
|
|
265
265
|
return report;
|
|
266
266
|
};
|
|
267
267
|
|
|
268
|
+
const runtimeSyntaxReport = (source: string): TaskUiDiagnosticsReport => {
|
|
269
|
+
const jsxReport = mapParseErrorsToBlocks(source, parseSync("runtime.jsx", source).errors);
|
|
270
|
+
const tsxReport = mapParseErrorsToBlocks(source, parseSync("runtime.tsx", source).errors);
|
|
271
|
+
return {
|
|
272
|
+
...tsxReport,
|
|
273
|
+
preprocess: jsxReport.preprocess,
|
|
274
|
+
postprocess: jsxReport.postprocess,
|
|
275
|
+
};
|
|
276
|
+
};
|
|
277
|
+
|
|
268
278
|
export const createTaskUiTypechecker = ({
|
|
269
279
|
reactDeclaration,
|
|
270
280
|
elementsDeclaration,
|
|
@@ -311,11 +321,7 @@ export const createTaskUiTypechecker = ({
|
|
|
311
321
|
...syntacticDiagnostics,
|
|
312
322
|
...otherDiagnostics,
|
|
313
323
|
]);
|
|
314
|
-
const
|
|
315
|
-
const runtimeReport = mapParseErrorsToBlocks(
|
|
316
|
-
runtimeSource,
|
|
317
|
-
parseSync("runtime.jsx", runtimeSource).errors,
|
|
318
|
-
);
|
|
324
|
+
const runtimeReport = runtimeSyntaxReport(buildRuntimeSource(input));
|
|
319
325
|
return mergeRuntimeDiagnostics(typeScriptReport, syntacticReport, runtimeReport);
|
|
320
326
|
},
|
|
321
327
|
|
|
@@ -325,11 +331,7 @@ export const createTaskUiTypechecker = ({
|
|
|
325
331
|
typeScriptSource,
|
|
326
332
|
parseSync(FILE_NAME, typeScriptSource).errors,
|
|
327
333
|
);
|
|
328
|
-
const
|
|
329
|
-
const runtimeReport = mapParseErrorsToBlocks(
|
|
330
|
-
runtimeSource,
|
|
331
|
-
parseSync("runtime.jsx", runtimeSource).errors,
|
|
332
|
-
);
|
|
334
|
+
const runtimeReport = runtimeSyntaxReport(buildRuntimeSource(input));
|
|
333
335
|
return mergeRuntimeDiagnostics(typeScriptReport, typeScriptReport, runtimeReport);
|
|
334
336
|
},
|
|
335
337
|
});
|