@toloka-platform/sdk 0.11.1 → 0.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +4 -2
  2. package/dist/cli/toloka.js +167 -160
  3. package/dist/cli/toloka.js.jsc +0 -0
  4. package/dist/cli/toloka.js.jsc.sha256 +1 -1
  5. package/dist/cli/toloka.js.sha256 +1 -1
  6. package/docs/authoring.md +15 -5
  7. package/docs/cli.md +48 -16
  8. package/docs/getting-started.md +1 -1
  9. package/docs/limitations.md +2 -2
  10. package/docs/testing.md +5 -3
  11. package/docs/troubleshooting.md +1 -1
  12. package/examples/README.md +11 -0
  13. package/examples/app-crowdtesting/README.md +41 -0
  14. package/examples/app-crowdtesting/fields.ts +6 -0
  15. package/examples/app-crowdtesting/nodes/app-flow-test/node.ts +381 -0
  16. package/examples/app-crowdtesting/nodes/collection/node.ts +8 -0
  17. package/examples/app-crowdtesting/pipeline.browser.integration.spec.ts +77 -0
  18. package/examples/app-crowdtesting/pipeline.ts +17 -0
  19. package/examples/egocentric-video-collection/README.md +35 -0
  20. package/examples/egocentric-video-collection/fields.ts +24 -0
  21. package/examples/egocentric-video-collection/nodes/collection/node.ts +5 -0
  22. package/examples/egocentric-video-collection/nodes/record-activity/node.ts +760 -0
  23. package/examples/egocentric-video-collection/pipeline.browser.integration.spec.ts +65 -0
  24. package/examples/egocentric-video-collection/pipeline.ts +17 -0
  25. package/examples/product-moderation/README.md +49 -0
  26. package/examples/product-moderation/fields.ts +37 -0
  27. package/examples/product-moderation/nodes/collection/node.ts +16 -0
  28. package/examples/product-moderation/nodes/moderate-product/instruction.ts +77 -0
  29. package/examples/product-moderation/nodes/moderate-product/node.ts +368 -0
  30. package/examples/product-moderation/pipeline.browser.integration.spec.ts +48 -0
  31. package/examples/product-moderation/pipeline.ts +33 -0
  32. package/examples/video-pair-rating/fields.ts +16 -4
  33. package/examples/video-pair-rating/nodes/adjudicate/node.ts +78 -3
  34. package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +28 -3
  35. package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +28 -3
  36. package/examples/video-pair-rating/nodes/rate_pair/node.ts +65 -2
  37. package/openapi/gen/api.ts +22 -0
  38. package/openapi/gen/models/generationBudget.ts +8 -0
  39. package/openapi/gen/models/generationBudgetReasoningEffort.ts +2 -0
  40. package/openapi/gen/models/generationModel.ts +12 -0
  41. package/openapi/gen/models/generationModelDefaultReasoningEffort.ts +2 -0
  42. package/openapi/gen/models/generationModelId.ts +2 -0
  43. package/openapi/gen/models/index.ts +8 -0
  44. package/openapi/gen/models/invalidFieldSchema.ts +6 -0
  45. package/openapi/gen/models/listGenerationModels200.ts +7 -0
  46. package/openapi/gen/models/listGenerationModels200DefaultGenerationModelId.ts +2 -0
  47. package/openapi/gen/models/pipelineGraphSnapshot.ts +2 -0
  48. package/package.json +2 -2
  49. package/skills/catalog.json +16 -12
  50. package/skills/toloka-build/SKILL.md +44 -56
  51. package/skills/toloka-build/references/human-work.md +156 -0
  52. package/skills/toloka-build/references/local-setup.md +136 -0
  53. package/skills/toloka-build/references/node-checks.md +11 -0
  54. package/skills/toloka-build/references/node-semantics.md +22 -213
  55. package/skills/toloka-build/references/task-ui.md +61 -0
  56. package/skills/toloka-build/references/workflow-and-cli.md +29 -227
  57. package/skills/toloka-fine-tuning/SKILL.md +1 -1
  58. package/skills/toloka-get-started/SKILL.md +1 -1
  59. package/skills/toloka-gisting/SKILL.md +1 -1
  60. package/src/authoring/index.ts +26 -7
  61. package/src/checks/task-ui-typecheck.ts +3 -3
  62. package/src/cli/analytics.ts +1 -0
  63. package/src/cli/auth.ts +51 -18
  64. package/src/cli/check.ts +88 -17
  65. package/src/cli/commands/auth.ts +5 -3
  66. package/src/cli/commands/generation-model.ts +18 -0
  67. package/src/cli/commands/pipeline.ts +1 -1
  68. package/src/cli/commands/project.ts +7 -2
  69. package/src/cli/commands/source.ts +14 -3
  70. package/src/cli/commands/ui.ts +23 -0
  71. package/src/cli/config.ts +15 -9
  72. package/src/cli/context.ts +30 -12
  73. package/src/cli/credentials.ts +2 -2
  74. package/src/cli/environment.ts +5 -1
  75. package/src/cli/generation-models.ts +46 -0
  76. package/src/cli/program.ts +7 -0
  77. package/src/cli/project-context.ts +5 -2
  78. package/src/cli/project.ts +288 -138
  79. package/src/cli/trainings.ts +12 -0
  80. package/src/cli/ui.ts +63 -0
  81. package/src/client.ts +5 -0
  82. package/src/core-entity-types.json +32 -0
  83. package/src/graph-rules.json +11 -0
  84. package/src/importer.ts +46 -40
  85. package/src/index.ts +4 -0
  86. package/src/testing/task-ui-browser.ts +45 -2
  87. package/src/testing/task-ui.ts +2 -9
  88. package/src/transport.ts +28 -6
  89. package/src/validate.ts +232 -2
Binary file
@@ -1 +1 @@
1
- 6a0f8dbdd98f49a9c663bb3051960c4f168961bc4ebd8b428ae75fd5f07ba994 toloka.js.jsc
1
+ 6f99f48f6bd7c933cd5d2f6b2382d9c4149e6cfb4a39a32364d49994a16f436e toloka.js.jsc
@@ -1 +1 @@
1
- 20b4b75534617c8aed19bf129f8b78fe18d43a5a91221f1aa4bf2fbc0f792175 toloka.js
1
+ a038b95e14fe030c3d9b59182799dc89f8c76fd501fe42036831ea3d1daa27c5 toloka.js
package/docs/authoring.md CHANGED
@@ -99,6 +99,14 @@ Choose entity cardinality from the task, not from the component's JavaScript val
99
99
 
100
100
  Use `dynamic: true` only when one labeler produces multiple independent entities. Set `maxEntities` to a task-grounded upper bound rather than a generic default. A component may expose an array while still producing one logical entity; for a single upload or recording, keep a scalar field and static entity and select the completed first value with a path such as `$.video_file[0].downloadLink`.
101
101
 
102
+ Entity types and labels are checked locally, for both generation and quorum nodes, before anything is sent to the platform:
103
+
104
+ - `entityType` on every input and output entity must be one of the platform's core entity types. The list ships with the SDK in `src/core-entity-types.json`, generated from the server schema: `JUDGEMENT`, `EXPLANATION`, `PROMPT`, `USER_PROMPT`, `RESPONSE`, `RESPONSE_EVALUATION`, `RUBRIC`, `RUBRICS_GROUP`, `DIALOG`, `DIALOG_STEP`, `DIALOG_EVALUATION`, `SHORT_ANSWER`, `FEEDBACK`, `CONTEXT`, `IMAGE`, `ANNOTATION`, `SINGLE_LABEL`, `MULTI_LABEL`, `VIDEO`, `DOCUMENT`, `GOLDEN_SET`, `LOGS`, `SOLUTION`, `CODE`, `METADATA`, `AUDIO`, `AGENT_TRAJECTORY`, `MCP_SERVER`, `CONTAINER`, `CUSTOM`.
105
+ - An output entity of type `SINGLE_LABEL` or `MULTI_LABEL` needs `allowedLabels` with at least two entries.
106
+ - Every label `value` must be snake_case (`first_video`, not `firstVideo` or `First Video`).
107
+
108
+ A violation is thrown as `PipelineGraphValidationError`, with one issue per offending path, as soon as the pipeline source is loaded — so `toloka check`, `toloka plan`, `toloka apply` and `toloka validate` all fail on it locally before any platform request.
109
+
102
110
  The following task is legitimately dynamic because the labeler can draw one independent box for every visible face:
103
111
 
104
112
  ```ts
@@ -217,7 +225,7 @@ A quorum node **requires** an `onboarding` block: the Training course is part of
217
225
 
218
226
  ### match
219
227
 
220
- Predicate routing. Conditions are evaluated in declaration order.
228
+ Predicate routing. Conditions are evaluated in declaration order. A condition's `logic` defaults to `'and'`, so every predicate must match; set `logic: 'or'` when any one predicate should match. `match()` writes the default into the graph, as the platform stores it, so `toloka plan` does not report the condition as changed after an apply.
221
229
 
222
230
  ```ts
223
231
  import { match } from "@toloka-platform/sdk";
@@ -276,7 +284,9 @@ export default definePipeline({
276
284
  });
277
285
  ```
278
286
 
279
- `dataset` names the source dataset. `plan` creates it if it is absent and records the resolved id in `toloka.json`.
287
+ `dataset` names the source dataset. `apply` creates it if it is absent, binds it to the pipeline, and records the resolved id in `toloka.json`.
288
+
289
+ Leave `dataset` out for a pipeline that has no dataset yet, such as one created from a template in the platform UI. `plan` and `apply` then edit its graph without binding a dataset, and `toloka import` writes such a pipeline the same way. The pipeline cannot run until it has a dataset: add `dataset` to its source later, or bind one in the platform UI and import again. Apply never removes a bound dataset, so it refuses to apply a source that drops `dataset` from a pipeline that has one.
280
290
 
281
291
  Each `pipelines/<name>/pipeline.ts` is one pipeline. A project can hold several; each names its own dataset, and a dataset can be shared.
282
292
 
@@ -288,7 +298,7 @@ Constructors mint deterministic ids from the node name, so authored source needs
288
298
 
289
299
  `toloka import` emits explicit `{ id }` options for nodes and fields that already exist on the platform. **Preserve them.** They are how reconciliation recognizes the same resource instead of replacing it.
290
300
 
291
- Never hand-write `fp_…`, `ds.…`, `pl.…`, or `pp.…` values into settings, and never author a `layout` block: layout is not authored, existing layout is preserved, and the server supplies defaults.
301
+ Never hand-write `fp_…`, `ds.…`, `pl.…`, or `pp.…` values into settings, and never author a `layout` block: layout is not authored, existing nodes keep their canvas position, and the server places new nodes one column after their rightmost input and moves End to the right when they reach it.
292
302
 
293
303
  ## Validate
294
304
 
@@ -297,9 +307,9 @@ toloka check
297
307
  toloka validate
298
308
  ```
299
309
 
300
- `definePipeline` and the typed ports add local declaration and connection invariants. `toloka validate` posts each bound pipeline to `POST /pipelines/:pipelineId/graph/validate` and reports platform spec errors, warnings, and the latest matching check jobs. It does not write the graph.
310
+ `definePipeline` and the typed ports add local declaration and connection invariants. `toloka check` also reports, as warnings, the platform rules that would reject a node connected to the root: node names, unique quality elements, quality-criterion thresholds, the minimum task and Human QA review times, a positive price per task, a media upload component on a human `data_collection` node, and the limits of an enabled exam. `toloka apply` refuses to write a new or changed pipeline that breaks them, before any platform write; an unchanged pipeline, such as a draft saved in the platform UI, does not block apply of the others. `toloka validate` posts each bound pipeline to `POST /pipelines/:pipelineId/graph/validate` and reports platform spec errors, warnings, and the latest matching check jobs. It does not write the graph.
301
311
 
302
- A pipeline that is not on the platform yet is skipped until `toloka apply` creates it. The published JSON contract is still available as `validate(graph)` for importer and fixture tests; that helper makes no network call.
312
+ A pipeline that is not on the platform yet is skipped until `toloka apply` creates it. The published JSON contract is still available as `validate(graph)` for importer and fixture tests; that helper also runs the local entity checks described under [generation](#generation) and makes no network call.
303
313
 
304
314
  A successful validate is necessary, not sufficient for launch. Gates outside the graph contract — an underpriced audience, a Training course that is enabled but unbuilt, a task UI that was never validated — can still block deployment. See [limitations](./limitations.md).
305
315
 
package/docs/cli.md CHANGED
@@ -56,14 +56,15 @@ If both installations exist, put the desired directory first on `PATH`, run `has
56
56
 
57
57
  ## Configuration
58
58
 
59
- Credentials and the platform base URL resolve from the same precedence order, highest first.
59
+ Credentials and the platform base URL resolve independently. Credentials use this precedence, highest first.
60
60
 
61
61
  | Precedence | Credential source |
62
62
  | ----------- | ------------------------------------------------ |
63
63
  | 1 (highest) | `--api-key` flag |
64
- | 2 | `TOLOKA_API_KEY` environment variable |
65
- | 3 | API key stored by `toloka login --api-key-stdin` |
66
- | 4 (lowest) | browser session stored by `toloka login` |
64
+ | 2 | `TOLOKA_SCOPED_TOKEN` environment variable |
65
+ | 3 | `TOLOKA_API_KEY` environment variable |
66
+ | 4 | API key stored by `toloka login --api-key-stdin` |
67
+ | 5 (lowest) | browser session stored by `toloka login` |
67
68
 
68
69
  | Precedence | Base URL source |
69
70
  | ----------- | --------------------------------------- |
@@ -81,6 +82,8 @@ Credentials and the platform base URL resolve from the same precedence order, hi
81
82
 
82
83
  Earlier releases named these variables `POTATO_API_KEY`, `POTATO_BASE_URL`, and `POTATO_PROJECT_ID`. The CLI still reads each legacy name when its `TOLOKA_*` counterpart is unset, prefers the `TOLOKA_*` value when both are set, and prints `POTATO_API_KEY is deprecated and will stop being read; set TOLOKA_API_KEY instead.` (and the same line for the other two) on stderr once per command. The aliases remain supported for compatibility; rename them to silence the warning and use the current names in new automation.
83
84
 
85
+ `TOLOKA_SCOPED_TOKEN` always sends its value in `X-Scoped-Token`, so it supports both issued `pst_…` tokens and secret placeholders such as `dtn_secret_…`. The `--api-key` and `TOLOKA_API_KEY` slots continue to recognize the `pst_` prefix for backward compatibility. Use the explicit scoped-token variable for new agent and CI configuration.
86
+
84
87
  `toloka login` verifies access against the platform before saving anything. A browser login is saved as a session file under `$TOLOKA_CLI_CREDENTIALS_DIR`, which defaults to `~/.toloka/cli/credentials/<auth-host>/<realm>/<client>.json`. When `TOLOKA_CREDENTIALS_FILE` points at an explicit credentials file, sessions go to a `browser-sessions` directory next to it instead, so that profile stays isolated. The platform choice, a stored API key, and local project aliases live in `$XDG_CONFIG_HOME/toloka/credentials.json`, which defaults to `~/.config/toloka/credentials.json`. Both are written with file mode `0600` in directories with mode `0700`. The legacy `~/.toloka/credentials.json` is still read when the new path has no file. `toloka logout` ends the browser session and deletes both files; it does not revoke an API key.
85
88
 
86
89
  Ordinary usage events (`sdk_cli_command_completed` / `sdk_cli_command_failed`) go to Toloka's PostHog project so we can see which commands run and which error classes they hit. Every failed command with an exception also emits a sanitized PostHog Error Tracking `$exception`; usage errors such as unknown commands or invalid flags do not. Command events contain only an allowlisted command classifier, duration, exit code, outcome or error code, optional HTTP status and server request ID, SDK and Bun versions, actor, and channel. Error Tracking receives the same bounded metadata plus a synthetic exception; raw arguments, flag values, environment variables, API keys, secret names, base URLs and hosts, local paths, project ids, error messages, stack traces, response bodies, and source context are not sent. GeoIP enrichment is disabled. Distinct id starts as a random UUID stored next to the credentials file. After a successful authenticated command, the CLI calls `GET /me` and aliases that UUID onto the platform user id without setting person properties. Set `TOLOKA_TELEMETRY=0` or `DO_NOT_TRACK=1` to disable capture. CI environments are disabled unless `TOLOKA_TELEMETRY=1`. The explicitly approved full-text `report` event is documented in [report](#report).
@@ -175,6 +178,16 @@ toloka audience specialization list [--json]
175
178
 
176
179
  Lists every expert specialization (platform qualification) with `ID`, `NAME`, and `EXPERTS` columns, sorted by name. Copy `ID` into `qualificationId` and `NAME` into `value` of `audience.specializations.values[]`; never invent a `qualificationId`. `EXPERTS` is the number of ID-verified experts currently eligible for tasks who hold the specialization; it can be `0` while the id is still valid. Pass `--json` for the full array. Empty output is `No specializations available.`
177
180
 
181
+ ## generation-model
182
+
183
+ Inspect the global, read-only catalog of models a synthetic `generation` node can use.
184
+
185
+ ```sh
186
+ toloka generation-model list [--json]
187
+ ```
188
+
189
+ Prints one row per model and reasoning effort with `ID`, `NAME`, `REASONING EFFORT`, `TYPICAL`, `MIN`, and `MAX` columns, then the default and recommended models. Copy `ID` into `generationModelId`. A `-` in `REASONING EFFORT` means the model does not support `reasoningEffort`; leave it unset there. `MIN` and `MAX` bound the `maxCostPerAttempt` apply accepts; a budget below `TYPICAL` is accepted with a warning because generation can stop before the result is complete. `--json` adds each model's description. Empty output is `No generation models available.`
190
+
178
191
  ## login
179
192
 
180
193
  Signs in through the browser, or stores an API key, after verifying access.
@@ -190,7 +203,7 @@ By default the command asks whether to sign in through the browser or with an AP
190
203
 
191
204
  Browser login is available for `https://platform.toloka.ai` and the `*.platform.toloka-test.ai` environments. Without a terminal the command opens the chooser page directly and fails fast when no browser can be opened; automation uses an API key instead.
192
205
 
193
- The API key choice does not expire: the key works until it is deleted in the Business Console, which suits scripts, CI, and coding agents. Choosing it asks for the key with masked input; `--api-key-stdin` reads it from standard input instead, so it never appears in shell history or the process list; `--api-key <key>` and `TOLOKA_API_KEY` also work but expose the key to both. Platforms without a known login service go straight to the API key prompt. The command verifies the key and prints `Authenticated against <base-url> with an API key`, then the number of projects visible, where the key is stored, and a reminder that the key does not expire until you delete it in the Business Console.
206
+ The API key choice does not expire: the key works until it is deleted in the Business Console, which suits scripts and CI. Choosing it asks for the key with masked input; `--api-key-stdin` reads it from standard input instead, so it never appears in shell history or the process list; `--api-key <key>` and `TOLOKA_API_KEY` also work but expose the key to both. For an already-issued project-scoped token, set `TOLOKA_SCOPED_TOKEN` for the command instead of storing it with `login`. Platforms without a known login service go straight to the API key prompt. The command verifies the key and prints `Authenticated against <base-url> with an API key`, then the number of projects visible, where the key is stored, and a reminder that the key does not expire until you delete it in the Business Console.
194
207
 
195
208
  ## logout
196
209
 
@@ -210,7 +223,7 @@ Shows which platform and login the CLI would use. Alias: `whoami`.
210
223
  toloka auth-status [--show-token]
211
224
  ```
212
225
 
213
- On success it prints the platform URL with its source, then either the masked API key or a `Login:` line for the browser session and a `Session:` line with the current refresh window, then the selected project, `Reachable: N project(s) visible.`, `Account:` with the email the platform reports, `Internal: true` or `Internal: false` for the account, and a `Teams:` line that marks each team `(internal)` and `(read-only)` where that applies. The project line is printed even when the probe fails. `--show-token` prints the full API key or the browser session's live access token instead of the mask; pass it only when you need the raw value on this terminal. It exits `1` when no login is in effect, when the browser session has expired, when the platform rejects the login (`Rejected: …`, followed by the advice to run `toloka login`), when the platform does not answer (`Not reachable: …`; the login was not evaluated), or when the platform fails (`Platform error: …`).
226
+ On success it prints the platform URL with its source, then either the masked API key, the masked scoped token with `from TOLOKA_SCOPED_TOKEN`, or a `Login:` line for the browser session and a `Session:` line with the current refresh window. It then prints the selected project, `Reachable: N project(s) visible.`, `Account:` with the email the platform reports, `Internal: true` or `Internal: false` for the account, and a `Teams:` line that marks each team `(internal)` and `(read-only)` where that applies. The project line is printed even when the probe fails. `--show-token` prints the full active credential instead of the mask; pass it only when you need the raw value on this terminal. It exits `1` when no login is in effect, when the browser session has expired, when the platform rejects the credential, or when the platform cannot be reached.
214
227
 
215
228
  ## config
216
229
 
@@ -247,27 +260,33 @@ Generates TypeScript source from a live project.
247
260
  toloka import [projectId] [-f]
248
261
  ```
249
262
 
250
- When `projectId` is omitted, the selected project is imported. If `pipelines/` already exists, the command exits with `Folder already exists: <path>. Pass --force to replace it.` Pass `-f` or `--force` to replace that directory. On success it prints `Imported project <projectId> with N pipelines and M datasets into <root>.`
263
+ When `projectId` is omitted, the selected project is imported. If `pipelines/` already exists, the command exits with `Folder already exists: <path>. Pass --force to replace it.` Pass `-f` or `--force` to replace that directory. On success it prints `Imported project <projectId> with N pipelines and M datasets into <root>.` A pipeline without a dataset is written without `dataset` in its `definePipeline`.
264
+
265
+ The generated nodes keep every stored check case, including cases of the retired `ui` and `llmqa` types. A check type with no cases is left out, and a node whose check types are all empty gets no `checks` at all. `plan` and `apply` treat a check type that source leaves out as unmanaged: they neither report nor remove the cases the platform has for it, so set a type to an empty array to remove its cases.
266
+
267
+ A field whose JSON Schema is stored on the platform as invalid JSON or as an unsupported schema, for example after an unfinished edit in the UI, is imported as `field("name", invalidJsonSchema({ field, reason, text }))` with the stored text. Loading the source then fails with `Field "<name>" has an unreadable JSON Schema on the platform.` until you replace the call with a valid JSON Schema object; the next `apply` writes that schema and repairs the field.
251
268
 
252
269
  ## plan
253
270
 
254
271
  Shows what `apply` would change.
255
272
 
256
273
  ```sh
257
- toloka plan
274
+ toloka plan [--json]
258
275
  ```
259
276
 
260
- On success it prints `No changes.` when the platform matches source, otherwise `Plan: N changes` followed by one line per change. The plan also lists missing project secrets and dataset field schema conflicts so you can resolve them before applying.
277
+ On success it prints `No changes.` when the platform matches source, otherwise `Plan: N changes` followed by one line per change. The plan also lists missing project secrets and dataset field schema conflicts so you can resolve them before applying. `--json` prints `{ "changes": [...], "missingSecrets": [...] }` instead, where each change has `kind`, `action`, `label`, and, when they apply, `pipeline`, `dataset`, and `id`.
261
278
 
262
279
  ## validate
263
280
 
264
281
  Runs platform spec validation for each bound pipeline and reports the latest matching check jobs. Does not write the graph.
265
282
 
266
283
  ```sh
267
- toloka validate
284
+ toloka validate [--wait]
268
285
  ```
269
286
 
270
- On success it prints one line per pipeline: `valid`, `invalid` with issue details, or `skipped` when the pipeline is not on the platform yet. Exits `1` when any submitted graph is invalid.
287
+ It prints one line per pipeline with its status, then its issues: `valid`; `invalid`; `checks-pending` when only check state is open, such as a stored test case or a lint that has not run on this graph; `checks-running` when `--wait` gave up on running checks; or `skipped` when the pipeline or its dataset is not on the platform yet. A `note:` line explains what to do next when `--wait` could not finish. Exits `1` when any submitted graph is invalid or its checks were still running at the `--wait` deadline.
288
+
289
+ The platform lints a pipeline a few seconds after every apply. While a lint or a check is running, or a lint is about to start after an apply, validate lists it as `running running_node_check`; a lint that has not run on this graph and is no longer expected to start is `pending not_ran_node_check`, and the user runs it from the node's Checks tab. `--wait` validates again every 5 seconds until nothing is running, for up to 5 minutes, then prints the final report; a pipeline whose source differs from the applied graph is not waited on, because its lints start only after `toloka apply`. Right after an apply, plain `toloka validate` can print `valid` and exit `0` while lints are still running, so run `toloka validate --wait` before treating an applied pipeline as valid.
271
290
 
272
291
  ## check
273
292
 
@@ -277,7 +296,7 @@ Runs the complete local gate without platform credentials.
277
296
  toloka check [pipeline] [--node <name-or-id>] [--json] [--watch]
278
297
  ```
279
298
 
280
- The command loads authored source, runs the project's TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and compares passing typed test evidence with every code node, human-work node, and pipeline. Statuses distinguish `missing`, `failed`, `partial`, and `infrastructure_error`. Exit code `0` means all required checks passed, `1` means source, types, tests, or required coverage failed, and `2` means a required checker could not run.
299
+ The command loads authored source, runs the project's TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and compares passing typed test evidence with every code node, human-work node, and pipeline. Statuses distinguish `missing`, `failed`, `partial`, `warning`, and `infrastructure_error`. `partial` marks a Task UI whose upload or recording fields the headless browser could not exercise; it is reported but does not fail the gate. `warning` marks a `platform-rules` result: nodes connected to the root that break a rule the platform enforces on write. It does not fail the gate, but `toloka apply` refuses to write that pipeline until it is fixed. Exit code `0` means all required checks passed, `1` means source, types, tests, or required coverage failed, and `2` means a required checker could not run.
281
300
 
282
301
  Use `test.code`, `test.ui`, and `test.pipeline` from `@toloka-platform/sdk/testing` to produce required evidence. Plain Vitest tests still run and can fail the command, but do not satisfy graph coverage. A focused check accepts a pipeline plus `--node`; `--watch` serializes reruns after relevant file changes.
283
302
 
@@ -285,7 +304,7 @@ With `--watch`, the command prints one complete report after each serialized run
285
304
 
286
305
  ## apply
287
306
 
288
- Runs the complete Local Check Gate, then makes the platform match source.
307
+ Runs the complete local gate, then makes the platform match source. It takes no options and never asks for confirmation.
289
308
 
290
309
  ```sh
291
310
  toloka apply
@@ -415,6 +434,16 @@ In a terminal the command asks `Send this report to Toloka? [y/N]`; outside a te
415
434
 
416
435
  One `sdk_cli_issue_report_submitted` event carries `report_id`, the complete `report_text`, its UTF-8 `report_length_bytes`, `authored_by`, and `platform`, plus the standard pseudonymous CLI telemetry context. The command never reads or sends an API key. It flushes PostHog before printing `Submitted issue report: <id>`; a delivery failure exits non-zero and prints no success ID. When CLI telemetry is disabled, issue reporting is unavailable rather than bypassing that preference.
417
436
 
437
+ ## ui
438
+
439
+ Inspect the Task UI components the launcher renders. The command opens the launcher in a headless Chromium (Playwright from the project's `node_modules`), so it needs the same local setup as `toloka check` and no credentials.
440
+
441
+ ```sh
442
+ toloka ui components [names...] [--json]
443
+ ```
444
+
445
+ Without names it prints one line per public component (`name: description When to use: …`). With names it prints each component's description, when to use it, capabilities, limitations, best practices, the `propsType` contract, the value shape, and the documented JSX example with its output. An unknown name fails with `Unknown Task UI components: <name>`. `--json` prints the same contracts keyed by component name. The output always reflects the currently deployed launcher, so run it instead of guessing component names or props from memory.
446
+
418
447
  ## skill
419
448
 
420
449
  Discover bundled agent skills and install them into the project. The Skill Catalog ships inside the SDK package, so these actions are local and deterministic: they need no credentials, make no network requests, and change with the package version only.
@@ -451,7 +480,7 @@ An unknown command exits 1 before anything else runs, even when `--help` is pres
451
480
 
452
481
  ## The reconcile loop
453
482
 
454
- Treat the TypeScript source as authoritative. The canonical sequence is: run the complete local gate, validate against the platform, plan, apply, then plan again to confirm convergence.
483
+ Treat the TypeScript source as authoritative. The canonical sequence is: run the complete local gate, validate against the platform, plan, apply, validate again once the platform lints finish, then plan again to confirm convergence.
455
484
 
456
485
  ```sh
457
486
  # 1. Run source, type, and typed test checks
@@ -466,11 +495,14 @@ toloka plan
466
495
  # 4. Apply only when the plan matches the intent
467
496
  toloka apply
468
497
 
469
- # 5. Confirm the platform matches source
498
+ # 5. Wait for the platform lints that start after apply
499
+ toloka validate --wait
500
+
501
+ # 6. Confirm the platform matches source
470
502
  toloka plan # expect: No changes.
471
503
  ```
472
504
 
473
- `toloka apply` reruns the complete local gate before resolving credentials. `toloka validate` posts each authored graph to `POST /pipelines/:pipelineId/graph/validate`; it reports platform spec validation and the latest matching Remote Node Check jobs without writing the graph. Resolve every unexpected plan change, missing secret, and immutable-field conflict before applying.
505
+ `toloka apply` reruns the complete local gate before resolving credentials. `toloka validate` posts each authored graph to `POST /pipelines/:pipelineId/graph/validate`; it reports platform spec validation and the latest matching check runs without writing the graph. Resolve every unexpected plan change, missing secret, and immutable-field conflict before applying.
474
506
 
475
507
  ## Further reading
476
508
 
@@ -37,7 +37,7 @@ The session renews itself for up to 14 days; after that, run `toloka login` agai
37
37
  toloka whoami
38
38
  ```
39
39
 
40
- CI does not need `login`. Set `TOLOKA_API_KEY` and the stored login is ignored; an agent without a terminal can pipe a key to `toloka login --api-key-stdin`. See the precedence table in the [CLI reference](./cli.md).
40
+ CI does not need `login`. Set `TOLOKA_API_KEY` for a long-lived API key or `TOLOKA_SCOPED_TOKEN` for an issued project-scoped token, and the stored login is ignored; an agent without a terminal can pipe a long-lived key to `toloka login --api-key-stdin`. See the precedence table in the [CLI reference](./cli.md).
41
41
 
42
42
  ## Start with an agent
43
43
 
@@ -19,7 +19,7 @@ The standalone CLI supports macOS and glibc Linux on ARM64 and x64. Native Windo
19
19
  | Training course content, when Training is enabled | platform validation | nothing |
20
20
  | labeler UI validation receipt | platform validation | nothing |
21
21
 
22
- The typed client can list, start, and poll Remote Node Checks. `toloka check` does not start them: it validates the authored source and its local tests. A `not_ran_node_check` result from `toloka validate` describes remote state rather than missing local evidence. Its severity is check-specific: lint checks report a blocking error, while other checks can report a warning.
22
+ The typed client can list, start, and poll Remote Node Checks. `toloka check` does not start them: it validates the authored source and its local tests. A `not_ran_node_check` or `running_node_check` warning from `toloka validate` describes remote state rather than missing local evidence.
23
23
 
24
24
  Python code nodes have executable behavior checks but no useful static Python typecheck yet. Their generated `Inputs` and `Outputs` aliases are currently `dict[str, Any]`; adding Pyright or mypy before those aliases become field-derived `TypedDict` contracts would provide little protection.
25
25
 
@@ -27,7 +27,7 @@ The standalone CLI's runtime does not provide project test dependencies. Executa
27
27
 
28
28
  ## Layout is not authored
29
29
 
30
- Node canvas positions are not part of authored source. Existing layout is preserved across applies and the server supplies defaults for new nodes. You cannot lay out a graph from code.
30
+ Node canvas positions are not part of authored source. Existing nodes keep their canvas position across applies. The server places a new node one column to the right of its rightmost input, below any node already in that spot, and moves End to the right when new nodes reach it. You cannot lay out a graph from code.
31
31
 
32
32
  ## One checkout targets one project
33
33
 
package/docs/testing.md CHANGED
@@ -17,7 +17,7 @@ bun add -d vitest playwright
17
17
  bunx playwright install chromium
18
18
  ```
19
19
 
20
- Executable code-node tests also require `python3` on the test machine, including any third-party Python packages imported by the authored program. Browser-backed Task UI tests use the separately downloaded Playwright Chromium binary and require network access to the hosted launcher. Training-only CLI workflows require none of these project test tools or a separate JavaScript runtime. See [testing examples](./testing-examples.md) for parameterization, validation failures, hardware coverage, and cleanup patterns.
20
+ Executable code-node tests also require `python3` on the test machine. The platform runs code nodes on Python 3.12 with the standard library only, so a program that imports a package installed only on the test machine passes locally and fails on the platform. Browser-backed Task UI tests use the separately downloaded Playwright Chromium binary and require network access to the hosted launcher. Training-only CLI workflows require none of these project test tools or a separate JavaScript runtime. See [testing examples](./testing-examples.md) for parameterization, validation failures, hardware coverage, and cleanup patterns.
21
21
 
22
22
  ## Test a code node
23
23
 
@@ -83,7 +83,7 @@ Use an ordinary TypeScript loop to register several `test.code` or `test.ui` sce
83
83
 
84
84
  The low-level `ui` fixture remains available inside ordinary Vitest tests for component discovery and exploratory work, but plain tests do not satisfy required node coverage. Use typed wrappers for committed gate evidence. Importing `@toloka-platform/sdk/testing` from a plain Bun script executes Vitest fixture setup and fails.
85
85
 
86
- - `ui.componentDocumentation(names?)` returns the current launcher's supported component contracts. Call it before choosing component names or props.
86
+ - `ui.componentDocumentation(names?)` returns the current launcher's supported component contracts. `ui` is a fixture argument of `test` (`test("…", async ({ ui }) => …)`), not an export. Outside a test, `toloka ui components [names...]` prints the same contracts.
87
87
  - `ui.render(pipeline, node, options)` creates a fresh browser context and returns a Task UI session for exactly that authored node. `test.ui` supplies the equivalent bound `render(options?)` helper.
88
88
 
89
89
  The returned Task UI session supports:
@@ -110,6 +110,8 @@ Valid JavaScript: `items.map((item) => item.value)`
110
110
 
111
111
  Invalid runtime JavaScript: `items.map((item: unknown) => item.value)`
112
112
 
113
+ Only `data` is in scope in `preprocess` and `postprocess`. `CONST` exists only in `ui.code`, so `toloka check` reports `Cannot find name 'CONST'` when either function reads it: write the value inline in the function, or read it from `ui.code` instead.
114
+
113
115
  `preprocess` can run again after form state changes. Use it to shape input data; if it prefills a bound field, return that key only while it is absent so later edits are not overwritten. A `Field`-bound top-level key already enters the submission. `postprocess` must return only new top-level keys: bind a component to a draft key when postprocess must derive a different entity key.
114
116
 
115
117
  The fixture loads the hosted TBX launcher in a fresh local Chromium context, so tests require network access but do not use Azure Playwright or Potato application services. The context has no platform cookies or credentials. Use synthetic or redacted fixtures because the hosted launcher executes with access to the supplied test data.
@@ -120,4 +122,4 @@ File pickers, video upload/recording, and other hardware-backed fields cannot be
120
122
 
121
123
  ## Final gate
122
124
 
123
- `toloka check` loads every authored pipeline, runs the project TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and requires passing `test.code`, `test.ui`, and `test.pipeline` evidence. `missing`, `failed`, `partial`, and `infrastructure_error` are distinct blocking outcomes. `toloka apply` reruns this complete gate before resolving credentials or writing platform state. Platform validation remains separate: a `not_ran_node_check` describes Remote Node Check state rather than missing local evidence, but lint checks report it as a blocking remote error while other checks can report it as a warning.
125
+ `toloka check` loads every authored pipeline, runs the project TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and requires passing `test.code`, `test.ui`, and `test.pipeline` evidence. `missing`, `failed`, and `infrastructure_error` are distinct blocking outcomes. `partial` is reported for a Task UI with hardware-backed fields and does not block: verify those fields on a real device before launching a run. `toloka apply` reruns this complete gate before resolving credentials or writing platform state. Platform validation remains separate: a `not_ran_node_check` warning describes Remote Node Check state rather than missing local evidence.
@@ -96,7 +96,7 @@ toloka pipeline results <runId> --failed
96
96
  Not logged in. Run toloka login once to sign in through your browser on this machine:
97
97
  ```
98
98
 
99
- Run `toloka login`. Where a browser is not available, pipe an API key to `toloka login --api-key-stdin` or set `TOLOKA_API_KEY`. `toloka whoami` reports which platform and which login source is in effect, and exits non-zero when the platform is unreachable.
99
+ Run `toloka login`. Where a browser is not available, pipe a long-lived API key to `toloka login --api-key-stdin` or set `TOLOKA_API_KEY`; set `TOLOKA_SCOPED_TOKEN` when using an issued project-scoped token. `toloka whoami` reports which platform and which credential source is in effect, and exits non-zero when the platform is unreachable.
100
100
 
101
101
  ## The browser login expired
102
102
 
@@ -0,0 +1,11 @@
1
+ # Examples
2
+
3
+ Complete pipeline projects to borrow structure from. Each directory has a README and a `pipeline.browser.integration.spec.ts` with its typed tests. Borrow the patterns, not the task-specific choices: design the graph, entities, criteria, and Task UI for the task at hand.
4
+
5
+ | example | read it for |
6
+ | ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
7
+ | `product-moderation` | **start here for a show-an-item, collect-a-judgment task**: one human `generation` node that shows an image and text and collects a label, a subcategory, and a rationale; label lists shared by fields and UI; variants for synthetic generation and several experts |
8
+ | `face-image-collection` | a human image-annotation node followed by a code node; a structured array output |
9
+ | `video-pair-rating` | side-by-side rating with a `quorum`, deterministic code cascades over rater arrays, `match` routing with `default` branches, and human adjudication |
10
+ | `egocentric-video-collection` | data collection with a recording field, a category label driven by one constant list, LLM QA with one criterion per concern, Training with quizzes, stored checks |
11
+ | `app-crowdtesting` | data collection with an upload field plus structured metadata answers, a step-by-step screen with long text in `ui.constants`, contributor diversity via the per-expert cap |
@@ -0,0 +1,41 @@
1
+ # App crowdtesting
2
+
3
+ A crowdtesting data-collection pipeline: each tester installs a pre-release Android build on their own phone, screen-records a full session through the app's key flows, uploads the recording, and reports their device model and Android version. An LLM reviews every submission against quality criteria before it is accepted.
4
+
5
+ The task content is built around a fictional app, Acme Fitness. To adopt it, replace the app everywhere it appears: the install link, node description and public name, entity descriptions, every quality criterion, the instruction, the Task UI constants, and the Training steps and quiz.
6
+
7
+ ## Shape
8
+
9
+ ```
10
+ root.started (item_id)
11
+ └─> app_flow_test generation a tester installs, records, uploads, and describes the device
12
+ └─> root.completed (app_test_video, device_model, os_version)
13
+ ```
14
+
15
+ Two nodes: one root and one `data_collection` `generation`. Each dataset row is one recording slot; `item_id` carries no task content, so the number of rows is the number of recordings.
16
+
17
+ ## What to read first
18
+
19
+ | file | why |
20
+ | -------------------------------------- | ------------------------------------------------------------------------- |
21
+ | `nodes/app-flow-test/node.ts` | the whole task: entities, criteria, instruction, Task UI, and Training |
22
+ | `fields.ts` | a scalar video field fed from a single upload, plus two free-text answers |
23
+ | `pipeline.browser.integration.spec.ts` | how to drive an upload component in a headless UI scenario |
24
+
25
+ ## Patterns this example demonstrates
26
+
27
+ **One upload is one entity.** `Field.VideoUpload` always produces an array, but a tester submits one recording. The output field is a scalar URI, the entity is static, and its `jsonPath` selects `$.app_test_video[0].sasLink`, matching the component's `linkType={'sasLink'}` (the contract's recommended link type; `downloadLink` may be inaccessible to end users). Duration bounds are set on both the entity and the component, and they agree.
28
+
29
+ **Structured metadata next to the upload.** Device model and OS version are separate `SHORT_ANSWER` outputs related to the recording, each with a `required` rule and its own `max` length rule, and each with a criterion that rejects placeholders.
30
+
31
+ **A step-by-step screen.** The Task UI stacks install, record, and upload as three cards in the order the tester works. Long markdown and the install link live in `ui.constants` and are read as `CONST.*`, so `ui.code` needs no nested template literals and the install link is declared once for both the instruction and the screen.
32
+
33
+ **LLM QA judges what is visible.** Criteria count attempted flow runs and required screens by on-screen text, never by which control was tapped, and a captured crash counts as a completed run rather than a failure.
34
+
35
+ **Training that matches the task.** Three steps, ending in a quiz on the rules that most often fail a recording: when to start recording, how many runs each flow needs, and what to do when the app crashes.
36
+
37
+ **Contributor diversity.** An unverified English-speaking crowd with a per-expert cap of 1, so every recording comes from a different person and device.
38
+
39
+ ## Check locally
40
+
41
+ Upload and recording controls are hardware-backed, and headless Chromium cannot pick, capture, or upload a file. The UI scenario renders with `allowPartialCoverage: true`, sets the upload value from the component's documented example output, and asserts that coverage is incomplete. It verifies browser validation and the output mapping, not the upload itself, so `toloka check` reports this node as `partial`.
@@ -0,0 +1,6 @@
1
+ import { field } from "@toloka-platform/sdk";
2
+
3
+ export const itemId = field("item_id", { type: "number" });
4
+ export const appTestVideo = field("app_test_video", { type: "string", format: "uri" });
5
+ export const deviceModel = field("device_model", { type: "string" });
6
+ export const osVersion = field("os_version", { type: "string" });