@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +7 -11
  2. package/dist/cli/toloka.js +208 -123
  3. package/dist/cli/toloka.js.jsc +0 -0
  4. package/dist/cli/toloka.js.jsc.sha256 +1 -1
  5. package/dist/cli/toloka.js.sha256 +1 -1
  6. package/docs/authoring.md +17 -4
  7. package/docs/cli-v2.md +3 -3
  8. package/docs/cli.md +29 -16
  9. package/docs/getting-started.md +7 -12
  10. package/docs/limitations.md +6 -6
  11. package/docs/testing-examples.md +248 -0
  12. package/docs/testing.md +115 -0
  13. package/examples/face-image-collection/README.md +7 -1
  14. package/examples/face-image-collection/fields.ts +20 -8
  15. package/examples/face-image-collection/nodes/annotate-faces/node.ts +1 -9
  16. package/examples/face-image-collection/nodes/order-faces/node.ts +1 -1
  17. package/examples/face-image-collection/pipeline.browser.integration.spec.ts +78 -0
  18. package/examples/video-pair-rating/README.md +14 -6
  19. package/examples/video-pair-rating/nodes/adjudicate/node.ts +1 -3
  20. package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +1 -3
  21. package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +1 -1
  22. package/examples/video-pair-rating/nodes/rate_pair/node.ts +1 -1
  23. package/examples/video-pair-rating/pipeline.browser.integration.spec.ts +267 -0
  24. package/openapi/gen/api.ts +48 -11
  25. package/openapi/gen/models/createDatasetExportBody.ts +3 -3
  26. package/openapi/gen/models/createDatasetExportBodyOneOf.ts +6 -0
  27. package/openapi/gen/models/createDatasetExportBodyOneOfFormat.ts +2 -0
  28. package/openapi/gen/models/createDatasetExportBodyOneOfThree.ts +10 -0
  29. package/openapi/gen/models/createDatasetExportBodyOneOfThreeContract.ts +2 -0
  30. package/openapi/gen/models/createDatasetExportBodyOneOfThreeFormat.ts +2 -0
  31. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilter.ts +9 -0
  32. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem.ts +6 -0
  33. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  34. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  35. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  36. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItem.ts +12 -0
  37. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator.ts +2 -0
  38. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
  39. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  40. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  41. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue.ts +2 -0
  42. package/openapi/gen/models/datasetExportReport.ts +9 -0
  43. package/openapi/gen/models/datasetExportReportErrorsItem.ts +5 -0
  44. package/openapi/gen/models/datasetExportReportTokens.ts +7 -0
  45. package/openapi/gen/models/exportJob.ts +2 -0
  46. package/openapi/gen/models/exportJobReport.ts +3 -0
  47. package/openapi/gen/models/index.ts +48 -0
  48. package/openapi/gen/models/sampleDatasetItemsBody.ts +4 -0
  49. package/openapi/gen/models/sampleDatasetItemsBodyAnyOf.ts +8 -0
  50. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilter.ts +9 -0
  51. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItem.ts +6 -0
  52. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  53. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  54. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  55. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItem.ts +12 -0
  56. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemOperator.ts +2 -0
  57. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItem.ts +4 -0
  58. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  59. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  60. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemValue.ts +2 -0
  61. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethree.ts +8 -0
  62. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilter.ts +9 -0
  63. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItem.ts +6 -0
  64. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  65. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  66. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  67. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItem.ts +12 -0
  68. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemOperator.ts +2 -0
  69. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
  70. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  71. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  72. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemValue.ts +2 -0
  73. package/openapi/gen/models/sampleJob.ts +13 -0
  74. package/openapi/gen/models/sampleJobStatus.ts +2 -0
  75. package/openapi/gen/models/sampleResult.ts +5 -0
  76. package/package.json +23 -2
  77. package/skills/catalog.json +14 -6
  78. package/skills/toloka-build/SKILL.md +34 -12
  79. package/skills/toloka-build/references/node-semantics.md +27 -24
  80. package/skills/toloka-build/references/workflow-and-cli.md +55 -26
  81. package/skills/toloka-report-issue/references/report-template.md +1 -1
  82. package/src/checks/ast.ts +32 -0
  83. package/src/checks/index.ts +14 -0
  84. package/src/checks/postprocess-collisions.ts +145 -0
  85. package/src/checks/task-ui-declarations.generated.ts +19 -0
  86. package/src/checks/task-ui-typecheck.ts +345 -0
  87. package/src/checks/typescript-vfs.d.ts +26 -0
  88. package/src/checks/typescript-vfs.js +5 -0
  89. package/src/cli/analytics.ts +1 -0
  90. package/src/cli/check.ts +977 -0
  91. package/src/cli/program.ts +147 -19
  92. package/src/cli/project-context.ts +14 -4
  93. package/src/cli/runs.ts +21 -11
  94. package/src/cli/skill.ts +17 -6
  95. package/src/testing/code-node.ts +160 -0
  96. package/src/testing/index.ts +15 -0
  97. package/src/testing/task-ui-browser.ts +120 -0
  98. package/src/testing/task-ui-validation.ts +135 -0
  99. package/src/testing/task-ui.ts +557 -0
  100. package/src/testing/test-evidence.ts +170 -0
  101. package/src/testing/types.ts +141 -0
Binary file
@@ -1 +1 @@
1
- 2e4e22afc766a0a267262fe07718505b43f1cdf364e402a990279e224ad92923 toloka.js.jsc
1
+ 30967b83b7ae6c50962e2482725ca2fb43f1526f3f16b9af2ef2bbe29fd00b7c toloka.js.jsc
@@ -1 +1 @@
1
- 842c3cdfb47d5a899375c288cefa8ba550c22301dbd47d99d224a8b34acd01a2 toloka.js
1
+ 939f5c749f0a627df3f3f0b042e9ef0fffa4c2ccc77f61f20c52ae77b767b20b toloka.js
package/docs/authoring.md CHANGED
@@ -77,6 +77,8 @@ async def main(inputs: Inputs) -> Outputs:
77
77
 
78
78
  If the program calls an external service, list the secret names in `envVars`. The sandbox has no ambient environment.
79
79
 
80
+ Register authored code behavior with `test.code(pipeline, node, name, callback)` from `@toloka-platform/sdk/testing`, execute its bound `run` helper, and cover representative inputs, boundaries, deterministic behavior, and real failures. Bare `runCodeNode` calls remain useful for exploratory tests but do not satisfy `toloka check` coverage.
81
+
80
82
  ### generation
81
83
 
82
84
  One human-work task type. The node owns the entire expert journey: instruction, UI, entity schemas, quality criteria, quality assurance, audience, pricing, Training, and Exam.
@@ -132,6 +134,8 @@ export const annotateFaces = generation("annotate_faces", {
132
134
  });
133
135
  ```
134
136
 
137
+ Test the interface locally before applying it. The SDK's Vitest fixture renders this exact `ui` block in Chromium, interacts with Fields by `path`, runs postprocess, validates output entities, and saves screenshots. See [Test pipelines locally](./testing.md).
138
+
135
139
  ### quorum
136
140
 
137
141
  The same item labeled by N independent people, with disagreement preserved. Every output field becomes a top-level array with one element per labeler.
@@ -144,13 +148,22 @@ export const ratePair = quorum("rate_pair", {
144
148
  publicName: "Rate the pair",
145
149
  overlap: 3,
146
150
  entities: {
147
- input: [{ field: pairUrl, entityType: "TEXT" }],
148
- output: [{ field: verdict, entityType: "TEXT" }],
151
+ input: [{ field: pairUrl, entityType: "CONTEXT" }],
152
+ output: [
153
+ {
154
+ field: verdict,
155
+ entityType: "SINGLE_LABEL",
156
+ allowedLabels: [
157
+ { label: "First video", value: "first" },
158
+ { label: "Second video", value: "second" },
159
+ ],
160
+ },
161
+ ],
149
162
  },
150
163
  qualityCriteria: [],
151
164
  instruction: "Pick the better of the two videos.",
152
165
  ui: {
153
- code: "<></>",
166
+ code: "<View.Card><Field.RadioGroup path={'verdict'} label={'Better video'} options={[{ label: 'First video', value: 'first' }, { label: 'Second video', value: 'second' }]} validation={[{ required: true }]} /></View.Card>",
154
167
  data: "{}",
155
168
  preprocess: "return {}",
156
169
  postprocess: "return {};",
@@ -258,7 +271,7 @@ Never hand-write `fp_…`, `ds.…`, `pl.…`, or `pp.…` values into settings,
258
271
  ## Validate
259
272
 
260
273
  ```sh
261
- bunx tsc --noEmit
274
+ toloka check
262
275
  toloka validate
263
276
  ```
264
277
 
package/docs/cli-v2.md CHANGED
@@ -9,8 +9,8 @@ The platform object is a **project** (`pp.…`, today's pipeline project). Windm
9
9
  Today the checkout is a pipeline tree that happens to mention a project.
10
10
 
11
11
  - A directory is a project root if it has `toloka.json` **or** a `pipelines/` folder. No pipelines, no root.
12
- - `init` writes agent skills before a project exists; if one is already selected, it also binds the directory.
13
- - A checkout attaches when `project create` or `import` writes `toloka.json`, or when `init` finds an already selected project.
12
+ - `init` writes agent skills before a project exists. It writes `toloka.json` only from an explicit `--project-id`, `TOLOKA_PROJECT_ID` / `POTATO_PROJECT_ID`, or an existing checkout; it never binds a new checkout from the machine registry.
13
+ - A checkout attaches when `project create` or `import` writes `toloka.json`, or when `init` receives explicit project identity.
14
14
  - `run`, `status`, `runs`, and `cost` pick a pipeline from `toloka.json`. Zero pipelines is an error. Several pipelines opens a picker. The command names pretend there is "the" pipeline.
15
15
  - `status` means "latest run of some pipeline", not "this project".
16
16
 
@@ -106,7 +106,7 @@ UI links follow the resource-first shape `toloka <resource> url`. `project url`
106
106
  toloka init
107
107
  ```
108
108
 
109
- It writes the agent entry points without calling the platform. When a project is already selected, it also writes `toloka.json` with empty bindings. It does not require `pipelines/`; `project create` creates the live project.
109
+ It writes the agent entry points without calling the platform. It never binds a new checkout from the machine registry. With an explicit `--project-id` or project-id environment variable, it also writes `toloka.json` with empty bindings. It does not require `pipelines/`; `project create` creates the live project.
110
110
 
111
111
  `import` stays the "pull an existing live project into this checkout" path. After v2 it uses the selected project when the id is omitted:
112
112
 
package/docs/cli.md CHANGED
@@ -4,8 +4,7 @@ The `toloka` CLI reads TypeScript source as the desired state, compares it with
4
4
 
5
5
  This CLI is Preview software. Commands, flags, and output can change without a major version bump.
6
6
 
7
- `toloka` requires Bun. The binary declares a `bun` shebang and the package ships TypeScript source with no
8
- build step, so it does not run under plain Node.
7
+ `toloka` requires Bun 1.4.0. The published package ships matching CLI bytecode; the launcher falls back to TypeScript source in a development checkout when the bytecode is absent.
9
8
 
10
9
  ## Configuration
11
10
 
@@ -65,7 +64,7 @@ Resources may also expose specialized actions beyond the Base Actions, such as `
65
64
  Register and select the platform project later commands use.
66
65
 
67
66
  ```sh
68
- toloka project list
67
+ toloka project list [--limit <n>]
69
68
  toloka project add [alias] [projectId]
70
69
  toloka project switch <alias>
71
70
  toloka project current
@@ -152,15 +151,29 @@ toloka validate
152
151
 
153
152
  On success it prints one line per pipeline: `valid`, `invalid` with issue details, or `skipped` when the pipeline is not on the platform yet. Exits `1` when any submitted graph is invalid.
154
153
 
154
+ ## check
155
+
156
+ Runs the complete local gate without platform credentials.
157
+
158
+ ```sh
159
+ toloka check [pipeline] [--node <name-or-id>] [--json] [--watch]
160
+ ```
161
+
162
+ The command loads authored source, runs the project's TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and compares passing typed test evidence with every code node, human-work node, and pipeline. Statuses distinguish `missing`, `failed`, `partial`, and `infrastructure_error`. Exit code `0` means all required checks passed, `1` means source, types, tests, or required coverage failed, and `2` means a required checker could not run.
163
+
164
+ Use `test.code`, `test.ui`, and `test.pipeline` from `@toloka-platform/sdk/testing` to produce required evidence. Plain Vitest tests still run and can fail the command, but do not satisfy graph coverage. A focused check accepts a pipeline plus `--node`; `--watch` serializes reruns after relevant file changes.
165
+
166
+ With `--watch`, the command prints one complete report after each serialized run. With `--watch --json`, each report is one JSON line.
167
+
155
168
  ## apply
156
169
 
157
- Makes the platform match source.
170
+ Runs the complete Local Check Gate, then makes the platform match source.
158
171
 
159
172
  ```sh
160
173
  toloka apply
161
174
  ```
162
175
 
163
- A preflight pass refuses before any write when a pipeline or dataset is removed from source, a project secret is missing, or a dataset field schema conflicts. On success it prints `No changes.` or `Applied N writes to project <projectId>.`
176
+ Checks run before credentials are resolved or any platform request is made. A failed or missing required check exits `1`; a checker infrastructure error exits `2`. After a pass, the preflight refuses before any write when a pipeline or dataset is removed from source, a project secret is missing, or a dataset field schema conflicts. On success it prints `No changes.` or `Applied N writes to project <projectId>.`
164
177
 
165
178
  ## secret set
166
179
 
@@ -187,7 +200,7 @@ On success it prints the project id and name, team id, live counts, and either `
187
200
  Operate on a live pipeline in the selected project. Name or id is required unless a TTY picker can choose among several live pipelines. There is no implicit "the only pipeline."
188
201
 
189
202
  ```sh
190
- toloka pipeline list
203
+ toloka pipeline list [--limit <n>]
191
204
  toloka pipeline run [name] [--wait] [--timeout <ms>]
192
205
  toloka pipeline status [name]
193
206
  toloka pipeline runs [name] [--limit <n>]
@@ -282,21 +295,21 @@ After an SDK upgrade `list` reports installed skills whose catalog digest change
282
295
 
283
296
  `toloka pipeline run --wait` maps the run outcome to the process exit code.
284
297
 
285
- | Code | Meaning |
286
- | ---- | --------------------------------------------- |
287
- | 0 | Run succeeded (terminal state) |
288
- | 1 | Run failed (terminal state) |
289
- | 2 | Run still running when the wait window closed |
298
+ | Code | `pipeline run --wait` | `check` / apply local gate |
299
+ | ---- | ----------------------------------------------- | ------------------------------------------------- |
300
+ | 0 | Run succeeded, or was accepted without `--wait` | All required local checks passed |
301
+ | 1 | Run failed | Source, types, tests, or required coverage failed |
302
+ | 2 | Run remained active when the wait window closed | A required checker could not run |
290
303
 
291
- Without `--wait`, exit code `0` means only that the run was accepted. Every other failure exits non-zero with a message on stderr: missing `toloka.json`, a rejected login, an unauthenticated `auth-status`, or a failed `secret set`.
304
+ Every other failure exits non-zero with a message on stderr: missing project identity, a rejected login, an unauthenticated `auth-status`, or a failed `secret set`.
292
305
 
293
306
  ## The reconcile loop
294
307
 
295
- Treat the TypeScript source as authoritative. The canonical sequence is: typecheck, validate against the platform, plan, apply, then plan again to confirm convergence.
308
+ Treat the TypeScript source as authoritative. The canonical sequence is: run the complete local gate, validate against the platform, plan, apply, then plan again to confirm convergence.
296
309
 
297
310
  ```sh
298
- # 1. Typecheck the project
299
- bun run typecheck
311
+ # 1. Run source, type, and typed test checks
312
+ toloka check
300
313
 
301
314
  # 2. Validate each bound pipeline against the platform
302
315
  toloka validate
@@ -311,7 +324,7 @@ toloka apply
311
324
  toloka plan # expect: No changes.
312
325
  ```
313
326
 
314
- `toloka validate` posts each authored graph to `POST /pipelines/:pipelineId/graph/validate`. It runs platform spec validation and reports the latest matching check jobs. It does not write the graph. A pipeline that is not on the platform yet is skipped until `toloka apply` creates it. Resolve every unexpected plan change, missing secret, and immutable-field conflict before applying.
327
+ `toloka apply` reruns the complete local gate before resolving credentials. `toloka validate` posts each authored graph to `POST /pipelines/:pipelineId/graph/validate`; it reports platform spec validation and the latest matching Remote Node Check jobs without writing the graph. Resolve every unexpected plan change, missing secret, and immutable-field conflict before applying.
315
328
 
316
329
  ## Further reading
317
330
 
@@ -4,14 +4,7 @@ Install the SDK, sign in, and get a pipeline project reconciling from source in
4
4
 
5
5
  ## Install
6
6
 
7
- The package is not published to npm yet. Link it from a checkout of this repository:
8
-
9
- ```sh
10
- cd packages/sdk && bun link
11
- cd /path/to/your/project && bun link @toloka-platform/sdk
12
- ```
13
-
14
- Declare the dependency in `package.json` so `toloka init` recognizes it. Once the hosted installer is deployed, install the bytecode-cached CLI and add the SDK to the project:
7
+ Install the bytecode-cached CLI globally and declare the Preview SDK in the project:
15
8
 
16
9
  ```sh
17
10
  curl -fsSL https://platform.toloka.ai/cli/install.sh | sh
@@ -44,6 +37,8 @@ CI does not need `login`. Set `POTATO_API_KEY` and the stored file is ignored. S
44
37
 
45
38
  ## Start with an agent
46
39
 
40
+ For a guided first run, start a fresh coding-agent session and ask it to follow `https://platform.toloka.ai/onboarding.md`. The guide takes the agent through installation, credentials, project selection, local tests, plan approval, apply, cost review, and a bounded sample run.
41
+
47
42
  Initialize the current directory and install the Agent Bootstrap. No platform project needs to be selected yet.
48
43
 
49
44
  ```sh
@@ -54,7 +49,7 @@ Open your coding agent in this directory and describe the project you want, for
54
49
 
55
50
  > Create a Toloka project that collects captions for images. Show me the plan before applying it.
56
51
 
57
- The bootstrap directs the agent to `toloka --help` and to search and install the matching bundled skill; the `toloka-build` skill then guides the agent through creating and selecting the platform project, authoring the TypeScript source, validating it, showing the plan, and applying the approved changes. `toloka.json` is created when the project is selected.
52
+ The bootstrap directs the agent to `toloka --help` and to search and install the matching bundled skill; the `toloka-build` skill then guides the agent through creating and binding the platform project, authoring the TypeScript source, validating it, showing the plan, and applying the approved changes. `project create`, `import`, or explicit project identity on `init` writes `toloka.json`.
58
53
 
59
54
  ## Manual project setup
60
55
 
@@ -65,8 +60,8 @@ You are either adopting a project that already exists on the platform, or creati
65
60
  ```sh
66
61
  mkdir my-project && cd my-project
67
62
  toloka project add cats pp.019ffa5f-29a7-74de-a76c-7ba5ceedf37f
63
+ toloka import pp.019ffa5f-29a7-74de-a76c-7ba5ceedf37f
68
64
  toloka init
69
- toloka import
70
65
  ```
71
66
 
72
67
  `import` writes a `pipelines/` tree and a `toloka.json` that maps your source to live IDs. If `pipelines/` already exists, pass `--force` to replace it. Read the generated TypeScript: it is the same shape you will edit from now on.
@@ -90,10 +85,10 @@ Then create `pipelines/<name>/pipeline.ts` with a default export and run `plan`
90
85
 
91
86
  ## The loop you will live in
92
87
 
93
- Every change follows the same steps. `validate`, `plan`, and `apply` talk to the platform.
88
+ Every change follows the same steps. `toloka check` is credential-free; `validate`, `plan`, and `apply` talk to the platform. Apply reruns the complete local gate before any platform request.
94
89
 
95
90
  ```sh
96
- bunx tsc --noEmit
91
+ toloka check
97
92
  toloka validate
98
93
  toloka plan
99
94
  toloka apply
@@ -2,22 +2,22 @@
2
2
 
3
3
  What this Preview SDK does not do yet, stated plainly so you can decide what to depend on.
4
4
 
5
- `@toloka-platform/sdk` is `0.1.0-preview.6`. This is a Preview: the authoring API, the CLI surface, and the generated source layout can change without a major version bump. Pin an exact version. Do not depend on this in production yet.
5
+ `@toloka-platform/sdk` is `0.1.0-preview.7`. This is a Preview: the authoring API, the CLI surface, and the generated source layout can change without a major version bump. Pin an exact version. Do not depend on this in production yet.
6
6
 
7
- ## The SDK does not certify deployability
7
+ ## Local checks do not certify deployability
8
8
 
9
- `toloka validate` posts each bound graph to the platform and reports spec errors, warnings, and the latest matching check jobs. It does not write the graph.
9
+ `toloka check` is the required credential-free local gate. It runs project TypeScript, automatic Task UI typechecking, Vitest, and typed node and pipeline-flow coverage. `toloka apply` reruns it before any platform write.
10
10
 
11
- A successful validate, and a successful `apply`, are still not proof that the pipeline can be launched. Gates that live outside that report are invisible to the CLI:
11
+ `toloka validate` separately posts each bound graph to the platform and reports spec errors, warnings, and the latest matching Remote Node Check jobs. It does not write the graph. A passing local gate and platform validation are still not proof that the pipeline can launch:
12
12
 
13
13
  | gate | where it is enforced | what the CLI tells you |
14
14
  | ------------------------------------------------- | -------------------- | ---------------------- |
15
15
  | Training course content, when Training is enabled | platform validation | nothing |
16
16
  | labeler UI validation receipt | platform validation | nothing |
17
17
 
18
- ## Node checks are reported, not started
18
+ The typed client can list, start, and poll Remote Node Checks. `toloka check` does not start them: it validates the authored source and its local tests. A `not_ran_node_check` result from `toloka validate` describes remote state, not missing local evidence.
19
19
 
20
- `toloka validate` includes the latest matching check jobs for the submitted graph. The typed client can list, start, and poll checks, but there is no `toloka check` command that discovers check contracts, configures cases, starts each required lint, waits, and explains reports. Configure and start missing checks in the platform UI for now.
20
+ Python code nodes have executable behavior checks but no useful static Python typecheck yet. Their generated `Inputs` and `Outputs` aliases are currently `dict[str, Any]`; adding Pyright or mypy before those aliases become field-derived `TypedDict` contracts would provide little protection.
21
21
 
22
22
  ## Layout is not authored
23
23
 
@@ -0,0 +1,248 @@
1
+ # Pipeline testing examples
2
+
3
+ Import the ready-to-use Vitest helpers from `@toloka-platform/sdk/testing`. The same entry point executes authored Python code nodes and renders human-work Task UIs in an isolated Chromium context.
4
+
5
+ ## Code node scenarios
6
+
7
+ ```ts
8
+ import { expect, test } from "@toloka-platform/sdk/testing";
9
+ import pipeline from "../../pipeline";
10
+ import { classify } from "./node";
11
+
12
+ const scenarios = [
13
+ { text: "A refund was requested", expected: "billing" },
14
+ { text: "The app crashes", expected: "technical" },
15
+ ];
16
+
17
+ for (const scenario of scenarios) {
18
+ test.code(pipeline, classify, `classifies ${scenario.text}`, async ({ run }) => {
19
+ await expect(run({ text: scenario.text })).resolves.toEqual({ category: scenario.expected });
20
+ });
21
+ }
22
+
23
+ test.code(pipeline, classify, "surfaces authored failures", async ({ run }) => {
24
+ await expect(run({ text: "" })).rejects.toMatchObject({
25
+ name: "CodeNodeExecutionError",
26
+ exitCode: 1,
27
+ stderr: expect.stringContaining("text is required"),
28
+ });
29
+ });
30
+ ```
31
+
32
+ Pass test-only secret values through the bound `run(inputs, { env })`; authored static environment entries are applied automatically. Use `pythonExecutable` only when the project intentionally tests another Python binary, and lower `timeoutMs` for timeout behavior.
33
+
34
+ ## Complete pipeline flow
35
+
36
+ Start with the first row from the same local dataset file intended for upload. Keep that row as a TypeScript fixture; if it contains sensitive values, redact the values while preserving the real field names, nesting, optionality, and media/value shapes. Use synthetic input only when no local row exists.
37
+
38
+ ```ts
39
+ import { expect, test } from "@toloka-platform/sdk/testing";
40
+ import pipeline from "../pipeline";
41
+ import { representativeTicket } from "./fixtures/support-ticket";
42
+ import { normalizeTicket } from "../nodes/normalize-ticket/node";
43
+ import { classifyTicket } from "../nodes/classify-ticket/node";
44
+ import { finalizeTicket } from "../nodes/finalize-ticket/node";
45
+
46
+ test.pipeline(
47
+ pipeline,
48
+ "moves a representative item from root.started to root.completed",
49
+ async ({ runCode, renderUi }) => {
50
+ let item: Record<string, unknown> = { ...representativeTicket };
51
+
52
+ item = { ...item, ...(await runCode(normalizeTicket, item)) };
53
+
54
+ const classificationTask = await renderUi(classifyTicket, { data: item });
55
+ await classificationTask.setValue("category", "billing");
56
+ await classificationTask.setValue("rationale", "The customer disputes a duplicate charge.");
57
+ const classification = await classificationTask.submit();
58
+ item = {
59
+ ...item,
60
+ category: classification.category,
61
+ rationale: classification.rationale,
62
+ };
63
+
64
+ item = { ...item, ...(await runCode(finalizeTicket, item)) };
65
+
66
+ expect(item).toMatchObject({
67
+ ticket_id: representativeTicket.ticket_id,
68
+ category: "billing",
69
+ resolution_status: "ready",
70
+ });
71
+ },
72
+ );
73
+ ```
74
+
75
+ This explicit item-state threading is the local equivalent of following pipeline connections: pass root input to the first node, merge its output, and continue until the completed fields are available. Keep this happy path in one test when the graph is linear. Use separate tests for each match branch or materially different path. For quorum, collect several simulated Task UI submissions and assemble each outer annotation array before calling a downstream code node.
76
+
77
+ ## Default Task UI fixture
78
+
79
+ ```ts
80
+ import { expect, test } from "@toloka-platform/sdk/testing";
81
+ import pipeline from "../../pipeline";
82
+ import { annotate } from "./node";
83
+
84
+ test.ui(pipeline, annotate, "submits a valid answer", async ({ render }) => {
85
+ const task = await render({
86
+ data: { prompt: "What is two plus two?" },
87
+ });
88
+
89
+ await task.setValue("raw_answer", "four");
90
+
91
+ expect(await task.validationErrors()).toEqual([]);
92
+ expect(await task.submit()).toMatchObject({ answer: "four" });
93
+ });
94
+ ```
95
+
96
+ The browser is created once per Vitest worker. The typed wrapper's `render` helper creates an isolated browser context and page for the scenario. Every opened Task UI session is closed automatically after the scenario, including failed scenarios.
97
+
98
+ ## Several TypeScript scenarios
99
+
100
+ Scenarios are ordinary TypeScript. Use shared builders and loops instead of generating a separate JSON scenario format.
101
+
102
+ ```ts
103
+ import { expect, test } from "@toloka-platform/sdk/testing";
104
+ import pipeline from "../../pipeline";
105
+ import { classify } from "./node";
106
+
107
+ const scenarios = [
108
+ { name: "short text", data: { text: "yes" } },
109
+ { name: "long text", data: { text: "a long paragraph" } },
110
+ { name: "empty optional context", data: { text: "yes", context: null } },
111
+ ];
112
+
113
+ for (const scenario of scenarios) {
114
+ test.ui(pipeline, classify, scenario.name, async ({ render }) => {
115
+ const task = await render({ data: scenario.data });
116
+
117
+ await task.setValue("label", "accepted");
118
+
119
+ expect(await task.validationErrors()).toEqual([]);
120
+ expect(await task.submit()).toMatchObject({ label: "accepted" });
121
+ });
122
+ }
123
+ ```
124
+
125
+ Each scenario receives a fresh browser context, so values set by one scenario cannot satisfy another scenario.
126
+
127
+ ## Inspect component contracts
128
+
129
+ Component documentation comes from the same TBX launcher used to render the interface.
130
+
131
+ ```ts
132
+ import { expect, test } from "@toloka-platform/sdk/testing";
133
+ import pipeline from "../../pipeline";
134
+ import { classify } from "./node";
135
+
136
+ test("uses the expected Field contract", async ({ ui }) => {
137
+ const task = await ui.render(pipeline, classify);
138
+ const documentation = await task.componentDocumentation(["Field.RadioGroup", "View.Card"]);
139
+
140
+ expect(documentation).toMatchObject({
141
+ "Field.RadioGroup": { valueType: expect.stringContaining("string") },
142
+ "View.Card": { propsType: expect.stringContaining("title") },
143
+ });
144
+ });
145
+ ```
146
+
147
+ Requesting an unknown component rejects with an actionable error. It does not silently return an empty entry.
148
+
149
+ ```ts
150
+ await expect(task.componentDocumentation(["Field.Invented"])).rejects.toThrow(
151
+ "Unknown Task UI components: Field.Invented",
152
+ );
153
+ ```
154
+
155
+ ## Assert browser validation failures
156
+
157
+ ```ts
158
+ import { expect, TaskUiValidationError, test } from "@toloka-platform/sdk/testing";
159
+ import pipeline from "../../pipeline";
160
+ import { annotate } from "./node";
161
+
162
+ test("requires an answer", async ({ ui }) => {
163
+ const task = await ui.render(pipeline, annotate);
164
+
165
+ expect(await task.validationErrors()).toEqual([expect.objectContaining({ path: "raw_answer" })]);
166
+
167
+ await expect(task.submit()).rejects.toBeInstanceOf(TaskUiValidationError);
168
+ });
169
+ ```
170
+
171
+ `TaskUiValidationError` exposes three structured fields:
172
+
173
+ - `formErrors` contains browser component validation failures.
174
+ - `entityErrors` contains postprocessed output values that fail entity schemas.
175
+ - `coverage` reports untested hardware-backed fields.
176
+
177
+ ## Assert postprocessed output
178
+
179
+ `submit()` returns the final Task UI output after postprocess runs. Launcher-only `_context` metadata is removed.
180
+
181
+ ```ts
182
+ const task = await ui.render(pipeline, normalizeBox);
183
+
184
+ await task.setValue("raw_box", {
185
+ x: 20,
186
+ y: 10,
187
+ width: 100,
188
+ height: 50,
189
+ });
190
+
191
+ expect(await task.submit()).toEqual({
192
+ raw_box: { x: 20, y: 10, width: 100, height: 50 },
193
+ box: { x: 0.2, y: 0.1, width: 1, height: 0.5 },
194
+ });
195
+ ```
196
+
197
+ The submitted result is also checked against every non-hardware output entity schema declared by the authored node.
198
+
199
+ ## Hardware-backed fields
200
+
201
+ Headless Chromium cannot complete native file pickers, cameras, microphones, or recording components. By default, `submit()` rejects rather than claiming complete coverage.
202
+
203
+ ```ts
204
+ const task = await ui.render(pipeline, recordVideo);
205
+
206
+ expect(task.coverage.complete).toBe(false);
207
+ expect(task.coverage.untestedHardwareFields).toContain("video");
208
+ await expect(task.submit()).rejects.toThrow("hardware-backed fields");
209
+ ```
210
+
211
+ When the scenario deliberately verifies only the testable fields, acknowledge partial coverage explicitly:
212
+
213
+ ```ts
214
+ const task = await ui.render(pipeline, recordVideo, {
215
+ allowPartialCoverage: true,
216
+ });
217
+
218
+ await task.setValue("description", "A short outdoor video");
219
+ expect(await task.submit()).toMatchObject({
220
+ description: "A short outdoor video",
221
+ });
222
+ expect(task.coverage.complete).toBe(false);
223
+ ```
224
+
225
+ A partial scenario is useful for structure and supporting fields, but it is not full verification of the hardware workflow.
226
+
227
+ ## Manual cleanup
228
+
229
+ Automatic teardown is sufficient for normal tests. Call `close()` only when a long scenario no longer needs a page and should release it early.
230
+
231
+ ```ts
232
+ const task = await ui.render(pipeline, annotate);
233
+
234
+ await task.screenshot("test-results/initial.png");
235
+ await task.close();
236
+ ```
237
+
238
+ `close()` is idempotent. The fixture’s automatic teardown can safely call it again.
239
+
240
+ ## Setup and execution
241
+
242
+ ```sh
243
+ bun add --dev vitest playwright
244
+ bunx playwright install chromium
245
+ toloka check
246
+ ```
247
+
248
+ The local browser still downloads the hosted launcher, so use synthetic or redacted test data. `toloka check` runs project types, automatic Task UI types, typed node tests, and pipeline-flow evidence. Remote Node Checks remain separate platform jobs.
@@ -0,0 +1,115 @@
1
+ # Test pipelines locally
2
+
3
+ The `@toloka-platform/sdk/testing` entry point provides typed Vitest wrappers for executable Python `code` nodes, Chromium Task UI scenarios, and complete pipeline flows. `toloka check` runs those tests together with project TypeScript and automatic Task UI typechecking without creating or modifying a platform project.
4
+
5
+ ## Install test dependencies
6
+
7
+ ```sh
8
+ bun add --dev vitest playwright
9
+ bunx playwright install chromium
10
+ ```
11
+
12
+ Code-node tests also require `python3` on the test machine. See [testing examples](./testing-examples.md) for parameterization, validation failures, hardware coverage, and cleanup patterns.
13
+
14
+ ## Test a code node
15
+
16
+ `test.code(pipeline, node, name, callback)` binds one test to one authored code node. Its `run(inputs, options?)` helper executes the Python source with the same async `main(inputs)` entry contract. The test counts as node coverage only after `run` executes, at least one assertion runs, and the callback passes.
17
+
18
+ ```ts
19
+ import { expect, test } from "@toloka-platform/sdk/testing";
20
+ import pipeline from "../../pipeline";
21
+ import { normalize } from "./node";
22
+
23
+ test.code(pipeline, normalize, "normalizes whitespace", async ({ run }) => {
24
+ await expect(run({ text: " hello " })).resolves.toEqual({
25
+ normalized_text: "hello",
26
+ });
27
+ });
28
+ ```
29
+
30
+ Failures from the bound executor throw `CodeNodeExecutionError` with the Python exit code and stderr. Authored static environment values are included; pass secret or test-specific values through the optional `run` options. Third-party imports must be available to the selected Python executable.
31
+
32
+ `test.code` defaults to 30 seconds, which exceeds the code runner's 10-second default. Pass `{ timeout: milliseconds }` after the callback when the scenario also raises the runner timeout.
33
+
34
+ ## Test a complete pipeline flow
35
+
36
+ Register the representative connected scenario with `test.pipeline(pipeline, name, callback)`. Its callback provides `runCode(node, inputs, options?)` and `renderUi(node, options?)` bound to that pipeline.
37
+
38
+ Use the first row from the same local dataset file intended for upload when one is available. If it contains sensitive values, keep a redacted fixture that preserves field names, nesting, optionality, and media/value shapes. Use synthetic input only when no local row exists. `test.pipeline` does not fetch a live dataset: the local gate remains deterministic, credential-free, and usable before apply. Treat the fixture as `root.started` item state, execute connected nodes in order, merge only each node's declared outputs, and assert the fields accepted by `root.completed`. A linear happy path can stay in one test; use separate scenarios for materially different match branches, optional paths, and failures.
39
+
40
+ ## Test a Task UI
41
+
42
+ Create a file ending in `.ui.spec.ts` next to the human-work node. `test.ui` binds the test to the pipeline and node, and its callback receives `render(options?)` for that target. The test counts as coverage only after a successful submission, at least one assertion, and a passing callback.
43
+
44
+ Browser-backed `test.ui` scenarios default to 60 seconds, which exceeds the launcher's 45-second setup budget. `test.pipeline` scenarios default to 120 seconds for connected flows with multiple UI steps. Pass `{ timeout: milliseconds }` after the callback to tune either wrapper without weakening its inner launcher deadlines.
45
+
46
+ ```ts
47
+ import { expect, test } from "@toloka-platform/sdk/testing";
48
+ import pipeline from "../../pipeline";
49
+ import { annotate } from "./node";
50
+
51
+ test.ui(pipeline, annotate, "submits a valid answer", async ({ render }) => {
52
+ const task = await render({
53
+ data: { prompt: "What is two plus two?" },
54
+ });
55
+
56
+ await task.setValue("raw_answer", "four");
57
+
58
+ expect(await task.validationErrors()).toEqual([]);
59
+ expect(await task.submit()).toMatchObject({
60
+ answer: "four",
61
+ });
62
+ await task.screenshot("test-results/answer.png");
63
+ });
64
+ ```
65
+
66
+ Run the complete gate rather than invoking Vitest separately:
67
+
68
+ ```sh
69
+ toloka check
70
+ ```
71
+
72
+ Use an ordinary TypeScript loop to register several `test.code` or `test.ui` scenarios. There is no separate JSON scenario format.
73
+
74
+ ## Fixture API
75
+
76
+ The low-level `ui` fixture remains available inside ordinary Vitest tests for component discovery and exploratory work, but plain tests do not satisfy required node coverage. Use typed wrappers for committed gate evidence. Importing `@toloka-platform/sdk/testing` from a plain Bun script executes Vitest fixture setup and fails.
77
+
78
+ - `ui.componentDocumentation(names?)` returns the current launcher's supported component contracts. Call it before choosing component names or props.
79
+ - `ui.render(pipeline, node, options)` creates a fresh browser context and returns a Task UI session for exactly that authored node. `test.ui` supplies the equivalent bound `render(options?)` helper.
80
+
81
+ The returned Task UI session supports:
82
+
83
+ - `componentTree()` returns the rendered component hierarchy.
84
+ - `componentDocumentation(names?)` returns component props, value shapes, capabilities, and limitations.
85
+ - `setValue(path, value)` and `getValue(path)` address Fields by their stable `path`, not generated component ids.
86
+ - `validationErrors()` returns browser-level required-field and component-rule failures with Field paths.
87
+ - `submit()` executes preprocess/postprocess, removes the launcher's `_context`, and validates declared output entities. The returned form data can also contain echoed inputs; assert output-entity keys and values without treating those inputs as output entities.
88
+ - `screenshot(path)` writes a full-page PNG to the local filesystem.
89
+ - `coverage` states whether hardware-backed fields were exercised completely.
90
+
91
+ One Chromium process is shared per Vitest worker. Every scenario gets a fresh context and page and is cleaned up automatically.
92
+
93
+ For a structured or array-valued Field, derive the dataset field schema from the component's documented `valueType`. In its typed UI scenario, adapt only semantic values such as labels, then pass the documented example value through `setValue`, `getValue`, and `submit`. This catches component-shape drift before a postprocess can read absent keys and silently write fallback data.
94
+
95
+ ## Runtime rules
96
+
97
+ `ui.code` must parse as one JSX root. Every identifier must be a component returned by `componentDocumentation()` or a documented `actions` API. `toloka check` applies full TypeScript semantic checking with declarations bundled in the SDK; the real launcher then renders the same JSX in browser scenarios. Treat a component's `propsType` and `example` together as its contract.
98
+
99
+ `preprocess` and `postprocess` are JavaScript function bodies, not TypeScript:
100
+
101
+ Valid JavaScript: `items.map((item) => item.value)`
102
+
103
+ Invalid runtime JavaScript: `items.map((item: unknown) => item.value)`
104
+
105
+ `preprocess` can run again after form state changes. Use it to shape input data; if it prefills a bound field, return that key only while it is absent so later edits are not overwritten. A `Field`-bound top-level key already enters the submission. `postprocess` must return only new top-level keys: bind a component to a draft key when postprocess must derive a different entity key.
106
+
107
+ The fixture loads the hosted TBX launcher in a fresh local Chromium context, so tests require network access but do not use Azure Playwright or Potato application services. The context has no platform cookies or credentials. Use synthetic or redacted fixtures because the hosted launcher executes with access to the supplied test data.
108
+
109
+ ## Hardware-backed fields
110
+
111
+ File pickers, video upload/recording, and other hardware-backed fields cannot be completed reliably in headless Chromium. `task.coverage.complete` is `false` when these components are present. Render with `allowPartialCoverage: true`, inject one value matching the component's documented `valueType` through `setValue`, and assert the postprocessed output shape yourself: entity validation skips hardware-backed paths, and `getValue` may not serialize their launcher state. This checks browser validation and mapping only, not capture, upload, link generation, or record-time constraints. A partial result must not be described as complete coverage.
112
+
113
+ ## Final gate
114
+
115
+ `toloka check` loads every authored pipeline, runs the project TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and requires passing `test.code`, `test.ui`, and `test.pipeline` evidence. `missing`, `failed`, `partial`, and `infrastructure_error` are distinct blocking outcomes. `toloka apply` reruns this complete gate before resolving credentials or writing platform state. Platform source/spec errors from `toloka validate` remain separate; `not_ran_node_check` describes a Remote Node Check and does not replace local evidence.