@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -15
- package/dist/cli/toloka.js +209 -121
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +20 -9
- package/docs/cli-v2.md +3 -3
- package/docs/cli.md +44 -16
- package/docs/getting-started.md +7 -12
- package/docs/limitations.md +6 -6
- package/docs/testing-examples.md +248 -0
- package/docs/testing.md +115 -0
- package/examples/face-image-collection/README.md +7 -1
- package/examples/face-image-collection/fields.ts +20 -8
- package/examples/face-image-collection/nodes/annotate-faces/node.ts +1 -9
- package/examples/face-image-collection/nodes/order-faces/node.ts +3 -5
- package/examples/face-image-collection/pipeline.browser.integration.spec.ts +78 -0
- package/examples/video-pair-rating/README.md +14 -6
- package/examples/video-pair-rating/nodes/adjudicate/node.ts +1 -3
- package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +1 -3
- package/examples/video-pair-rating/nodes/adjudication_match/node.ts +15 -17
- package/examples/video-pair-rating/nodes/assemble/node.ts +25 -27
- package/examples/video-pair-rating/nodes/cascade/node.ts +24 -26
- package/examples/video-pair-rating/nodes/finalize_overall/node.ts +17 -19
- package/examples/video-pair-rating/nodes/merge_adjudication/node.ts +13 -15
- package/examples/video-pair-rating/nodes/merge_op_adjudication/node.ts +12 -14
- package/examples/video-pair-rating/nodes/op_quorum_resolve/node.ts +13 -15
- package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +1 -1
- package/examples/video-pair-rating/nodes/rate_pair/node.ts +1 -1
- package/examples/video-pair-rating/nodes/route_adjudication/node.ts +1 -3
- package/examples/video-pair-rating/nodes/route_op_adjudication/node.ts +5 -7
- package/examples/video-pair-rating/nodes/route_overall/node.ts +1 -3
- package/examples/video-pair-rating/nodes/shuffle_ab/node.ts +11 -13
- package/examples/video-pair-rating/pipeline.browser.integration.spec.ts +267 -0
- package/openapi/gen/api.ts +48 -11
- package/openapi/gen/models/createDatasetExportBody.ts +3 -3
- package/openapi/gen/models/createDatasetExportBodyOneOf.ts +6 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfFormat.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThree.ts +10 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeContract.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeFormat.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilter.ts +9 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/datasetExportReport.ts +9 -0
- package/openapi/gen/models/datasetExportReportErrorsItem.ts +5 -0
- package/openapi/gen/models/datasetExportReportTokens.ts +7 -0
- package/openapi/gen/models/datasetItem.ts +0 -2
- package/openapi/gen/models/exportJob.ts +2 -0
- package/openapi/gen/models/exportJobReport.ts +3 -0
- package/openapi/gen/models/index.ts +48 -1
- package/openapi/gen/models/sampleDatasetItemsBody.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOf.ts +8 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilter.ts +9 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethree.ts +8 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilter.ts +9 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/sampleJob.ts +13 -0
- package/openapi/gen/models/sampleJobStatus.ts +2 -0
- package/openapi/gen/models/sampleResult.ts +5 -0
- package/package.json +23 -2
- package/skills/catalog.json +14 -6
- package/skills/toloka-build/SKILL.md +35 -12
- package/skills/toloka-build/references/node-semantics.md +32 -29
- package/skills/toloka-build/references/workflow-and-cli.md +61 -27
- package/skills/toloka-report-issue/references/report-template.md +1 -1
- package/src/authoring/index.ts +27 -26
- package/src/checks/ast.ts +32 -0
- package/src/checks/index.ts +14 -0
- package/src/checks/postprocess-collisions.ts +145 -0
- package/src/checks/task-ui-declarations.generated.ts +19 -0
- package/src/checks/task-ui-typecheck.ts +345 -0
- package/src/checks/typescript-vfs.d.ts +26 -0
- package/src/checks/typescript-vfs.js +5 -0
- package/src/cli/analytics.ts +9 -19
- package/src/cli/auto-update.ts +367 -0
- package/src/cli/boolean-flag.ts +13 -0
- package/src/cli/check.ts +977 -0
- package/src/cli/credentials.ts +3 -0
- package/src/cli/program.ts +184 -20
- package/src/cli/project-context.ts +14 -4
- package/src/cli/runs.ts +21 -11
- package/src/cli/settings.ts +93 -0
- package/src/cli/skill.ts +17 -6
- package/src/importer.ts +12 -16
- package/src/testing/code-node.ts +160 -0
- package/src/testing/index.ts +15 -0
- package/src/testing/task-ui-browser.ts +120 -0
- package/src/testing/task-ui-validation.ts +135 -0
- package/src/testing/task-ui.ts +557 -0
- package/src/testing/test-evidence.ts +170 -0
- package/src/testing/types.ts +141 -0
- package/openapi/gen/models/datasetItemDuplicateOfSubmissionId.ts +0 -2
package/dist/cli/toloka.js.jsc
CHANGED
|
Binary file
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
ce1d79afa2e57b085b55b98cf5c4cb90ab84b8a6ba50e586586f208a70f7f088 toloka.js.jsc
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
44e401115084c2e912fe03234c4104090d1698c95e170f16cac9c4505952709a toloka.js
|
package/docs/authoring.md
CHANGED
|
@@ -61,10 +61,8 @@ import { code } from "@toloka-platform/sdk";
|
|
|
61
61
|
export const orderFaces = code("order_faces", {
|
|
62
62
|
about: { name: "order_faces", description: "Sort annotations top-left to bottom-right." },
|
|
63
63
|
envVars: [],
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
output: [{ field: faces, optional: false }],
|
|
67
|
-
},
|
|
64
|
+
input: [{ field: draftFaces, optional: false }],
|
|
65
|
+
output: [{ field: faces, optional: false }],
|
|
68
66
|
code: {
|
|
69
67
|
code: `from main_types import Inputs, Outputs
|
|
70
68
|
|
|
@@ -77,6 +75,8 @@ async def main(inputs: Inputs) -> Outputs:
|
|
|
77
75
|
|
|
78
76
|
If the program calls an external service, list the secret names in `envVars`. The sandbox has no ambient environment.
|
|
79
77
|
|
|
78
|
+
Register authored code behavior with `test.code(pipeline, node, name, callback)` from `@toloka-platform/sdk/testing`, execute its bound `run` helper, and cover representative inputs, boundaries, deterministic behavior, and real failures. Bare `runCodeNode` calls remain useful for exploratory tests but do not satisfy `toloka check` coverage.
|
|
79
|
+
|
|
80
80
|
### generation
|
|
81
81
|
|
|
82
82
|
One human-work task type. The node owns the entire expert journey: instruction, UI, entity schemas, quality criteria, quality assurance, audience, pricing, Training, and Exam.
|
|
@@ -132,6 +132,8 @@ export const annotateFaces = generation("annotate_faces", {
|
|
|
132
132
|
});
|
|
133
133
|
```
|
|
134
134
|
|
|
135
|
+
Test the interface locally before applying it. The SDK's Vitest fixture renders this exact `ui` block in Chromium, interacts with Fields by `path`, runs postprocess, validates output entities, and saves screenshots. See [Test pipelines locally](./testing.md).
|
|
136
|
+
|
|
135
137
|
### quorum
|
|
136
138
|
|
|
137
139
|
The same item labeled by N independent people, with disagreement preserved. Every output field becomes a top-level array with one element per labeler.
|
|
@@ -144,13 +146,22 @@ export const ratePair = quorum("rate_pair", {
|
|
|
144
146
|
publicName: "Rate the pair",
|
|
145
147
|
overlap: 3,
|
|
146
148
|
entities: {
|
|
147
|
-
input: [{ field: pairUrl, entityType: "
|
|
148
|
-
output: [
|
|
149
|
+
input: [{ field: pairUrl, entityType: "CONTEXT" }],
|
|
150
|
+
output: [
|
|
151
|
+
{
|
|
152
|
+
field: verdict,
|
|
153
|
+
entityType: "SINGLE_LABEL",
|
|
154
|
+
allowedLabels: [
|
|
155
|
+
{ label: "First video", value: "first" },
|
|
156
|
+
{ label: "Second video", value: "second" },
|
|
157
|
+
],
|
|
158
|
+
},
|
|
159
|
+
],
|
|
149
160
|
},
|
|
150
161
|
qualityCriteria: [],
|
|
151
162
|
instruction: "Pick the better of the two videos.",
|
|
152
163
|
ui: {
|
|
153
|
-
code: "
|
|
164
|
+
code: "<View.Card><Field.RadioGroup path={'verdict'} label={'Better video'} options={[{ label: 'First video', value: 'first' }, { label: 'Second video', value: 'second' }]} validation={[{ required: true }]} /></View.Card>",
|
|
154
165
|
data: "{}",
|
|
155
166
|
preprocess: "return {}",
|
|
156
167
|
postprocess: "return {};",
|
|
@@ -197,7 +208,7 @@ export const routeDisputed = match("route_disputed", {
|
|
|
197
208
|
predicates: [{ field: verdict, jsonPath: "$", operator: "text_eq", value: "disputed" }],
|
|
198
209
|
},
|
|
199
210
|
],
|
|
200
|
-
|
|
211
|
+
input: [{ field: verdict, optional: false }],
|
|
201
212
|
});
|
|
202
213
|
```
|
|
203
214
|
|
|
@@ -258,7 +269,7 @@ Never hand-write `fp_…`, `ds.…`, `pl.…`, or `pp.…` values into settings,
|
|
|
258
269
|
## Validate
|
|
259
270
|
|
|
260
271
|
```sh
|
|
261
|
-
|
|
272
|
+
toloka check
|
|
262
273
|
toloka validate
|
|
263
274
|
```
|
|
264
275
|
|
package/docs/cli-v2.md
CHANGED
|
@@ -9,8 +9,8 @@ The platform object is a **project** (`pp.…`, today's pipeline project). Windm
|
|
|
9
9
|
Today the checkout is a pipeline tree that happens to mention a project.
|
|
10
10
|
|
|
11
11
|
- A directory is a project root if it has `toloka.json` **or** a `pipelines/` folder. No pipelines, no root.
|
|
12
|
-
- `init` writes agent skills before a project exists
|
|
13
|
-
- A checkout attaches when `project create` or `import` writes `toloka.json`, or when `init`
|
|
12
|
+
- `init` writes agent skills before a project exists. It writes `toloka.json` only from an explicit `--project-id`, `TOLOKA_PROJECT_ID` / `POTATO_PROJECT_ID`, or an existing checkout; it never binds a new checkout from the machine registry.
|
|
13
|
+
- A checkout attaches when `project create` or `import` writes `toloka.json`, or when `init` receives explicit project identity.
|
|
14
14
|
- `run`, `status`, `runs`, and `cost` pick a pipeline from `toloka.json`. Zero pipelines is an error. Several pipelines opens a picker. The command names pretend there is "the" pipeline.
|
|
15
15
|
- `status` means "latest run of some pipeline", not "this project".
|
|
16
16
|
|
|
@@ -106,7 +106,7 @@ UI links follow the resource-first shape `toloka <resource> url`. `project url`
|
|
|
106
106
|
toloka init
|
|
107
107
|
```
|
|
108
108
|
|
|
109
|
-
It writes the agent entry points without calling the platform.
|
|
109
|
+
It writes the agent entry points without calling the platform. It never binds a new checkout from the machine registry. With an explicit `--project-id` or project-id environment variable, it also writes `toloka.json` with empty bindings. It does not require `pipelines/`; `project create` creates the live project.
|
|
110
110
|
|
|
111
111
|
`import` stays the "pull an existing live project into this checkout" path. After v2 it uses the selected project when the id is omitted:
|
|
112
112
|
|
package/docs/cli.md
CHANGED
|
@@ -4,8 +4,7 @@ The `toloka` CLI reads TypeScript source as the desired state, compares it with
|
|
|
4
4
|
|
|
5
5
|
This CLI is Preview software. Commands, flags, and output can change without a major version bump.
|
|
6
6
|
|
|
7
|
-
`toloka` requires Bun. The
|
|
8
|
-
build step, so it does not run under plain Node.
|
|
7
|
+
`toloka` requires Bun 1.4.0. The published package ships matching CLI bytecode; the launcher falls back to TypeScript source in a development checkout when the bytecode is absent.
|
|
9
8
|
|
|
10
9
|
## Configuration
|
|
11
10
|
|
|
@@ -33,6 +32,8 @@ Credentials and the platform base URL resolve from the same precedence order, hi
|
|
|
33
32
|
|
|
34
33
|
`toloka login` verifies the key against the platform before saving anything. It writes `$XDG_CONFIG_HOME/toloka/credentials.json`, which defaults to `~/.config/toloka/credentials.json`, with file mode `0600` and a parent directory with mode `0700`. The file may also store local project aliases. The legacy `~/.toloka/credentials.json` is still read when the new path has no file. `toloka logout` deletes the stored file but does not revoke the key.
|
|
35
34
|
|
|
35
|
+
Machine settings live next to that credentials file in `settings.json` and default `auto-update` to `true`. A globally installed `toloka` checks `https://registry.npmjs.org/` at most once per day and, when a newer `@toloka-platform/sdk` version has been published for at least 24 hours, installs it with `bun add --global` and re-runs the original command. The check is pinned to that registry; a checkout `.env` cannot retarget it. `config`, `version`, and `help` never trigger a check. Project-local installs, CI, and `TOLOKA_SKIP_AUTO_UPDATE=1` also skip it. `TOLOKA_AUTO_UPDATE=0` turns it off for one command; `TOLOKA_AUTO_UPDATE=1` turns it on even when the setting is false.
|
|
36
|
+
|
|
36
37
|
Ordinary usage events (`sdk_cli_command_completed` / `sdk_cli_command_failed`) go to Toloka's PostHog project so we can see which commands run and which error classes they hit. Their payloads contain only an allowlisted command classifier, duration, exit code, outcome or error code, SDK and Bun versions, actor, and channel. Raw arguments, flag values, environment variables, API keys, secret names, base URLs and hosts, local paths, project ids, error messages, stack traces, and source context are not sent. GeoIP enrichment is disabled. Distinct id starts as a random UUID stored next to the credentials file. After a successful authenticated command, the CLI calls `GET /me` and aliases that UUID onto the platform user id without setting person properties. Set `TOLOKA_TELEMETRY=0` or `DO_NOT_TRACK=1` to disable capture. CI environments are disabled unless `TOLOKA_TELEMETRY=1`. The explicitly approved full-text `report` event is documented in [report](#report).
|
|
37
38
|
|
|
38
39
|
The root program accepts these flags before any command:
|
|
@@ -65,7 +66,7 @@ Resources may also expose specialized actions beyond the Base Actions, such as `
|
|
|
65
66
|
Register and select the platform project later commands use.
|
|
66
67
|
|
|
67
68
|
```sh
|
|
68
|
-
toloka project list
|
|
69
|
+
toloka project list [--limit <n>]
|
|
69
70
|
toloka project add [alias] [projectId]
|
|
70
71
|
toloka project switch <alias>
|
|
71
72
|
toloka project current
|
|
@@ -108,6 +109,19 @@ toloka auth-status [--show-token]
|
|
|
108
109
|
|
|
109
110
|
On success it prints the platform URL with its source, the API key masked unless `--show-token` is passed, the selected project, and `Reachable: N project(s) visible.` The project line is printed even when no key is in effect or the platform is not reachable. It exits `1` when no key is in effect or the platform is not reachable.
|
|
110
111
|
|
|
112
|
+
## config
|
|
113
|
+
|
|
114
|
+
Reads and writes machine CLI settings. The only setting today is `auto-update`, which defaults to `true`.
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
toloka config
|
|
118
|
+
toloka config get
|
|
119
|
+
toloka config get auto-update
|
|
120
|
+
toloka config set auto-update false
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`config` and `config get` with no name print `auto-update=true` or `auto-update=false`. `config get auto-update` prints only the boolean. `config set auto-update <value>` accepts `true` / `false` / `1` / `0` / `on` / `off` / `yes` / `no` and prints `Set auto-update to <boolean>.` Unknown names and values exit `1`.
|
|
124
|
+
|
|
111
125
|
## init
|
|
112
126
|
|
|
113
127
|
Initializes the current project directory and installs the Agent Bootstrap entry points.
|
|
@@ -152,15 +166,29 @@ toloka validate
|
|
|
152
166
|
|
|
153
167
|
On success it prints one line per pipeline: `valid`, `invalid` with issue details, or `skipped` when the pipeline is not on the platform yet. Exits `1` when any submitted graph is invalid.
|
|
154
168
|
|
|
169
|
+
## check
|
|
170
|
+
|
|
171
|
+
Runs the complete local gate without platform credentials.
|
|
172
|
+
|
|
173
|
+
```sh
|
|
174
|
+
toloka check [pipeline] [--node <name-or-id>] [--json] [--watch]
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
The command loads authored source, runs the project's TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and compares passing typed test evidence with every code node, human-work node, and pipeline. Statuses distinguish `missing`, `failed`, `partial`, and `infrastructure_error`. Exit code `0` means all required checks passed, `1` means source, types, tests, or required coverage failed, and `2` means a required checker could not run.
|
|
178
|
+
|
|
179
|
+
Use `test.code`, `test.ui`, and `test.pipeline` from `@toloka-platform/sdk/testing` to produce required evidence. Plain Vitest tests still run and can fail the command, but do not satisfy graph coverage. A focused check accepts a pipeline plus `--node`; `--watch` serializes reruns after relevant file changes.
|
|
180
|
+
|
|
181
|
+
With `--watch`, the command prints one complete report after each serialized run. With `--watch --json`, each report is one JSON line.
|
|
182
|
+
|
|
155
183
|
## apply
|
|
156
184
|
|
|
157
|
-
|
|
185
|
+
Runs the complete Local Check Gate, then makes the platform match source.
|
|
158
186
|
|
|
159
187
|
```sh
|
|
160
188
|
toloka apply
|
|
161
189
|
```
|
|
162
190
|
|
|
163
|
-
A
|
|
191
|
+
Checks run before credentials are resolved or any platform request is made. A failed or missing required check exits `1`; a checker infrastructure error exits `2`. After a pass, the preflight refuses before any write when a pipeline or dataset is removed from source, a project secret is missing, or a dataset field schema conflicts. On success it prints `No changes.` or `Applied N writes to project <projectId>.`
|
|
164
192
|
|
|
165
193
|
## secret set
|
|
166
194
|
|
|
@@ -187,7 +215,7 @@ On success it prints the project id and name, team id, live counts, and either `
|
|
|
187
215
|
Operate on a live pipeline in the selected project. Name or id is required unless a TTY picker can choose among several live pipelines. There is no implicit "the only pipeline."
|
|
188
216
|
|
|
189
217
|
```sh
|
|
190
|
-
toloka pipeline list
|
|
218
|
+
toloka pipeline list [--limit <n>]
|
|
191
219
|
toloka pipeline run [name] [--wait] [--timeout <ms>]
|
|
192
220
|
toloka pipeline status [name]
|
|
193
221
|
toloka pipeline runs [name] [--limit <n>]
|
|
@@ -282,21 +310,21 @@ After an SDK upgrade `list` reports installed skills whose catalog digest change
|
|
|
282
310
|
|
|
283
311
|
`toloka pipeline run --wait` maps the run outcome to the process exit code.
|
|
284
312
|
|
|
285
|
-
| Code |
|
|
286
|
-
| ---- |
|
|
287
|
-
| 0 | Run succeeded
|
|
288
|
-
| 1 | Run failed
|
|
289
|
-
| 2 | Run
|
|
313
|
+
| Code | `pipeline run --wait` | `check` / apply local gate |
|
|
314
|
+
| ---- | ----------------------------------------------- | ------------------------------------------------- |
|
|
315
|
+
| 0 | Run succeeded, or was accepted without `--wait` | All required local checks passed |
|
|
316
|
+
| 1 | Run failed | Source, types, tests, or required coverage failed |
|
|
317
|
+
| 2 | Run remained active when the wait window closed | A required checker could not run |
|
|
290
318
|
|
|
291
|
-
|
|
319
|
+
Every other failure exits non-zero with a message on stderr: missing project identity, a rejected login, an unauthenticated `auth-status`, or a failed `secret set`.
|
|
292
320
|
|
|
293
321
|
## The reconcile loop
|
|
294
322
|
|
|
295
|
-
Treat the TypeScript source as authoritative. The canonical sequence is:
|
|
323
|
+
Treat the TypeScript source as authoritative. The canonical sequence is: run the complete local gate, validate against the platform, plan, apply, then plan again to confirm convergence.
|
|
296
324
|
|
|
297
325
|
```sh
|
|
298
|
-
# 1.
|
|
299
|
-
|
|
326
|
+
# 1. Run source, type, and typed test checks
|
|
327
|
+
toloka check
|
|
300
328
|
|
|
301
329
|
# 2. Validate each bound pipeline against the platform
|
|
302
330
|
toloka validate
|
|
@@ -311,7 +339,7 @@ toloka apply
|
|
|
311
339
|
toloka plan # expect: No changes.
|
|
312
340
|
```
|
|
313
341
|
|
|
314
|
-
`toloka validate` posts each authored graph to `POST /pipelines/:pipelineId/graph/validate
|
|
342
|
+
`toloka apply` reruns the complete local gate before resolving credentials. `toloka validate` posts each authored graph to `POST /pipelines/:pipelineId/graph/validate`; it reports platform spec validation and the latest matching Remote Node Check jobs without writing the graph. Resolve every unexpected plan change, missing secret, and immutable-field conflict before applying.
|
|
315
343
|
|
|
316
344
|
## Further reading
|
|
317
345
|
|
package/docs/getting-started.md
CHANGED
|
@@ -4,14 +4,7 @@ Install the SDK, sign in, and get a pipeline project reconciling from source in
|
|
|
4
4
|
|
|
5
5
|
## Install
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
```sh
|
|
10
|
-
cd packages/sdk && bun link
|
|
11
|
-
cd /path/to/your/project && bun link @toloka-platform/sdk
|
|
12
|
-
```
|
|
13
|
-
|
|
14
|
-
Declare the dependency in `package.json` so `toloka init` recognizes it. Once the hosted installer is deployed, install the bytecode-cached CLI and add the SDK to the project:
|
|
7
|
+
Install the bytecode-cached CLI globally and declare the Preview SDK in the project:
|
|
15
8
|
|
|
16
9
|
```sh
|
|
17
10
|
curl -fsSL https://platform.toloka.ai/cli/install.sh | sh
|
|
@@ -44,6 +37,8 @@ CI does not need `login`. Set `POTATO_API_KEY` and the stored file is ignored. S
|
|
|
44
37
|
|
|
45
38
|
## Start with an agent
|
|
46
39
|
|
|
40
|
+
For a guided first run, start a fresh coding-agent session and ask it to follow `https://platform.toloka.ai/onboarding.md`. The guide takes the agent through installation, credentials, project selection, local tests, plan approval, apply, cost review, and a bounded sample run.
|
|
41
|
+
|
|
47
42
|
Initialize the current directory and install the Agent Bootstrap. No platform project needs to be selected yet.
|
|
48
43
|
|
|
49
44
|
```sh
|
|
@@ -54,7 +49,7 @@ Open your coding agent in this directory and describe the project you want, for
|
|
|
54
49
|
|
|
55
50
|
> Create a Toloka project that collects captions for images. Show me the plan before applying it.
|
|
56
51
|
|
|
57
|
-
The bootstrap directs the agent to `toloka --help` and to search and install the matching bundled skill; the `toloka-build` skill then guides the agent through creating and
|
|
52
|
+
The bootstrap directs the agent to `toloka --help` and to search and install the matching bundled skill; the `toloka-build` skill then guides the agent through creating and binding the platform project, authoring the TypeScript source, validating it, showing the plan, and applying the approved changes. `project create`, `import`, or explicit project identity on `init` writes `toloka.json`.
|
|
58
53
|
|
|
59
54
|
## Manual project setup
|
|
60
55
|
|
|
@@ -65,8 +60,8 @@ You are either adopting a project that already exists on the platform, or creati
|
|
|
65
60
|
```sh
|
|
66
61
|
mkdir my-project && cd my-project
|
|
67
62
|
toloka project add cats pp.019ffa5f-29a7-74de-a76c-7ba5ceedf37f
|
|
63
|
+
toloka import pp.019ffa5f-29a7-74de-a76c-7ba5ceedf37f
|
|
68
64
|
toloka init
|
|
69
|
-
toloka import
|
|
70
65
|
```
|
|
71
66
|
|
|
72
67
|
`import` writes a `pipelines/` tree and a `toloka.json` that maps your source to live IDs. If `pipelines/` already exists, pass `--force` to replace it. Read the generated TypeScript: it is the same shape you will edit from now on.
|
|
@@ -90,10 +85,10 @@ Then create `pipelines/<name>/pipeline.ts` with a default export and run `plan`
|
|
|
90
85
|
|
|
91
86
|
## The loop you will live in
|
|
92
87
|
|
|
93
|
-
Every change follows the same steps. `validate`, `plan`, and `apply` talk to the platform.
|
|
88
|
+
Every change follows the same steps. `toloka check` is credential-free; `validate`, `plan`, and `apply` talk to the platform. Apply reruns the complete local gate before any platform request.
|
|
94
89
|
|
|
95
90
|
```sh
|
|
96
|
-
|
|
91
|
+
toloka check
|
|
97
92
|
toloka validate
|
|
98
93
|
toloka plan
|
|
99
94
|
toloka apply
|
package/docs/limitations.md
CHANGED
|
@@ -2,22 +2,22 @@
|
|
|
2
2
|
|
|
3
3
|
What this Preview SDK does not do yet, stated plainly so you can decide what to depend on.
|
|
4
4
|
|
|
5
|
-
`@toloka-platform/sdk` is `0.1.0-preview.
|
|
5
|
+
`@toloka-platform/sdk` is `0.1.0-preview.8`. This is a Preview: the authoring API, the CLI surface, and the generated source layout can change without a major version bump. Pin an exact version. Do not depend on this in production yet.
|
|
6
6
|
|
|
7
|
-
##
|
|
7
|
+
## Local checks do not certify deployability
|
|
8
8
|
|
|
9
|
-
`toloka
|
|
9
|
+
`toloka check` is the required credential-free local gate. It runs project TypeScript, automatic Task UI typechecking, Vitest, and typed node and pipeline-flow coverage. `toloka apply` reruns it before any platform write.
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
`toloka validate` separately posts each bound graph to the platform and reports spec errors, warnings, and the latest matching Remote Node Check jobs. It does not write the graph. A passing local gate and platform validation are still not proof that the pipeline can launch:
|
|
12
12
|
|
|
13
13
|
| gate | where it is enforced | what the CLI tells you |
|
|
14
14
|
| ------------------------------------------------- | -------------------- | ---------------------- |
|
|
15
15
|
| Training course content, when Training is enabled | platform validation | nothing |
|
|
16
16
|
| labeler UI validation receipt | platform validation | nothing |
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
The typed client can list, start, and poll Remote Node Checks. `toloka check` does not start them: it validates the authored source and its local tests. A `not_ran_node_check` result from `toloka validate` describes remote state, not missing local evidence.
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
Python code nodes have executable behavior checks but no useful static Python typecheck yet. Their generated `Inputs` and `Outputs` aliases are currently `dict[str, Any]`; adding Pyright or mypy before those aliases become field-derived `TypedDict` contracts would provide little protection.
|
|
21
21
|
|
|
22
22
|
## Layout is not authored
|
|
23
23
|
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# Pipeline testing examples
|
|
2
|
+
|
|
3
|
+
Import the ready-to-use Vitest helpers from `@toloka-platform/sdk/testing`. The same entry point executes authored Python code nodes and renders human-work Task UIs in an isolated Chromium context.
|
|
4
|
+
|
|
5
|
+
## Code node scenarios
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
9
|
+
import pipeline from "../../pipeline";
|
|
10
|
+
import { classify } from "./node";
|
|
11
|
+
|
|
12
|
+
const scenarios = [
|
|
13
|
+
{ text: "A refund was requested", expected: "billing" },
|
|
14
|
+
{ text: "The app crashes", expected: "technical" },
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
for (const scenario of scenarios) {
|
|
18
|
+
test.code(pipeline, classify, `classifies ${scenario.text}`, async ({ run }) => {
|
|
19
|
+
await expect(run({ text: scenario.text })).resolves.toEqual({ category: scenario.expected });
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
test.code(pipeline, classify, "surfaces authored failures", async ({ run }) => {
|
|
24
|
+
await expect(run({ text: "" })).rejects.toMatchObject({
|
|
25
|
+
name: "CodeNodeExecutionError",
|
|
26
|
+
exitCode: 1,
|
|
27
|
+
stderr: expect.stringContaining("text is required"),
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Pass test-only secret values through the bound `run(inputs, { env })`; authored static environment entries are applied automatically. Use `pythonExecutable` only when the project intentionally tests another Python binary, and lower `timeoutMs` for timeout behavior.
|
|
33
|
+
|
|
34
|
+
## Complete pipeline flow
|
|
35
|
+
|
|
36
|
+
Start with the first row from the same local dataset file intended for upload. Keep that row as a TypeScript fixture; if it contains sensitive values, redact the values while preserving the real field names, nesting, optionality, and media/value shapes. Use synthetic input only when no local row exists.
|
|
37
|
+
|
|
38
|
+
```ts
|
|
39
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
40
|
+
import pipeline from "../pipeline";
|
|
41
|
+
import { representativeTicket } from "./fixtures/support-ticket";
|
|
42
|
+
import { normalizeTicket } from "../nodes/normalize-ticket/node";
|
|
43
|
+
import { classifyTicket } from "../nodes/classify-ticket/node";
|
|
44
|
+
import { finalizeTicket } from "../nodes/finalize-ticket/node";
|
|
45
|
+
|
|
46
|
+
test.pipeline(
|
|
47
|
+
pipeline,
|
|
48
|
+
"moves a representative item from root.started to root.completed",
|
|
49
|
+
async ({ runCode, renderUi }) => {
|
|
50
|
+
let item: Record<string, unknown> = { ...representativeTicket };
|
|
51
|
+
|
|
52
|
+
item = { ...item, ...(await runCode(normalizeTicket, item)) };
|
|
53
|
+
|
|
54
|
+
const classificationTask = await renderUi(classifyTicket, { data: item });
|
|
55
|
+
await classificationTask.setValue("category", "billing");
|
|
56
|
+
await classificationTask.setValue("rationale", "The customer disputes a duplicate charge.");
|
|
57
|
+
const classification = await classificationTask.submit();
|
|
58
|
+
item = {
|
|
59
|
+
...item,
|
|
60
|
+
category: classification.category,
|
|
61
|
+
rationale: classification.rationale,
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
item = { ...item, ...(await runCode(finalizeTicket, item)) };
|
|
65
|
+
|
|
66
|
+
expect(item).toMatchObject({
|
|
67
|
+
ticket_id: representativeTicket.ticket_id,
|
|
68
|
+
category: "billing",
|
|
69
|
+
resolution_status: "ready",
|
|
70
|
+
});
|
|
71
|
+
},
|
|
72
|
+
);
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
This explicit item-state threading is the local equivalent of following pipeline connections: pass root input to the first node, merge its output, and continue until the completed fields are available. Keep this happy path in one test when the graph is linear. Use separate tests for each match branch or materially different path. For quorum, collect several simulated Task UI submissions and assemble each outer annotation array before calling a downstream code node.
|
|
76
|
+
|
|
77
|
+
## Default Task UI fixture
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
81
|
+
import pipeline from "../../pipeline";
|
|
82
|
+
import { annotate } from "./node";
|
|
83
|
+
|
|
84
|
+
test.ui(pipeline, annotate, "submits a valid answer", async ({ render }) => {
|
|
85
|
+
const task = await render({
|
|
86
|
+
data: { prompt: "What is two plus two?" },
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
await task.setValue("raw_answer", "four");
|
|
90
|
+
|
|
91
|
+
expect(await task.validationErrors()).toEqual([]);
|
|
92
|
+
expect(await task.submit()).toMatchObject({ answer: "four" });
|
|
93
|
+
});
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
The browser is created once per Vitest worker. The typed wrapper's `render` helper creates an isolated browser context and page for the scenario. Every opened Task UI session is closed automatically after the scenario, including failed scenarios.
|
|
97
|
+
|
|
98
|
+
## Several TypeScript scenarios
|
|
99
|
+
|
|
100
|
+
Scenarios are ordinary TypeScript. Use shared builders and loops instead of generating a separate JSON scenario format.
|
|
101
|
+
|
|
102
|
+
```ts
|
|
103
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
104
|
+
import pipeline from "../../pipeline";
|
|
105
|
+
import { classify } from "./node";
|
|
106
|
+
|
|
107
|
+
const scenarios = [
|
|
108
|
+
{ name: "short text", data: { text: "yes" } },
|
|
109
|
+
{ name: "long text", data: { text: "a long paragraph" } },
|
|
110
|
+
{ name: "empty optional context", data: { text: "yes", context: null } },
|
|
111
|
+
];
|
|
112
|
+
|
|
113
|
+
for (const scenario of scenarios) {
|
|
114
|
+
test.ui(pipeline, classify, scenario.name, async ({ render }) => {
|
|
115
|
+
const task = await render({ data: scenario.data });
|
|
116
|
+
|
|
117
|
+
await task.setValue("label", "accepted");
|
|
118
|
+
|
|
119
|
+
expect(await task.validationErrors()).toEqual([]);
|
|
120
|
+
expect(await task.submit()).toMatchObject({ label: "accepted" });
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Each scenario receives a fresh browser context, so values set by one scenario cannot satisfy another scenario.
|
|
126
|
+
|
|
127
|
+
## Inspect component contracts
|
|
128
|
+
|
|
129
|
+
Component documentation comes from the same TBX launcher used to render the interface.
|
|
130
|
+
|
|
131
|
+
```ts
|
|
132
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
133
|
+
import pipeline from "../../pipeline";
|
|
134
|
+
import { classify } from "./node";
|
|
135
|
+
|
|
136
|
+
test("uses the expected Field contract", async ({ ui }) => {
|
|
137
|
+
const task = await ui.render(pipeline, classify);
|
|
138
|
+
const documentation = await task.componentDocumentation(["Field.RadioGroup", "View.Card"]);
|
|
139
|
+
|
|
140
|
+
expect(documentation).toMatchObject({
|
|
141
|
+
"Field.RadioGroup": { valueType: expect.stringContaining("string") },
|
|
142
|
+
"View.Card": { propsType: expect.stringContaining("title") },
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Requesting an unknown component rejects with an actionable error. It does not silently return an empty entry.
|
|
148
|
+
|
|
149
|
+
```ts
|
|
150
|
+
await expect(task.componentDocumentation(["Field.Invented"])).rejects.toThrow(
|
|
151
|
+
"Unknown Task UI components: Field.Invented",
|
|
152
|
+
);
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
## Assert browser validation failures
|
|
156
|
+
|
|
157
|
+
```ts
|
|
158
|
+
import { expect, TaskUiValidationError, test } from "@toloka-platform/sdk/testing";
|
|
159
|
+
import pipeline from "../../pipeline";
|
|
160
|
+
import { annotate } from "./node";
|
|
161
|
+
|
|
162
|
+
test("requires an answer", async ({ ui }) => {
|
|
163
|
+
const task = await ui.render(pipeline, annotate);
|
|
164
|
+
|
|
165
|
+
expect(await task.validationErrors()).toEqual([expect.objectContaining({ path: "raw_answer" })]);
|
|
166
|
+
|
|
167
|
+
await expect(task.submit()).rejects.toBeInstanceOf(TaskUiValidationError);
|
|
168
|
+
});
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
`TaskUiValidationError` exposes three structured fields:
|
|
172
|
+
|
|
173
|
+
- `formErrors` contains browser component validation failures.
|
|
174
|
+
- `entityErrors` contains postprocessed output values that fail entity schemas.
|
|
175
|
+
- `coverage` reports untested hardware-backed fields.
|
|
176
|
+
|
|
177
|
+
## Assert postprocessed output
|
|
178
|
+
|
|
179
|
+
`submit()` returns the final Task UI output after postprocess runs. Launcher-only `_context` metadata is removed.
|
|
180
|
+
|
|
181
|
+
```ts
|
|
182
|
+
const task = await ui.render(pipeline, normalizeBox);
|
|
183
|
+
|
|
184
|
+
await task.setValue("raw_box", {
|
|
185
|
+
x: 20,
|
|
186
|
+
y: 10,
|
|
187
|
+
width: 100,
|
|
188
|
+
height: 50,
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
expect(await task.submit()).toEqual({
|
|
192
|
+
raw_box: { x: 20, y: 10, width: 100, height: 50 },
|
|
193
|
+
box: { x: 0.2, y: 0.1, width: 1, height: 0.5 },
|
|
194
|
+
});
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
The submitted result is also checked against every non-hardware output entity schema declared by the authored node.
|
|
198
|
+
|
|
199
|
+
## Hardware-backed fields
|
|
200
|
+
|
|
201
|
+
Headless Chromium cannot complete native file pickers, cameras, microphones, or recording components. By default, `submit()` rejects rather than claiming complete coverage.
|
|
202
|
+
|
|
203
|
+
```ts
|
|
204
|
+
const task = await ui.render(pipeline, recordVideo);
|
|
205
|
+
|
|
206
|
+
expect(task.coverage.complete).toBe(false);
|
|
207
|
+
expect(task.coverage.untestedHardwareFields).toContain("video");
|
|
208
|
+
await expect(task.submit()).rejects.toThrow("hardware-backed fields");
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
When the scenario deliberately verifies only the testable fields, acknowledge partial coverage explicitly:
|
|
212
|
+
|
|
213
|
+
```ts
|
|
214
|
+
const task = await ui.render(pipeline, recordVideo, {
|
|
215
|
+
allowPartialCoverage: true,
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
await task.setValue("description", "A short outdoor video");
|
|
219
|
+
expect(await task.submit()).toMatchObject({
|
|
220
|
+
description: "A short outdoor video",
|
|
221
|
+
});
|
|
222
|
+
expect(task.coverage.complete).toBe(false);
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
A partial scenario is useful for structure and supporting fields, but it is not full verification of the hardware workflow.
|
|
226
|
+
|
|
227
|
+
## Manual cleanup
|
|
228
|
+
|
|
229
|
+
Automatic teardown is sufficient for normal tests. Call `close()` only when a long scenario no longer needs a page and should release it early.
|
|
230
|
+
|
|
231
|
+
```ts
|
|
232
|
+
const task = await ui.render(pipeline, annotate);
|
|
233
|
+
|
|
234
|
+
await task.screenshot("test-results/initial.png");
|
|
235
|
+
await task.close();
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
`close()` is idempotent. The fixture’s automatic teardown can safely call it again.
|
|
239
|
+
|
|
240
|
+
## Setup and execution
|
|
241
|
+
|
|
242
|
+
```sh
|
|
243
|
+
bun add --dev vitest playwright
|
|
244
|
+
bunx playwright install chromium
|
|
245
|
+
toloka check
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
The local browser still downloads the hosted launcher, so use synthetic or redacted test data. `toloka check` runs project types, automatic Task UI types, typed node tests, and pipeline-flow evidence. Remote Node Checks remain separate platform jobs.
|