@toloka-platform/sdk 0.1.0-preview.5 → 0.1.0-preview.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -11
- package/dist/cli/toloka.js +211 -125
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +17 -4
- package/docs/cli-v2.md +3 -3
- package/docs/cli.md +55 -17
- package/docs/datasets.md +8 -9
- package/docs/getting-started.md +7 -12
- package/docs/limitations.md +10 -13
- package/docs/testing-examples.md +248 -0
- package/docs/testing.md +115 -0
- package/docs/troubleshooting.md +0 -11
- package/examples/face-image-collection/README.md +7 -1
- package/examples/face-image-collection/fields.ts +20 -8
- package/examples/face-image-collection/nodes/annotate-faces/node.ts +1 -9
- package/examples/face-image-collection/nodes/order-faces/node.ts +1 -1
- package/examples/face-image-collection/pipeline.browser.integration.spec.ts +78 -0
- package/examples/video-pair-rating/README.md +14 -6
- package/examples/video-pair-rating/nodes/adjudicate/node.ts +1 -3
- package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +1 -3
- package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +1 -1
- package/examples/video-pair-rating/nodes/rate_pair/node.ts +1 -1
- package/examples/video-pair-rating/pipeline.browser.integration.spec.ts +267 -0
- package/openapi/gen/api.ts +819 -898
- package/openapi/gen/models/baseModel.ts +9 -0
- package/openapi/gen/models/baseModelId.ts +2 -0
- package/openapi/gen/models/baseModelIdProperty.ts +2 -0
- package/openapi/gen/models/createDatasetExportBody.ts +3 -3
- package/openapi/gen/models/createDatasetExportBodyOneOf.ts +6 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfFormat.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThree.ts +10 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeContract.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeFormat.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilter.ts +9 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/createProjectModelBody.ts +8 -0
- package/openapi/gen/models/createProjectModelBodyDescription.ts +2 -0
- package/openapi/gen/models/datasetExportReport.ts +9 -0
- package/openapi/gen/models/datasetExportReportErrorsItem.ts +5 -0
- package/openapi/gen/models/datasetExportReportTokens.ts +7 -0
- package/openapi/gen/models/exportJob.ts +2 -0
- package/openapi/gen/models/exportJobReport.ts +3 -0
- package/openapi/gen/models/generationNodeSettingsAudienceAndPricingOneOfOneone.ts +1 -0
- package/openapi/gen/models/index.ts +64 -0
- package/openapi/gen/models/listBaseModels200.ts +5 -0
- package/openapi/gen/models/listModelVersions200.ts +7 -0
- package/openapi/gen/models/listModelVersionsParams.ts +5 -0
- package/openapi/gen/models/listProjectModels200.ts +7 -0
- package/openapi/gen/models/listProjectModelsParams.ts +5 -0
- package/openapi/gen/models/model.ts +16 -0
- package/openapi/gen/models/modelArchivedAt.ts +2 -0
- package/openapi/gen/models/modelDescription.ts +2 -0
- package/openapi/gen/models/modelId.ts +2 -0
- package/openapi/gen/models/modelVersion.ts +9 -0
- package/openapi/gen/models/modelVersionId.ts +2 -0
- package/openapi/gen/models/pipelineGraphSnapshotGraphNodesItemOneOfThreeoneSettingsAudienceAndPricingOneOfOneone.ts +1 -0
- package/openapi/gen/models/sampleDatasetItemsBody.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOf.ts +8 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilter.ts +9 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethree.ts +8 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilter.ts +9 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/sampleJob.ts +13 -0
- package/openapi/gen/models/sampleJobStatus.ts +2 -0
- package/openapi/gen/models/sampleResult.ts +5 -0
- package/openapi/gen/models/validatePipelineGraphBodyGraphNodesItemOneOfThreeoneSettingsAudienceAndPricingOneOfOneone.ts +1 -0
- package/package.json +23 -2
- package/skills/catalog.json +14 -6
- package/skills/toloka-build/SKILL.md +34 -12
- package/skills/toloka-build/references/node-semantics.md +27 -24
- package/skills/toloka-build/references/workflow-and-cli.md +67 -27
- package/skills/toloka-report-issue/references/report-template.md +1 -1
- package/src/checks/ast.ts +32 -0
- package/src/checks/index.ts +14 -0
- package/src/checks/postprocess-collisions.ts +145 -0
- package/src/checks/task-ui-declarations.generated.ts +19 -0
- package/src/checks/task-ui-typecheck.ts +345 -0
- package/src/checks/typescript-vfs.d.ts +26 -0
- package/src/checks/typescript-vfs.js +5 -0
- package/src/cli/analytics.ts +1 -0
- package/src/cli/check.ts +977 -0
- package/src/cli/data.ts +148 -25
- package/src/cli/models.ts +188 -0
- package/src/cli/program.ts +246 -20
- package/src/cli/project-context.ts +14 -4
- package/src/cli/runs.ts +21 -11
- package/src/cli/skill.ts +17 -6
- package/src/client.ts +109 -35
- package/src/errors.ts +25 -10
- package/src/testing/code-node.ts +160 -0
- package/src/testing/index.ts +15 -0
- package/src/testing/task-ui-browser.ts +120 -0
- package/src/testing/task-ui-validation.ts +135 -0
- package/src/testing/task-ui.ts +557 -0
- package/src/testing/test-evidence.ts +170 -0
- package/src/testing/types.ts +141 -0
- package/src/transport.ts +16 -13
package/docs/testing.md
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# Test pipelines locally
|
|
2
|
+
|
|
3
|
+
The `@toloka-platform/sdk/testing` entry point provides typed Vitest wrappers for executable Python `code` nodes, Chromium Task UI scenarios, and complete pipeline flows. `toloka check` runs those tests together with project TypeScript and automatic Task UI typechecking without creating or modifying a platform project.
|
|
4
|
+
|
|
5
|
+
## Install test dependencies
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
bun add --dev vitest playwright
|
|
9
|
+
bunx playwright install chromium
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Code-node tests also require `python3` on the test machine. See [testing examples](./testing-examples.md) for parameterization, validation failures, hardware coverage, and cleanup patterns.
|
|
13
|
+
|
|
14
|
+
## Test a code node
|
|
15
|
+
|
|
16
|
+
`test.code(pipeline, node, name, callback)` binds one test to one authored code node. Its `run(inputs, options?)` helper executes the Python source with the same async `main(inputs)` entry contract. The test counts as node coverage only after `run` executes, at least one assertion runs, and the callback passes.
|
|
17
|
+
|
|
18
|
+
```ts
|
|
19
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
20
|
+
import pipeline from "../../pipeline";
|
|
21
|
+
import { normalize } from "./node";
|
|
22
|
+
|
|
23
|
+
test.code(pipeline, normalize, "normalizes whitespace", async ({ run }) => {
|
|
24
|
+
await expect(run({ text: " hello " })).resolves.toEqual({
|
|
25
|
+
normalized_text: "hello",
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Failures from the bound executor throw `CodeNodeExecutionError` with the Python exit code and stderr. Authored static environment values are included; pass secret or test-specific values through the optional `run` options. Third-party imports must be available to the selected Python executable.
|
|
31
|
+
|
|
32
|
+
`test.code` defaults to 30 seconds, which exceeds the code runner's 10-second default. Pass `{ timeout: milliseconds }` after the callback when the scenario also raises the runner timeout.
|
|
33
|
+
|
|
34
|
+
## Test a complete pipeline flow
|
|
35
|
+
|
|
36
|
+
Register the representative connected scenario with `test.pipeline(pipeline, name, callback)`. Its callback provides `runCode(node, inputs, options?)` and `renderUi(node, options?)` bound to that pipeline.
|
|
37
|
+
|
|
38
|
+
Use the first row from the same local dataset file intended for upload when one is available. If it contains sensitive values, keep a redacted fixture that preserves field names, nesting, optionality, and media/value shapes. Use synthetic input only when no local row exists. `test.pipeline` does not fetch a live dataset: the local gate remains deterministic, credential-free, and usable before apply. Treat the fixture as `root.started` item state, execute connected nodes in order, merge only each node's declared outputs, and assert the fields accepted by `root.completed`. A linear happy path can stay in one test; use separate scenarios for materially different match branches, optional paths, and failures.
|
|
39
|
+
|
|
40
|
+
## Test a Task UI
|
|
41
|
+
|
|
42
|
+
Create a file ending in `.ui.spec.ts` next to the human-work node. `test.ui` binds the test to the pipeline and node, and its callback receives `render(options?)` for that target. The test counts as coverage only after a successful submission, at least one assertion, and a passing callback.
|
|
43
|
+
|
|
44
|
+
Browser-backed `test.ui` scenarios default to 60 seconds, which exceeds the launcher's 45-second setup budget. `test.pipeline` scenarios default to 120 seconds for connected flows with multiple UI steps. Pass `{ timeout: milliseconds }` after the callback to tune either wrapper without weakening its inner launcher deadlines.
|
|
45
|
+
|
|
46
|
+
```ts
|
|
47
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
48
|
+
import pipeline from "../../pipeline";
|
|
49
|
+
import { annotate } from "./node";
|
|
50
|
+
|
|
51
|
+
test.ui(pipeline, annotate, "submits a valid answer", async ({ render }) => {
|
|
52
|
+
const task = await render({
|
|
53
|
+
data: { prompt: "What is two plus two?" },
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
await task.setValue("raw_answer", "four");
|
|
57
|
+
|
|
58
|
+
expect(await task.validationErrors()).toEqual([]);
|
|
59
|
+
expect(await task.submit()).toMatchObject({
|
|
60
|
+
answer: "four",
|
|
61
|
+
});
|
|
62
|
+
await task.screenshot("test-results/answer.png");
|
|
63
|
+
});
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Run the complete gate rather than invoking Vitest separately:
|
|
67
|
+
|
|
68
|
+
```sh
|
|
69
|
+
toloka check
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Use an ordinary TypeScript loop to register several `test.code` or `test.ui` scenarios. There is no separate JSON scenario format.
|
|
73
|
+
|
|
74
|
+
## Fixture API
|
|
75
|
+
|
|
76
|
+
The low-level `ui` fixture remains available inside ordinary Vitest tests for component discovery and exploratory work, but plain tests do not satisfy required node coverage. Use typed wrappers for committed gate evidence. Importing `@toloka-platform/sdk/testing` from a plain Bun script executes Vitest fixture setup and fails.
|
|
77
|
+
|
|
78
|
+
- `ui.componentDocumentation(names?)` returns the current launcher's supported component contracts. Call it before choosing component names or props.
|
|
79
|
+
- `ui.render(pipeline, node, options)` creates a fresh browser context and returns a Task UI session for exactly that authored node. `test.ui` supplies the equivalent bound `render(options?)` helper.
|
|
80
|
+
|
|
81
|
+
The returned Task UI session supports:
|
|
82
|
+
|
|
83
|
+
- `componentTree()` returns the rendered component hierarchy.
|
|
84
|
+
- `componentDocumentation(names?)` returns component props, value shapes, capabilities, and limitations.
|
|
85
|
+
- `setValue(path, value)` and `getValue(path)` address Fields by their stable `path`, not generated component ids.
|
|
86
|
+
- `validationErrors()` returns browser-level required-field and component-rule failures with Field paths.
|
|
87
|
+
- `submit()` executes preprocess/postprocess, removes the launcher's `_context`, and validates declared output entities. The returned form data can also contain echoed inputs; assert output-entity keys and values without treating those inputs as output entities.
|
|
88
|
+
- `screenshot(path)` writes a full-page PNG to the local filesystem.
|
|
89
|
+
- `coverage` states whether hardware-backed fields were exercised completely.
|
|
90
|
+
|
|
91
|
+
One Chromium process is shared per Vitest worker. Every scenario gets a fresh context and page and is cleaned up automatically.
|
|
92
|
+
|
|
93
|
+
For a structured or array-valued Field, derive the dataset field schema from the component's documented `valueType`. In its typed UI scenario, adapt only semantic values such as labels, then pass the documented example value through `setValue`, `getValue`, and `submit`. This catches component-shape drift before a postprocess can read absent keys and silently write fallback data.
|
|
94
|
+
|
|
95
|
+
## Runtime rules
|
|
96
|
+
|
|
97
|
+
`ui.code` must parse as one JSX root. Every identifier must be a component returned by `componentDocumentation()` or a documented `actions` API. `toloka check` applies full TypeScript semantic checking with declarations bundled in the SDK; the real launcher then renders the same JSX in browser scenarios. Treat a component's `propsType` and `example` together as its contract.
|
|
98
|
+
|
|
99
|
+
`preprocess` and `postprocess` are JavaScript function bodies, not TypeScript:
|
|
100
|
+
|
|
101
|
+
Valid JavaScript: `items.map((item) => item.value)`
|
|
102
|
+
|
|
103
|
+
Invalid runtime JavaScript: `items.map((item: unknown) => item.value)`
|
|
104
|
+
|
|
105
|
+
`preprocess` can run again after form state changes. Use it to shape input data; if it prefills a bound field, return that key only while it is absent so later edits are not overwritten. A `Field`-bound top-level key already enters the submission. `postprocess` must return only new top-level keys: bind a component to a draft key when postprocess must derive a different entity key.
|
|
106
|
+
|
|
107
|
+
The fixture loads the hosted TBX launcher in a fresh local Chromium context, so tests require network access but do not use Azure Playwright or Potato application services. The context has no platform cookies or credentials. Use synthetic or redacted fixtures because the hosted launcher executes with access to the supplied test data.
|
|
108
|
+
|
|
109
|
+
## Hardware-backed fields
|
|
110
|
+
|
|
111
|
+
File pickers, video upload/recording, and other hardware-backed fields cannot be completed reliably in headless Chromium. `task.coverage.complete` is `false` when these components are present. Render with `allowPartialCoverage: true`, inject one value matching the component's documented `valueType` through `setValue`, and assert the postprocessed output shape yourself: entity validation skips hardware-backed paths, and `getValue` may not serialize their launcher state. This checks browser validation and mapping only, not capture, upload, link generation, or record-time constraints. A partial result must not be described as complete coverage.
|
|
112
|
+
|
|
113
|
+
## Final gate
|
|
114
|
+
|
|
115
|
+
`toloka check` loads every authored pipeline, runs the project TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and requires passing `test.code`, `test.ui`, and `test.pipeline` evidence. `missing`, `failed`, `partial`, and `infrastructure_error` are distinct blocking outcomes. `toloka apply` reruns this complete gate before resolving credentials or writing platform state. Platform source/spec errors from `toloka validate` remain separate; `not_ran_node_check` describes a Remote Node Check and does not replace local evidence.
|
package/docs/troubleshooting.md
CHANGED
|
@@ -51,17 +51,6 @@ A resource exists on the platform but not in your source tree. `apply` reports i
|
|
|
51
51
|
|
|
52
52
|
Another writer changed the graph after your source was read. Do not retry blindly: `plan` again to re-read live state, review the new diff, carry any intentional remote change into source, then `apply`.
|
|
53
53
|
|
|
54
|
-
## Second upload of the same filename fails
|
|
55
|
-
|
|
56
|
-
Uploading a file whose basename already exists in that dataset fails with a bodyless `500`. `toloka dataset upload` sends the file's basename, so running the same documented command twice reproduces it:
|
|
57
|
-
|
|
58
|
-
```sh
|
|
59
|
-
toloka dataset upload cat-images ./data/rows.jsonl # first time: fine
|
|
60
|
-
toloka dataset upload cat-images ./data/rows.jsonl # second time: 500
|
|
61
|
-
```
|
|
62
|
-
|
|
63
|
-
Until the platform returns a proper conflict, use a distinct filename per upload — for example `rows-2026-08-13.jsonl`. This is a platform defect, tracked; the SDK cannot detect it in advance.
|
|
64
|
-
|
|
65
54
|
## Another upload is already in progress
|
|
66
55
|
|
|
67
56
|
```
|
|
@@ -17,7 +17,13 @@ toloka import pp.0190abcd-abcd-7abc-8abc-abcdefabcdef
|
|
|
17
17
|
# Imported project pp.0190abcd-abcd-7abc-8abc-abcdefabcdef with 1 pipeline and 1 dataset into ...
|
|
18
18
|
```
|
|
19
19
|
|
|
20
|
-
Edit a node under `pipelines/face-image-collection/nodes/`, then
|
|
20
|
+
Edit a node under `pipelines/face-image-collection/nodes/`, then run the shipped typed code, UI, and connected-flow scenarios before inspecting any platform changes:
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
toloka check face-image-collection
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
After the local gate passes, inspect and apply the project-wide reconciliation:
|
|
21
27
|
|
|
22
28
|
```sh
|
|
23
29
|
toloka plan
|
|
@@ -5,19 +5,31 @@ const faceAnnotationsSchema: Parameters<typeof field>[1] = {
|
|
|
5
5
|
items: {
|
|
6
6
|
type: "object",
|
|
7
7
|
properties: {
|
|
8
|
-
|
|
8
|
+
id: { type: "string" },
|
|
9
|
+
type: { type: "string" },
|
|
10
|
+
geometry: {
|
|
9
11
|
type: "object",
|
|
10
12
|
properties: {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
bounds: {
|
|
14
|
+
type: "object",
|
|
15
|
+
properties: {
|
|
16
|
+
minX: { type: "number" },
|
|
17
|
+
minY: { type: "number" },
|
|
18
|
+
maxX: { type: "number" },
|
|
19
|
+
maxY: { type: "number" },
|
|
20
|
+
},
|
|
21
|
+
required: ["minX", "minY", "maxX", "maxY"],
|
|
22
|
+
},
|
|
15
23
|
},
|
|
16
|
-
required: ["
|
|
24
|
+
required: ["bounds"],
|
|
25
|
+
},
|
|
26
|
+
properties: {
|
|
27
|
+
type: "object",
|
|
28
|
+
properties: { value: { type: "string", const: "face" } },
|
|
29
|
+
required: ["value"],
|
|
17
30
|
},
|
|
18
|
-
label: { type: "string", const: "face" },
|
|
19
31
|
},
|
|
20
|
-
required: ["
|
|
32
|
+
required: ["id", "type", "geometry", "properties"],
|
|
21
33
|
},
|
|
22
34
|
};
|
|
23
35
|
|
|
@@ -55,7 +55,7 @@ export const annotateFaces = generation("annotate_faces", {
|
|
|
55
55
|
instruction:
|
|
56
56
|
"Draw one tight rectangular bounding box around every visible human face. Do not annotate faces shown only in photographs, posters, or reflections.",
|
|
57
57
|
ui: {
|
|
58
|
-
code: "<Field.ImageAnnotation path={'draft_faces'} image={actions.get('image_url'
|
|
58
|
+
code: "<Field.ImageAnnotation path={'draft_faces'} image={actions.get('image_url')} tools={['rectangle']} labels={[{ label: 'Face', value: 'face', color: 'blue' }]} navigator={true} />",
|
|
59
59
|
data: '{"image_url":"https://example.com/face.jpg"}',
|
|
60
60
|
preprocess: "return {}",
|
|
61
61
|
postprocess: "return {};",
|
|
@@ -131,12 +131,4 @@ export const annotateFaces = generation("annotate_faces", {
|
|
|
131
131
|
successCriteria: "Every face has one tight bounding box",
|
|
132
132
|
},
|
|
133
133
|
},
|
|
134
|
-
exam: {
|
|
135
|
-
enabled: false,
|
|
136
|
-
gradedEntities: [],
|
|
137
|
-
tasksPerExpert: 1,
|
|
138
|
-
passThreshold: 1,
|
|
139
|
-
},
|
|
140
|
-
reuseAudienceWithNodeId: null,
|
|
141
|
-
smartSearch: null,
|
|
142
134
|
});
|
|
@@ -19,7 +19,7 @@ export const orderFaces = code("order_faces", {
|
|
|
19
19
|
async def main(inputs: Inputs) -> Outputs:
|
|
20
20
|
ordered_faces = sorted(
|
|
21
21
|
inputs["draft_faces"],
|
|
22
|
-
key=lambda face: (face["
|
|
22
|
+
key=lambda face: (face["geometry"]["bounds"]["minY"], face["geometry"]["bounds"]["minX"]),
|
|
23
23
|
)
|
|
24
24
|
return {"faces": ordered_faces}`,
|
|
25
25
|
},
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
2
|
+
import { describe } from "vitest";
|
|
3
|
+
import { assert } from "es-toolkit";
|
|
4
|
+
import type { TaskUiSession } from "@toloka-platform/sdk/testing";
|
|
5
|
+
|
|
6
|
+
import pipeline from "./pipeline";
|
|
7
|
+
import { annotateFaces } from "./nodes/annotate-faces/node";
|
|
8
|
+
import { orderFaces } from "./nodes/order-faces/node";
|
|
9
|
+
const upperFace = {
|
|
10
|
+
id: "upper-face",
|
|
11
|
+
type: "RECTANGLE",
|
|
12
|
+
geometry: { x: 40, y: 10, w: 20, h: 20, bounds: { minX: 40, minY: 10, maxX: 60, maxY: 30 } },
|
|
13
|
+
properties: { value: "face" },
|
|
14
|
+
};
|
|
15
|
+
const lowerFace = {
|
|
16
|
+
id: "lower-face",
|
|
17
|
+
type: "RECTANGLE",
|
|
18
|
+
geometry: { x: 10, y: 60, w: 30, h: 30, bounds: { minX: 10, minY: 60, maxX: 40, maxY: 90 } },
|
|
19
|
+
properties: { value: "face" },
|
|
20
|
+
};
|
|
21
|
+
const imageUrl =
|
|
22
|
+
"https://tlkfrontprod.azureedge.net/frontend-testing/static/gzarip/cats/cat_001.jpg";
|
|
23
|
+
|
|
24
|
+
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
25
|
+
typeof value === "object" && value !== null && !Array.isArray(value);
|
|
26
|
+
|
|
27
|
+
const documentedFaceAnnotations = async (
|
|
28
|
+
session: TaskUiSession,
|
|
29
|
+
): Promise<Record<string, unknown>[]> => {
|
|
30
|
+
const documentation = await session.componentDocumentation(["Field.ImageAnnotation"]);
|
|
31
|
+
const output = documentation["Field.ImageAnnotation"]?.example?.output;
|
|
32
|
+
assert(output, "Field.ImageAnnotation has no documented example output");
|
|
33
|
+
const example: unknown = JSON.parse(output);
|
|
34
|
+
assert(isRecord(example), "Field.ImageAnnotation example output is not an object");
|
|
35
|
+
const annotations = example.roi;
|
|
36
|
+
assert(Array.isArray(annotations), "Field.ImageAnnotation example output has no roi array");
|
|
37
|
+
return annotations.map((annotation) => {
|
|
38
|
+
assert(isRecord(annotation), "Field.ImageAnnotation example contains a non-object value");
|
|
39
|
+
assert(
|
|
40
|
+
isRecord(annotation.properties),
|
|
41
|
+
"Field.ImageAnnotation example annotation has no properties",
|
|
42
|
+
);
|
|
43
|
+
return {
|
|
44
|
+
...annotation,
|
|
45
|
+
properties: { ...annotation.properties, value: "face" },
|
|
46
|
+
};
|
|
47
|
+
});
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
describe("face image collection example", () => {
|
|
51
|
+
test.code(pipeline, orderFaces, "orders faces from top to bottom", async ({ run }) => {
|
|
52
|
+
await expect(run({ draft_faces: [lowerFace, upperFace] })).resolves.toEqual({
|
|
53
|
+
faces: [upperFace, lowerFace],
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test.ui(pipeline, annotateFaces, "submits face annotations", async ({ render }) => {
|
|
58
|
+
const session = await render({ data: { image_url: imageUrl, draft_faces: [] } });
|
|
59
|
+
const annotations = await documentedFaceAnnotations(session);
|
|
60
|
+
await session.setValue("draft_faces", annotations);
|
|
61
|
+
|
|
62
|
+
expect(await session.getValue("draft_faces")).toEqual(annotations);
|
|
63
|
+
await expect(session.submit()).resolves.toMatchObject({ draft_faces: annotations });
|
|
64
|
+
await session.close();
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test.pipeline(pipeline, "collects and orders face annotations", async ({ renderUi, runCode }) => {
|
|
68
|
+
const session = await renderUi(annotateFaces, {
|
|
69
|
+
data: { image_url: imageUrl, draft_faces: [] },
|
|
70
|
+
});
|
|
71
|
+
await session.setValue("draft_faces", [lowerFace, upperFace]);
|
|
72
|
+
const annotation = await session.submit();
|
|
73
|
+
const result = await runCode(orderFaces, { draft_faces: annotation.draft_faces });
|
|
74
|
+
|
|
75
|
+
expect(result).toEqual({ faces: [upperFace, lowerFace] });
|
|
76
|
+
await session.close();
|
|
77
|
+
});
|
|
78
|
+
});
|
|
@@ -33,12 +33,12 @@ Sixteen nodes: one root, eight `code`, three `match`, two `generation`, two `quo
|
|
|
33
33
|
|
|
34
34
|
## What to read first
|
|
35
35
|
|
|
36
|
-
| file
|
|
37
|
-
|
|
38
|
-
| `pipeline.ts`
|
|
39
|
-
| `nodes/cascade/node.ts`
|
|
40
|
-
| `nodes/rate_pair/node.ts`
|
|
41
|
-
| `nodes/route_overall/node.ts` | predicate routing, and what the `default` branch is for
|
|
36
|
+
| file | why |
|
|
37
|
+
| ----------------------------- | ------------------------------------------------------------------- |
|
|
38
|
+
| `pipeline.ts` | the whole topology on one screen — every edge is explicit |
|
|
39
|
+
| `nodes/cascade/node.ts` | the deterministic core that resolves every metric from rater arrays |
|
|
40
|
+
| `nodes/rate_pair/node.ts` | a quorum node with real quality criteria and a Training course |
|
|
41
|
+
| `nodes/route_overall/node.ts` | predicate routing, and what the `default` branch is for |
|
|
42
42
|
|
|
43
43
|
## Conventions this example demonstrates
|
|
44
44
|
|
|
@@ -52,6 +52,14 @@ Sixteen nodes: one root, eight `code`, three `match`, two `generation`, two `quo
|
|
|
52
52
|
|
|
53
53
|
**Adjudication is a branch, not a loop.** The graph stays acyclic. Redo, appeal, and payment retry live inside the human-work nodes; only the tie-breaking hand-off is a graph edge.
|
|
54
54
|
|
|
55
|
+
## Check locally
|
|
56
|
+
|
|
57
|
+
The shipped typed scenarios cover all eight code nodes, all four Task UIs, and an agreed-pair connected flow:
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
toloka check video-pair-rating
|
|
61
|
+
```
|
|
62
|
+
|
|
55
63
|
## Provenance
|
|
56
64
|
|
|
57
65
|
The topology, the resolution cascade, and the per-node layout are adapted from a design spike by @hexode. Media URLs, sample payloads, prompts, and prose here are synthetic: no customer data, project identifiers, or asset URLs from the original are carried over.
|
|
@@ -64,7 +64,7 @@ export const adjudicate = generation("adjudicate", {
|
|
|
64
64
|
},
|
|
65
65
|
],
|
|
66
66
|
ui: {
|
|
67
|
-
code: "<View.
|
|
67
|
+
code: "<View.Card>{actions.get('disagreed_metrics', []).includes('image_text_alignment') && <Field.RadioGroup path={'adjudicated_metrics.image_text_alignment'} label={'Imagery matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('motion_text_alignment') && <Field.RadioGroup path={'adjudicated_metrics.motion_text_alignment'} label={'Motion matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('image_quality') && <Field.RadioGroup path={'adjudicated_metrics.image_quality'} label={'Image quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('motion_quality') && <Field.RadioGroup path={'adjudicated_metrics.motion_quality'} label={'Motion quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('subject_background_consistency') && <Field.RadioGroup path={'adjudicated_metrics.subject_background_consistency'} label={'Subject and background consistency'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('video_realism') && <Field.RadioGroup path={'adjudicated_metrics.video_realism'} label={'Realism'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}</View.Card>",
|
|
68
68
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4","disagreed_metrics":["image_quality","motion_quality"]}',
|
|
69
69
|
preprocess: "return {}",
|
|
70
70
|
postprocess: "return {};",
|
|
@@ -157,6 +157,4 @@ export const adjudicate = generation("adjudicate", {
|
|
|
157
157
|
perExpertCapEnabled: false,
|
|
158
158
|
perExpertCapValue: 1,
|
|
159
159
|
},
|
|
160
|
-
reuseAudienceWithNodeId: null,
|
|
161
|
-
smartSearch: null,
|
|
162
160
|
});
|
|
@@ -50,7 +50,7 @@ export const adjudicateOp = generation("adjudicate_op", {
|
|
|
50
50
|
},
|
|
51
51
|
],
|
|
52
52
|
ui: {
|
|
53
|
-
code: "<Field.
|
|
53
|
+
code: "<Field.RadioGroup path={'op_adjudicated_value'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
|
|
54
54
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
|
|
55
55
|
preprocess: "return {}",
|
|
56
56
|
postprocess: "return {};",
|
|
@@ -147,6 +147,4 @@ export const adjudicateOp = generation("adjudicate_op", {
|
|
|
147
147
|
perExpertCapEnabled: false,
|
|
148
148
|
perExpertCapValue: 1,
|
|
149
149
|
},
|
|
150
|
-
reuseAudienceWithNodeId: null,
|
|
151
|
-
smartSearch: null,
|
|
152
150
|
});
|
|
@@ -48,7 +48,7 @@ export const overallPreferencePick = quorum("overall_preference_pick", {
|
|
|
48
48
|
instruction:
|
|
49
49
|
"Watch both videos in full, then pick the one you would ship. This is a single judgment about the whole video, not a sum of its parts. Use tie only when you genuinely have no preference. Unlike the metric rating task, this one shows the pair in its original order, so your pick needs no un-shuffling downstream.",
|
|
50
50
|
ui: {
|
|
51
|
-
code: "<Field.
|
|
51
|
+
code: "<Field.RadioGroup path={'overall_manual_votes'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
|
|
52
52
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
|
|
53
53
|
preprocess: "return {}",
|
|
54
54
|
postprocess: "return {};",
|
|
@@ -96,7 +96,7 @@ export const ratePair = quorum("rate_pair", {
|
|
|
96
96
|
instruction:
|
|
97
97
|
"Watch both videos in full before answering. For each metric, pick the video that is better on that metric alone, or tie when they are equal. The two videos are presented in random order, so left and right carry no meaning. Different metrics can have different winners; that is expected.",
|
|
98
98
|
ui: {
|
|
99
|
-
code: "<View.
|
|
99
|
+
code: "<View.Card><Field.RadioGroup path={'image_text_alignment'} label={'Which video matches the prompt in its imagery?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'motion_text_alignment'} label={'Which video matches the prompt in its motion?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'image_quality'} label={'Which video looks better frame by frame?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'motion_quality'} label={'Which video moves more plausibly?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'subject_background_consistency'} label={'Which video keeps its subject and background stable?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'video_realism'} label={'Which video reads as more real?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /></View.Card>",
|
|
100
100
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","left_video":"https://example.com/video-left.mp4","right_video":"https://example.com/video-right.mp4"}',
|
|
101
101
|
preprocess: "return {}",
|
|
102
102
|
postprocess: "return {};",
|