@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +10 -15
  2. package/dist/cli/toloka.js +209 -121
  3. package/dist/cli/toloka.js.jsc +0 -0
  4. package/dist/cli/toloka.js.jsc.sha256 +1 -1
  5. package/dist/cli/toloka.js.sha256 +1 -1
  6. package/docs/authoring.md +20 -9
  7. package/docs/cli-v2.md +3 -3
  8. package/docs/cli.md +44 -16
  9. package/docs/getting-started.md +7 -12
  10. package/docs/limitations.md +6 -6
  11. package/docs/testing-examples.md +248 -0
  12. package/docs/testing.md +115 -0
  13. package/examples/face-image-collection/README.md +7 -1
  14. package/examples/face-image-collection/fields.ts +20 -8
  15. package/examples/face-image-collection/nodes/annotate-faces/node.ts +1 -9
  16. package/examples/face-image-collection/nodes/order-faces/node.ts +3 -5
  17. package/examples/face-image-collection/pipeline.browser.integration.spec.ts +78 -0
  18. package/examples/video-pair-rating/README.md +14 -6
  19. package/examples/video-pair-rating/nodes/adjudicate/node.ts +1 -3
  20. package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +1 -3
  21. package/examples/video-pair-rating/nodes/adjudication_match/node.ts +15 -17
  22. package/examples/video-pair-rating/nodes/assemble/node.ts +25 -27
  23. package/examples/video-pair-rating/nodes/cascade/node.ts +24 -26
  24. package/examples/video-pair-rating/nodes/finalize_overall/node.ts +17 -19
  25. package/examples/video-pair-rating/nodes/merge_adjudication/node.ts +13 -15
  26. package/examples/video-pair-rating/nodes/merge_op_adjudication/node.ts +12 -14
  27. package/examples/video-pair-rating/nodes/op_quorum_resolve/node.ts +13 -15
  28. package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +1 -1
  29. package/examples/video-pair-rating/nodes/rate_pair/node.ts +1 -1
  30. package/examples/video-pair-rating/nodes/route_adjudication/node.ts +1 -3
  31. package/examples/video-pair-rating/nodes/route_op_adjudication/node.ts +5 -7
  32. package/examples/video-pair-rating/nodes/route_overall/node.ts +1 -3
  33. package/examples/video-pair-rating/nodes/shuffle_ab/node.ts +11 -13
  34. package/examples/video-pair-rating/pipeline.browser.integration.spec.ts +267 -0
  35. package/openapi/gen/api.ts +48 -11
  36. package/openapi/gen/models/createDatasetExportBody.ts +3 -3
  37. package/openapi/gen/models/createDatasetExportBodyOneOf.ts +6 -0
  38. package/openapi/gen/models/createDatasetExportBodyOneOfFormat.ts +2 -0
  39. package/openapi/gen/models/createDatasetExportBodyOneOfThree.ts +10 -0
  40. package/openapi/gen/models/createDatasetExportBodyOneOfThreeContract.ts +2 -0
  41. package/openapi/gen/models/createDatasetExportBodyOneOfThreeFormat.ts +2 -0
  42. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilter.ts +9 -0
  43. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem.ts +6 -0
  44. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  45. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  46. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  47. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItem.ts +12 -0
  48. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator.ts +2 -0
  49. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
  50. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  51. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  52. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue.ts +2 -0
  53. package/openapi/gen/models/datasetExportReport.ts +9 -0
  54. package/openapi/gen/models/datasetExportReportErrorsItem.ts +5 -0
  55. package/openapi/gen/models/datasetExportReportTokens.ts +7 -0
  56. package/openapi/gen/models/datasetItem.ts +0 -2
  57. package/openapi/gen/models/exportJob.ts +2 -0
  58. package/openapi/gen/models/exportJobReport.ts +3 -0
  59. package/openapi/gen/models/index.ts +48 -1
  60. package/openapi/gen/models/sampleDatasetItemsBody.ts +4 -0
  61. package/openapi/gen/models/sampleDatasetItemsBodyAnyOf.ts +8 -0
  62. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilter.ts +9 -0
  63. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItem.ts +6 -0
  64. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  65. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  66. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  67. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItem.ts +12 -0
  68. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemOperator.ts +2 -0
  69. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItem.ts +4 -0
  70. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  71. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  72. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemValue.ts +2 -0
  73. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethree.ts +8 -0
  74. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilter.ts +9 -0
  75. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItem.ts +6 -0
  76. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  77. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  78. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  79. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItem.ts +12 -0
  80. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemOperator.ts +2 -0
  81. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
  82. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  83. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  84. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemValue.ts +2 -0
  85. package/openapi/gen/models/sampleJob.ts +13 -0
  86. package/openapi/gen/models/sampleJobStatus.ts +2 -0
  87. package/openapi/gen/models/sampleResult.ts +5 -0
  88. package/package.json +23 -2
  89. package/skills/catalog.json +14 -6
  90. package/skills/toloka-build/SKILL.md +35 -12
  91. package/skills/toloka-build/references/node-semantics.md +32 -29
  92. package/skills/toloka-build/references/workflow-and-cli.md +61 -27
  93. package/skills/toloka-report-issue/references/report-template.md +1 -1
  94. package/src/authoring/index.ts +27 -26
  95. package/src/checks/ast.ts +32 -0
  96. package/src/checks/index.ts +14 -0
  97. package/src/checks/postprocess-collisions.ts +145 -0
  98. package/src/checks/task-ui-declarations.generated.ts +19 -0
  99. package/src/checks/task-ui-typecheck.ts +345 -0
  100. package/src/checks/typescript-vfs.d.ts +26 -0
  101. package/src/checks/typescript-vfs.js +5 -0
  102. package/src/cli/analytics.ts +9 -19
  103. package/src/cli/auto-update.ts +367 -0
  104. package/src/cli/boolean-flag.ts +13 -0
  105. package/src/cli/check.ts +977 -0
  106. package/src/cli/credentials.ts +3 -0
  107. package/src/cli/program.ts +184 -20
  108. package/src/cli/project-context.ts +14 -4
  109. package/src/cli/runs.ts +21 -11
  110. package/src/cli/settings.ts +93 -0
  111. package/src/cli/skill.ts +17 -6
  112. package/src/importer.ts +12 -16
  113. package/src/testing/code-node.ts +160 -0
  114. package/src/testing/index.ts +15 -0
  115. package/src/testing/task-ui-browser.ts +120 -0
  116. package/src/testing/task-ui-validation.ts +135 -0
  117. package/src/testing/task-ui.ts +557 -0
  118. package/src/testing/test-evidence.ts +170 -0
  119. package/src/testing/types.ts +141 -0
  120. package/openapi/gen/models/datasetItemDuplicateOfSubmissionId.ts +0 -2
@@ -0,0 +1,115 @@
1
+ # Test pipelines locally
2
+
3
+ The `@toloka-platform/sdk/testing` entry point provides typed Vitest wrappers for executable Python `code` nodes, Chromium Task UI scenarios, and complete pipeline flows. `toloka check` runs those tests together with project TypeScript and automatic Task UI typechecking without creating or modifying a platform project.
4
+
5
+ ## Install test dependencies
6
+
7
+ ```sh
8
+ bun add --dev vitest playwright
9
+ bunx playwright install chromium
10
+ ```
11
+
12
+ Code-node tests also require `python3` on the test machine. See [testing examples](./testing-examples.md) for parameterization, validation failures, hardware coverage, and cleanup patterns.
13
+
14
+ ## Test a code node
15
+
16
+ `test.code(pipeline, node, name, callback)` binds one test to one authored code node. Its `run(inputs, options?)` helper executes the Python source with the same async `main(inputs)` entry contract. The test counts as node coverage only after `run` executes, at least one assertion runs, and the callback passes.
17
+
18
+ ```ts
19
+ import { expect, test } from "@toloka-platform/sdk/testing";
20
+ import pipeline from "../../pipeline";
21
+ import { normalize } from "./node";
22
+
23
+ test.code(pipeline, normalize, "normalizes whitespace", async ({ run }) => {
24
+ await expect(run({ text: " hello " })).resolves.toEqual({
25
+ normalized_text: "hello",
26
+ });
27
+ });
28
+ ```
29
+
30
+ Failures from the bound executor throw `CodeNodeExecutionError` with the Python exit code and stderr. Authored static environment values are included; pass secret or test-specific values through the optional `run` options. Third-party imports must be available to the selected Python executable.
31
+
32
+ `test.code` defaults to 30 seconds, which exceeds the code runner's 10-second default. Pass `{ timeout: milliseconds }` after the callback when the scenario also raises the runner timeout.
33
+
34
+ ## Test a complete pipeline flow
35
+
36
+ Register the representative connected scenario with `test.pipeline(pipeline, name, callback)`. Its callback provides `runCode(node, inputs, options?)` and `renderUi(node, options?)` bound to that pipeline.
37
+
38
+ Use the first row from the same local dataset file intended for upload when one is available. If it contains sensitive values, keep a redacted fixture that preserves field names, nesting, optionality, and media/value shapes. Use synthetic input only when no local row exists. `test.pipeline` does not fetch a live dataset: the local gate remains deterministic, credential-free, and usable before apply. Treat the fixture as `root.started` item state, execute connected nodes in order, merge only each node's declared outputs, and assert the fields accepted by `root.completed`. A linear happy path can stay in one test; use separate scenarios for materially different match branches, optional paths, and failures.
39
+
40
+ ## Test a Task UI
41
+
42
+ Create a file ending in `.ui.spec.ts` next to the human-work node. `test.ui` binds the test to the pipeline and node, and its callback receives `render(options?)` for that target. The test counts as coverage only after a successful submission, at least one assertion, and a passing callback.
43
+
44
+ Browser-backed `test.ui` scenarios default to 60 seconds, which exceeds the launcher's 45-second setup budget. `test.pipeline` scenarios default to 120 seconds for connected flows with multiple UI steps. Pass `{ timeout: milliseconds }` after the callback to tune either wrapper without weakening its inner launcher deadlines.
45
+
46
+ ```ts
47
+ import { expect, test } from "@toloka-platform/sdk/testing";
48
+ import pipeline from "../../pipeline";
49
+ import { annotate } from "./node";
50
+
51
+ test.ui(pipeline, annotate, "submits a valid answer", async ({ render }) => {
52
+ const task = await render({
53
+ data: { prompt: "What is two plus two?" },
54
+ });
55
+
56
+ await task.setValue("raw_answer", "four");
57
+
58
+ expect(await task.validationErrors()).toEqual([]);
59
+ expect(await task.submit()).toMatchObject({
60
+ answer: "four",
61
+ });
62
+ await task.screenshot("test-results/answer.png");
63
+ });
64
+ ```
65
+
66
+ Run the complete gate rather than invoking Vitest separately:
67
+
68
+ ```sh
69
+ toloka check
70
+ ```
71
+
72
+ Use an ordinary TypeScript loop to register several `test.code` or `test.ui` scenarios. There is no separate JSON scenario format.
73
+
74
+ ## Fixture API
75
+
76
+ The low-level `ui` fixture remains available inside ordinary Vitest tests for component discovery and exploratory work, but plain tests do not satisfy required node coverage. Use typed wrappers for committed gate evidence. Importing `@toloka-platform/sdk/testing` from a plain Bun script executes Vitest fixture setup and fails.
77
+
78
+ - `ui.componentDocumentation(names?)` returns the current launcher's supported component contracts. Call it before choosing component names or props.
79
+ - `ui.render(pipeline, node, options)` creates a fresh browser context and returns a Task UI session for exactly that authored node. `test.ui` supplies the equivalent bound `render(options?)` helper.
80
+
81
+ The returned Task UI session supports:
82
+
83
+ - `componentTree()` returns the rendered component hierarchy.
84
+ - `componentDocumentation(names?)` returns component props, value shapes, capabilities, and limitations.
85
+ - `setValue(path, value)` and `getValue(path)` address Fields by their stable `path`, not generated component ids.
86
+ - `validationErrors()` returns browser-level required-field and component-rule failures with Field paths.
87
+ - `submit()` executes preprocess/postprocess, removes the launcher's `_context`, and validates declared output entities. The returned form data can also contain echoed inputs; assert output-entity keys and values without treating those inputs as output entities.
88
+ - `screenshot(path)` writes a full-page PNG to the local filesystem.
89
+ - `coverage` states whether hardware-backed fields were exercised completely.
90
+
91
+ One Chromium process is shared per Vitest worker. Every scenario gets a fresh context and page and is cleaned up automatically.
92
+
93
+ For a structured or array-valued Field, derive the dataset field schema from the component's documented `valueType`. In its typed UI scenario, adapt only semantic values such as labels, then pass the documented example value through `setValue`, `getValue`, and `submit`. This catches component-shape drift before a postprocess can read absent keys and silently write fallback data.
94
+
95
+ ## Runtime rules
96
+
97
+ `ui.code` must parse as one JSX root. Every identifier must be a component returned by `componentDocumentation()` or a documented `actions` API. `toloka check` applies full TypeScript semantic checking with declarations bundled in the SDK; the real launcher then renders the same JSX in browser scenarios. Treat a component's `propsType` and `example` together as its contract.
98
+
99
+ `preprocess` and `postprocess` are JavaScript function bodies, not TypeScript:
100
+
101
+ Valid JavaScript: `items.map((item) => item.value)`
102
+
103
+ Invalid runtime JavaScript: `items.map((item: unknown) => item.value)`
104
+
105
+ `preprocess` can run again after form state changes. Use it to shape input data; if it prefills a bound field, return that key only while it is absent so later edits are not overwritten. A `Field`-bound top-level key already enters the submission. `postprocess` must return only new top-level keys: bind a component to a draft key when postprocess must derive a different entity key.
106
+
107
+ The fixture loads the hosted TBX launcher in a fresh local Chromium context, so tests require network access but do not use Azure Playwright or Potato application services. The context has no platform cookies or credentials. Use synthetic or redacted fixtures because the hosted launcher executes with access to the supplied test data.
108
+
109
+ ## Hardware-backed fields
110
+
111
+ File pickers, video upload/recording, and other hardware-backed fields cannot be completed reliably in headless Chromium. `task.coverage.complete` is `false` when these components are present. Render with `allowPartialCoverage: true`, inject one value matching the component's documented `valueType` through `setValue`, and assert the postprocessed output shape yourself: entity validation skips hardware-backed paths, and `getValue` may not serialize their launcher state. This checks browser validation and mapping only, not capture, upload, link generation, or record-time constraints. A partial result must not be described as complete coverage.
112
+
113
+ ## Final gate
114
+
115
+ `toloka check` loads every authored pipeline, runs the project TypeScript compiler, typechecks every Task UI against declarations bundled with the installed SDK, runs Vitest, and requires passing `test.code`, `test.ui`, and `test.pipeline` evidence. `missing`, `failed`, `partial`, and `infrastructure_error` are distinct blocking outcomes. `toloka apply` reruns this complete gate before resolving credentials or writing platform state. Platform source/spec errors from `toloka validate` remain separate; `not_ran_node_check` describes a Remote Node Check and does not replace local evidence.
@@ -17,7 +17,13 @@ toloka import pp.0190abcd-abcd-7abc-8abc-abcdefabcdef
17
17
  # Imported project pp.0190abcd-abcd-7abc-8abc-abcdefabcdef with 1 pipeline and 1 dataset into ...
18
18
  ```
19
19
 
20
- Edit a node under `pipelines/face-image-collection/nodes/`, then inspect and apply the project-wide reconciliation:
20
+ Edit a node under `pipelines/face-image-collection/nodes/`, then run the shipped typed code, UI, and connected-flow scenarios before inspecting any platform changes:
21
+
22
+ ```sh
23
+ toloka check face-image-collection
24
+ ```
25
+
26
+ After the local gate passes, inspect and apply the project-wide reconciliation:
21
27
 
22
28
  ```sh
23
29
  toloka plan
@@ -5,19 +5,31 @@ const faceAnnotationsSchema: Parameters<typeof field>[1] = {
5
5
  items: {
6
6
  type: "object",
7
7
  properties: {
8
- boundingBox: {
8
+ id: { type: "string" },
9
+ type: { type: "string" },
10
+ geometry: {
9
11
  type: "object",
10
12
  properties: {
11
- x: { type: "number" },
12
- y: { type: "number" },
13
- width: { type: "number" },
14
- height: { type: "number" },
13
+ bounds: {
14
+ type: "object",
15
+ properties: {
16
+ minX: { type: "number" },
17
+ minY: { type: "number" },
18
+ maxX: { type: "number" },
19
+ maxY: { type: "number" },
20
+ },
21
+ required: ["minX", "minY", "maxX", "maxY"],
22
+ },
15
23
  },
16
- required: ["x", "y", "width", "height"],
24
+ required: ["bounds"],
25
+ },
26
+ properties: {
27
+ type: "object",
28
+ properties: { value: { type: "string", const: "face" } },
29
+ required: ["value"],
17
30
  },
18
- label: { type: "string", const: "face" },
19
31
  },
20
- required: ["boundingBox", "label"],
32
+ required: ["id", "type", "geometry", "properties"],
21
33
  },
22
34
  };
23
35
 
@@ -55,7 +55,7 @@ export const annotateFaces = generation("annotate_faces", {
55
55
  instruction:
56
56
  "Draw one tight rectangular bounding box around every visible human face. Do not annotate faces shown only in photographs, posters, or reflections.",
57
57
  ui: {
58
- code: "<Field.ImageAnnotation path={'draft_faces'} image={actions.get('image_url', '')} tools={['rectangle']} labels={[{ label: 'Face', value: 'face' }]} />",
58
+ code: "<Field.ImageAnnotation path={'draft_faces'} image={actions.get('image_url')} tools={['rectangle']} labels={[{ label: 'Face', value: 'face', color: 'blue' }]} navigator={true} />",
59
59
  data: '{"image_url":"https://example.com/face.jpg"}',
60
60
  preprocess: "return {}",
61
61
  postprocess: "return {};",
@@ -131,12 +131,4 @@ export const annotateFaces = generation("annotate_faces", {
131
131
  successCriteria: "Every face has one tight bounding box",
132
132
  },
133
133
  },
134
- exam: {
135
- enabled: false,
136
- gradedEntities: [],
137
- tasksPerExpert: 1,
138
- passThreshold: 1,
139
- },
140
- reuseAudienceWithNodeId: null,
141
- smartSearch: null,
142
134
  });
@@ -9,17 +9,15 @@ export const orderFaces = code("order_faces", {
9
9
  "Order face annotations from top left to bottom right for deterministic downstream data.",
10
10
  },
11
11
  envVars: [],
12
- fieldProxies: {
13
- input: [{ field: draftFaces, optional: false }],
14
- output: [{ field: faces, optional: false }],
15
- },
12
+ input: [{ field: draftFaces, optional: false }],
13
+ output: [{ field: faces, optional: false }],
16
14
  code: {
17
15
  code: `from main_types import Inputs, Outputs
18
16
 
19
17
  async def main(inputs: Inputs) -> Outputs:
20
18
  ordered_faces = sorted(
21
19
  inputs["draft_faces"],
22
- key=lambda face: (face["boundingBox"]["y"], face["boundingBox"]["x"]),
20
+ key=lambda face: (face["geometry"]["bounds"]["minY"], face["geometry"]["bounds"]["minX"]),
23
21
  )
24
22
  return {"faces": ordered_faces}`,
25
23
  },
@@ -0,0 +1,78 @@
1
+ import { expect, test } from "@toloka-platform/sdk/testing";
2
+ import { describe } from "vitest";
3
+ import { assert } from "es-toolkit";
4
+ import type { TaskUiSession } from "@toloka-platform/sdk/testing";
5
+
6
+ import pipeline from "./pipeline";
7
+ import { annotateFaces } from "./nodes/annotate-faces/node";
8
+ import { orderFaces } from "./nodes/order-faces/node";
9
+ const upperFace = {
10
+ id: "upper-face",
11
+ type: "RECTANGLE",
12
+ geometry: { x: 40, y: 10, w: 20, h: 20, bounds: { minX: 40, minY: 10, maxX: 60, maxY: 30 } },
13
+ properties: { value: "face" },
14
+ };
15
+ const lowerFace = {
16
+ id: "lower-face",
17
+ type: "RECTANGLE",
18
+ geometry: { x: 10, y: 60, w: 30, h: 30, bounds: { minX: 10, minY: 60, maxX: 40, maxY: 90 } },
19
+ properties: { value: "face" },
20
+ };
21
+ const imageUrl =
22
+ "https://tlkfrontprod.azureedge.net/frontend-testing/static/gzarip/cats/cat_001.jpg";
23
+
24
+ const isRecord = (value: unknown): value is Record<string, unknown> =>
25
+ typeof value === "object" && value !== null && !Array.isArray(value);
26
+
27
+ const documentedFaceAnnotations = async (
28
+ session: TaskUiSession,
29
+ ): Promise<Record<string, unknown>[]> => {
30
+ const documentation = await session.componentDocumentation(["Field.ImageAnnotation"]);
31
+ const output = documentation["Field.ImageAnnotation"]?.example?.output;
32
+ assert(output, "Field.ImageAnnotation has no documented example output");
33
+ const example: unknown = JSON.parse(output);
34
+ assert(isRecord(example), "Field.ImageAnnotation example output is not an object");
35
+ const annotations = example.roi;
36
+ assert(Array.isArray(annotations), "Field.ImageAnnotation example output has no roi array");
37
+ return annotations.map((annotation) => {
38
+ assert(isRecord(annotation), "Field.ImageAnnotation example contains a non-object value");
39
+ assert(
40
+ isRecord(annotation.properties),
41
+ "Field.ImageAnnotation example annotation has no properties",
42
+ );
43
+ return {
44
+ ...annotation,
45
+ properties: { ...annotation.properties, value: "face" },
46
+ };
47
+ });
48
+ };
49
+
50
+ describe("face image collection example", () => {
51
+ test.code(pipeline, orderFaces, "orders faces from top to bottom", async ({ run }) => {
52
+ await expect(run({ draft_faces: [lowerFace, upperFace] })).resolves.toEqual({
53
+ faces: [upperFace, lowerFace],
54
+ });
55
+ });
56
+
57
+ test.ui(pipeline, annotateFaces, "submits face annotations", async ({ render }) => {
58
+ const session = await render({ data: { image_url: imageUrl, draft_faces: [] } });
59
+ const annotations = await documentedFaceAnnotations(session);
60
+ await session.setValue("draft_faces", annotations);
61
+
62
+ expect(await session.getValue("draft_faces")).toEqual(annotations);
63
+ await expect(session.submit()).resolves.toMatchObject({ draft_faces: annotations });
64
+ await session.close();
65
+ });
66
+
67
+ test.pipeline(pipeline, "collects and orders face annotations", async ({ renderUi, runCode }) => {
68
+ const session = await renderUi(annotateFaces, {
69
+ data: { image_url: imageUrl, draft_faces: [] },
70
+ });
71
+ await session.setValue("draft_faces", [lowerFace, upperFace]);
72
+ const annotation = await session.submit();
73
+ const result = await runCode(orderFaces, { draft_faces: annotation.draft_faces });
74
+
75
+ expect(result).toEqual({ faces: [upperFace, lowerFace] });
76
+ await session.close();
77
+ });
78
+ });
@@ -33,12 +33,12 @@ Sixteen nodes: one root, eight `code`, three `match`, two `generation`, two `quo
33
33
 
34
34
  ## What to read first
35
35
 
36
- | file | why |
37
- |---|---|
38
- | `pipeline.ts` | the whole topology on one screen — every edge is explicit |
39
- | `nodes/cascade/node.ts` | the deterministic core that resolves every metric from rater arrays |
40
- | `nodes/rate_pair/node.ts` | a quorum node with real quality criteria and a Training course |
41
- | `nodes/route_overall/node.ts` | predicate routing, and what the `default` branch is for |
36
+ | file | why |
37
+ | ----------------------------- | ------------------------------------------------------------------- |
38
+ | `pipeline.ts` | the whole topology on one screen — every edge is explicit |
39
+ | `nodes/cascade/node.ts` | the deterministic core that resolves every metric from rater arrays |
40
+ | `nodes/rate_pair/node.ts` | a quorum node with real quality criteria and a Training course |
41
+ | `nodes/route_overall/node.ts` | predicate routing, and what the `default` branch is for |
42
42
 
43
43
  ## Conventions this example demonstrates
44
44
 
@@ -52,6 +52,14 @@ Sixteen nodes: one root, eight `code`, three `match`, two `generation`, two `quo
52
52
 
53
53
  **Adjudication is a branch, not a loop.** The graph stays acyclic. Redo, appeal, and payment retry live inside the human-work nodes; only the tie-breaking hand-off is a graph edge.
54
54
 
55
+ ## Check locally
56
+
57
+ The shipped typed scenarios cover all eight code nodes, all four Task UIs, and an agreed-pair connected flow:
58
+
59
+ ```sh
60
+ toloka check video-pair-rating
61
+ ```
62
+
55
63
  ## Provenance
56
64
 
57
65
  The topology, the resolution cascade, and the per-node layout are adapted from a design spike by @hexode. Media URLs, sample payloads, prompts, and prose here are synthetic: no customer data, project identifiers, or asset URLs from the original are carried over.
@@ -64,7 +64,7 @@ export const adjudicate = generation("adjudicate", {
64
64
  },
65
65
  ],
66
66
  ui: {
67
- code: "<View.Group><If condition={actions.get('disagreed_metrics', []).includes('image_text_alignment')}><Field.SelectGroup path={'adjudicated_metrics.image_text_alignment'} label={'Imagery matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('motion_text_alignment')}><Field.SelectGroup path={'adjudicated_metrics.motion_text_alignment'} label={'Motion matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('image_quality')}><Field.SelectGroup path={'adjudicated_metrics.image_quality'} label={'Image quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('motion_quality')}><Field.SelectGroup path={'adjudicated_metrics.motion_quality'} label={'Motion quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('subject_background_consistency')}><Field.SelectGroup path={'adjudicated_metrics.subject_background_consistency'} label={'Subject and background consistency'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('video_realism')}><Field.SelectGroup path={'adjudicated_metrics.video_realism'} label={'Realism'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If></View.Group>",
67
+ code: "<View.Card>{actions.get('disagreed_metrics', []).includes('image_text_alignment') && <Field.RadioGroup path={'adjudicated_metrics.image_text_alignment'} label={'Imagery matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('motion_text_alignment') && <Field.RadioGroup path={'adjudicated_metrics.motion_text_alignment'} label={'Motion matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('image_quality') && <Field.RadioGroup path={'adjudicated_metrics.image_quality'} label={'Image quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('motion_quality') && <Field.RadioGroup path={'adjudicated_metrics.motion_quality'} label={'Motion quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('subject_background_consistency') && <Field.RadioGroup path={'adjudicated_metrics.subject_background_consistency'} label={'Subject and background consistency'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('video_realism') && <Field.RadioGroup path={'adjudicated_metrics.video_realism'} label={'Realism'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}</View.Card>",
68
68
  data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4","disagreed_metrics":["image_quality","motion_quality"]}',
69
69
  preprocess: "return {}",
70
70
  postprocess: "return {};",
@@ -157,6 +157,4 @@ export const adjudicate = generation("adjudicate", {
157
157
  perExpertCapEnabled: false,
158
158
  perExpertCapValue: 1,
159
159
  },
160
- reuseAudienceWithNodeId: null,
161
- smartSearch: null,
162
160
  });
@@ -50,7 +50,7 @@ export const adjudicateOp = generation("adjudicate_op", {
50
50
  },
51
51
  ],
52
52
  ui: {
53
- code: "<Field.SelectGroup path={'op_adjudicated_value'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
53
+ code: "<Field.RadioGroup path={'op_adjudicated_value'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
54
54
  data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
55
55
  preprocess: "return {}",
56
56
  postprocess: "return {};",
@@ -147,6 +147,4 @@ export const adjudicateOp = generation("adjudicate_op", {
147
147
  perExpertCapEnabled: false,
148
148
  perExpertCapValue: 1,
149
149
  },
150
- reuseAudienceWithNodeId: null,
151
- smartSearch: null,
152
150
  });
@@ -20,23 +20,21 @@ export const adjudicationMatch = code("adjudication_match", {
20
20
  "Computes per-rater matches_adjudication booleans for image_text_alignment, image_quality, and overall preference. Only populated when adjudication actually ran; otherwise null.",
21
21
  },
22
22
  envVars: [],
23
- fieldProxies: {
24
- input: [
25
- { field: raterItaVotes, optional: false },
26
- { field: raterIqVotes, optional: false },
27
- { field: raterOpVotes, optional: true },
28
- { field: disagreedMetrics, optional: false },
29
- { field: finalMetrics, optional: false },
30
- { field: needsOverallManual, optional: false },
31
- { field: opNeedsAdjudication, optional: true },
32
- { field: finalOpValue, optional: true },
33
- ],
34
- output: [
35
- { field: raterItaMatchesAdjudication, optional: false },
36
- { field: raterIqMatchesAdjudication, optional: false },
37
- { field: raterOpMatchesAdjudication, optional: false },
38
- ],
39
- },
23
+ input: [
24
+ { field: raterItaVotes, optional: false },
25
+ { field: raterIqVotes, optional: false },
26
+ { field: raterOpVotes, optional: true },
27
+ { field: disagreedMetrics, optional: false },
28
+ { field: finalMetrics, optional: false },
29
+ { field: needsOverallManual, optional: false },
30
+ { field: opNeedsAdjudication, optional: true },
31
+ { field: finalOpValue, optional: true },
32
+ ],
33
+ output: [
34
+ { field: raterItaMatchesAdjudication, optional: false },
35
+ { field: raterIqMatchesAdjudication, optional: false },
36
+ { field: raterOpMatchesAdjudication, optional: false },
37
+ ],
40
38
  code: {
41
39
  code: `from main_types import Inputs, Outputs
42
40
 
@@ -8,32 +8,30 @@ export const assemble = code("assemble", {
8
8
  "Assembles the final JSONL delivery record containing all evaluation results: video identifiers, text prompt, per-metric scores, overall preference, annotator metadata, resolution method, rater vote history, and a UTC timestamp.",
9
9
  },
10
10
  envVars: [],
11
- fieldProxies: {
12
- input: [
13
- { field: f.recordId, optional: false },
14
- { field: f.prompt, optional: false },
15
- { field: f.videoA, optional: false },
16
- { field: f.videoB, optional: false },
17
- { field: f.finalMetrics, optional: false },
18
- { field: f.overallPreference, optional: false },
19
- { field: f.overallPreferenceSource, optional: false },
20
- { field: f.resolutionMethod, optional: false },
21
- { field: f.annotatorIdAdjudicator, optional: false },
22
- { field: f.raterItaVotes, optional: false },
23
- { field: f.raterMtaVotes, optional: false },
24
- { field: f.raterIqVotes, optional: false },
25
- { field: f.raterMqVotes, optional: false },
26
- { field: f.raterSbcVotes, optional: false },
27
- { field: f.raterVrVotes, optional: false },
28
- { field: f.raterOpVotes, optional: true },
29
- { field: f.raterDurationSeconds, optional: true },
30
- { field: f.raterBothPlayed, optional: true },
31
- { field: f.raterItaMatchesAdjudication, optional: false },
32
- { field: f.raterIqMatchesAdjudication, optional: false },
33
- { field: f.raterOpMatchesAdjudication, optional: false },
34
- ],
35
- output: [{ field: f.deliveryRecord, optional: false }],
36
- },
11
+ input: [
12
+ { field: f.recordId, optional: false },
13
+ { field: f.prompt, optional: false },
14
+ { field: f.videoA, optional: false },
15
+ { field: f.videoB, optional: false },
16
+ { field: f.finalMetrics, optional: false },
17
+ { field: f.overallPreference, optional: false },
18
+ { field: f.overallPreferenceSource, optional: false },
19
+ { field: f.resolutionMethod, optional: false },
20
+ { field: f.annotatorIdAdjudicator, optional: false },
21
+ { field: f.raterItaVotes, optional: false },
22
+ { field: f.raterMtaVotes, optional: false },
23
+ { field: f.raterIqVotes, optional: false },
24
+ { field: f.raterMqVotes, optional: false },
25
+ { field: f.raterSbcVotes, optional: false },
26
+ { field: f.raterVrVotes, optional: false },
27
+ { field: f.raterOpVotes, optional: true },
28
+ { field: f.raterDurationSeconds, optional: true },
29
+ { field: f.raterBothPlayed, optional: true },
30
+ { field: f.raterItaMatchesAdjudication, optional: false },
31
+ { field: f.raterIqMatchesAdjudication, optional: false },
32
+ { field: f.raterOpMatchesAdjudication, optional: false },
33
+ ],
34
+ output: [{ field: f.deliveryRecord, optional: false }],
37
35
  code: {
38
36
  code: `from datetime import datetime, timezone
39
37
  from main_types import Inputs, Outputs
@@ -60,7 +58,7 @@ async def main(inputs: Inputs) -> Outputs:
60
58
  "video_realism": final_metrics.get("video_realism"),
61
59
  "video_image_text_alignment": final_metrics.get("image_text_alignment"),
62
60
  "video_motion_text_alignment": final_metrics.get("motion_text_alignment"),
63
- # Platform limitation: rater ids are not surfaced as field proxies
61
+ # Platform limitation: rater ids are not surfaced as output fields
64
62
  # from the quorum node in this pilot. Stubs preserve schema conformance.
65
63
  "annotator_id_1": None,
66
64
  "annotator_id_2": None,
@@ -8,32 +8,30 @@ export const cascade = code("cascade", {
8
8
  "Un-swaps each rater per-metric vote back to canonical video_a/video_b/tie labels using the swap flag. Computes per-metric majority agreement and flags metrics requiring adjudication. Applies a deterministic Overall Preference cascade: image-text alignment decides first, then image quality, falling back to manual review.",
9
9
  },
10
10
  envVars: [],
11
- fieldProxies: {
12
- input: [
13
- { field: f.swap, optional: false },
14
- { field: f.imageTextAlignment, optional: false },
15
- { field: f.motionTextAlignment, optional: false },
16
- { field: f.imageQuality, optional: false },
17
- { field: f.motionQuality, optional: false },
18
- { field: f.subjectBackgroundConsistency, optional: false },
19
- { field: f.videoRealism, optional: false },
20
- ],
21
- output: [
22
- { field: f.perMetricResolved, optional: false },
23
- { field: f.needsAdjudication, optional: false },
24
- { field: f.disagreedTask, optional: false },
25
- { field: f.disagreedMetrics, optional: false },
26
- { field: f.opValueOrNull, optional: false },
27
- { field: f.opSource, optional: false },
28
- { field: f.needsOverallManual, optional: false },
29
- { field: f.raterItaVotes, optional: false },
30
- { field: f.raterMtaVotes, optional: false },
31
- { field: f.raterIqVotes, optional: false },
32
- { field: f.raterMqVotes, optional: false },
33
- { field: f.raterSbcVotes, optional: false },
34
- { field: f.raterVrVotes, optional: false },
35
- ],
36
- },
11
+ input: [
12
+ { field: f.swap, optional: false },
13
+ { field: f.imageTextAlignment, optional: false },
14
+ { field: f.motionTextAlignment, optional: false },
15
+ { field: f.imageQuality, optional: false },
16
+ { field: f.motionQuality, optional: false },
17
+ { field: f.subjectBackgroundConsistency, optional: false },
18
+ { field: f.videoRealism, optional: false },
19
+ ],
20
+ output: [
21
+ { field: f.perMetricResolved, optional: false },
22
+ { field: f.needsAdjudication, optional: false },
23
+ { field: f.disagreedTask, optional: false },
24
+ { field: f.disagreedMetrics, optional: false },
25
+ { field: f.opValueOrNull, optional: false },
26
+ { field: f.opSource, optional: false },
27
+ { field: f.needsOverallManual, optional: false },
28
+ { field: f.raterItaVotes, optional: false },
29
+ { field: f.raterMtaVotes, optional: false },
30
+ { field: f.raterIqVotes, optional: false },
31
+ { field: f.raterMqVotes, optional: false },
32
+ { field: f.raterSbcVotes, optional: false },
33
+ { field: f.raterVrVotes, optional: false },
34
+ ],
37
35
  code: {
38
36
  code: `from main_types import Inputs, Outputs
39
37
 
@@ -8,25 +8,23 @@ export const finalizeOverall = code("finalize_overall", {
8
8
  "Combines cascade-derived and manually adjudicated Overall Preference into the final output. Re-runs the cascade against post-adjudication metrics and falls back to the manual quorum majority vote when both cascade tiers are tied.",
9
9
  },
10
10
  envVars: [],
11
- fieldProxies: {
12
- input: [
13
- { field: f.opValueOrNull, optional: true },
14
- { field: f.opSource, optional: false },
15
- { field: f.needsOverallManual, optional: false },
16
- { field: f.finalMetrics, optional: false },
17
- { field: f.finalOpValue, optional: true },
18
- { field: f.finalOpResolutionMethod, optional: true },
19
- { field: f.finalOpAdjudicatorId, optional: true },
20
- { field: f.resolutionMethod, optional: false },
21
- { field: f.annotatorIdAdjudicator, optional: true },
22
- ],
23
- output: [
24
- { field: f.overallPreference, optional: false },
25
- { field: f.overallPreferenceSource, optional: false },
26
- { field: f.resolutionMethod, optional: false },
27
- { field: f.annotatorIdAdjudicator, optional: false },
28
- ],
29
- },
11
+ input: [
12
+ { field: f.opValueOrNull, optional: true },
13
+ { field: f.opSource, optional: false },
14
+ { field: f.needsOverallManual, optional: false },
15
+ { field: f.finalMetrics, optional: false },
16
+ { field: f.finalOpValue, optional: true },
17
+ { field: f.finalOpResolutionMethod, optional: true },
18
+ { field: f.finalOpAdjudicatorId, optional: true },
19
+ { field: f.resolutionMethod, optional: false },
20
+ { field: f.annotatorIdAdjudicator, optional: true },
21
+ ],
22
+ output: [
23
+ { field: f.overallPreference, optional: false },
24
+ { field: f.overallPreferenceSource, optional: false },
25
+ { field: f.resolutionMethod, optional: false },
26
+ { field: f.annotatorIdAdjudicator, optional: false },
27
+ ],
30
28
  code: {
31
29
  code: `from main_types import Inputs, Outputs
32
30
 
@@ -18,21 +18,19 @@ export const mergeAdjudication = code("merge_adjudication", {
18
18
  "Merges majority-vote per-metric resolutions from the cascade with adjudicator overrides on disagreed metrics into final_metrics. Also stamps annotator_id_adjudicator and resolution_method.",
19
19
  },
20
20
  envVars: [],
21
- fieldProxies: {
22
- input: [
23
- { field: perMetricResolved, optional: false },
24
- { field: adjudicatedMetrics, optional: true },
25
- { field: disagreedMetrics, optional: true },
26
- ],
27
- output: [
28
- { field: finalMetrics, optional: false },
29
- { field: resolutionMethod, optional: false },
30
- { field: annotatorIdAdjudicator, optional: false },
31
- { field: opValueOrNull, optional: false },
32
- { field: opSource, optional: false },
33
- { field: needsOverallManual, optional: false },
34
- ],
35
- },
21
+ input: [
22
+ { field: perMetricResolved, optional: false },
23
+ { field: adjudicatedMetrics, optional: true },
24
+ { field: disagreedMetrics, optional: true },
25
+ ],
26
+ output: [
27
+ { field: finalMetrics, optional: false },
28
+ { field: resolutionMethod, optional: false },
29
+ { field: annotatorIdAdjudicator, optional: false },
30
+ { field: opValueOrNull, optional: false },
31
+ { field: opSource, optional: false },
32
+ { field: needsOverallManual, optional: false },
33
+ ],
36
34
  code: {
37
35
  code: `from main_types import Inputs, Outputs
38
36