@toloka-platform/sdk 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/toloka.js +157 -154
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +6 -4
- package/docs/cli.md +30 -11
- package/docs/limitations.md +2 -2
- package/docs/testing.md +4 -2
- package/examples/README.md +11 -0
- package/examples/app-crowdtesting/README.md +41 -0
- package/examples/app-crowdtesting/fields.ts +6 -0
- package/examples/app-crowdtesting/nodes/app-flow-test/node.ts +381 -0
- package/examples/app-crowdtesting/nodes/collection/node.ts +8 -0
- package/examples/app-crowdtesting/pipeline.browser.integration.spec.ts +77 -0
- package/examples/app-crowdtesting/pipeline.ts +17 -0
- package/examples/egocentric-video-collection/README.md +35 -0
- package/examples/egocentric-video-collection/fields.ts +24 -0
- package/examples/egocentric-video-collection/nodes/collection/node.ts +5 -0
- package/examples/egocentric-video-collection/nodes/record-activity/node.ts +760 -0
- package/examples/egocentric-video-collection/pipeline.browser.integration.spec.ts +65 -0
- package/examples/egocentric-video-collection/pipeline.ts +17 -0
- package/examples/product-moderation/README.md +49 -0
- package/examples/product-moderation/fields.ts +37 -0
- package/examples/product-moderation/nodes/collection/node.ts +16 -0
- package/examples/product-moderation/nodes/moderate-product/instruction.ts +77 -0
- package/examples/product-moderation/nodes/moderate-product/node.ts +368 -0
- package/examples/product-moderation/pipeline.browser.integration.spec.ts +48 -0
- package/examples/product-moderation/pipeline.ts +33 -0
- package/examples/video-pair-rating/fields.ts +16 -4
- package/examples/video-pair-rating/nodes/adjudicate/node.ts +78 -3
- package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +28 -3
- package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +28 -3
- package/examples/video-pair-rating/nodes/rate_pair/node.ts +65 -2
- package/openapi/gen/api.ts +22 -0
- package/openapi/gen/models/generationBudget.ts +8 -0
- package/openapi/gen/models/generationBudgetReasoningEffort.ts +2 -0
- package/openapi/gen/models/generationModel.ts +12 -0
- package/openapi/gen/models/generationModelDefaultReasoningEffort.ts +2 -0
- package/openapi/gen/models/generationModelId.ts +2 -0
- package/openapi/gen/models/index.ts +8 -0
- package/openapi/gen/models/invalidFieldSchema.ts +6 -0
- package/openapi/gen/models/listGenerationModels200.ts +7 -0
- package/openapi/gen/models/listGenerationModels200DefaultGenerationModelId.ts +2 -0
- package/openapi/gen/models/pipelineGraphSnapshot.ts +2 -0
- package/package.json +2 -2
- package/skills/catalog.json +16 -12
- package/skills/toloka-build/SKILL.md +44 -58
- package/skills/toloka-build/references/human-work.md +156 -0
- package/skills/toloka-build/references/local-setup.md +136 -0
- package/skills/toloka-build/references/node-checks.md +11 -0
- package/skills/toloka-build/references/node-semantics.md +22 -213
- package/skills/toloka-build/references/task-ui.md +61 -0
- package/skills/toloka-build/references/workflow-and-cli.md +29 -229
- package/skills/toloka-fine-tuning/SKILL.md +1 -1
- package/skills/toloka-get-started/SKILL.md +1 -1
- package/skills/toloka-gisting/SKILL.md +1 -1
- package/src/authoring/index.ts +26 -7
- package/src/checks/task-ui-typecheck.ts +3 -3
- package/src/cli/analytics.ts +1 -0
- package/src/cli/check.ts +83 -12
- package/src/cli/commands/generation-model.ts +18 -0
- package/src/cli/commands/pipeline.ts +1 -1
- package/src/cli/commands/source.ts +14 -3
- package/src/cli/generation-models.ts +46 -0
- package/src/cli/program.ts +2 -0
- package/src/cli/project.ts +288 -138
- package/src/cli/trainings.ts +12 -0
- package/src/cli/ui.ts +3 -1
- package/src/client.ts +5 -0
- package/src/graph-rules.json +11 -0
- package/src/importer.ts +46 -40
- package/src/index.ts +4 -0
- package/src/validate.ts +183 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
2
|
+
import { describe } from "vitest";
|
|
3
|
+
|
|
4
|
+
import pipeline from "./pipeline";
|
|
5
|
+
import { moderateProduct } from "./nodes/moderate-product/node";
|
|
6
|
+
|
|
7
|
+
const product = {
|
|
8
|
+
product_id: 101,
|
|
9
|
+
image_link: "https://example.com/products/flag-tee.jpg",
|
|
10
|
+
product_category: "Apparel & Accessories > Clothing > Clothing Tops > T-Shirts",
|
|
11
|
+
product_description:
|
|
12
|
+
'Women\'s flag tee with "Victory for Freedonia" printed on the chest. Show where you stand!',
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
const verdict = {
|
|
16
|
+
moderation_label: "opinionated",
|
|
17
|
+
moderation_subcategory: "political",
|
|
18
|
+
moderation_rationale:
|
|
19
|
+
"The flag is paired with 'Victory for Freedonia', a slogan that takes a side.",
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
describe("product moderation example", () => {
|
|
23
|
+
test.ui(pipeline, moderateProduct, "requires a verdict and submits it", async ({ render }) => {
|
|
24
|
+
const session = await render({ data: product });
|
|
25
|
+
|
|
26
|
+
expect((await session.validationErrors()).map((error) => error.path)).toEqual(
|
|
27
|
+
expect.arrayContaining([
|
|
28
|
+
"moderation_label",
|
|
29
|
+
"moderation_subcategory",
|
|
30
|
+
"moderation_rationale",
|
|
31
|
+
]),
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
for (const [path, value] of Object.entries(verdict)) await session.setValue(path, value);
|
|
35
|
+
|
|
36
|
+
expect(await session.validationErrors()).toEqual([]);
|
|
37
|
+
await expect(session.submit()).resolves.toMatchObject(verdict);
|
|
38
|
+
await session.close();
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test.pipeline(pipeline, "writes the verdict back to the product row", async ({ renderUi }) => {
|
|
42
|
+
const session = await renderUi(moderateProduct, { data: product });
|
|
43
|
+
for (const [path, value] of Object.entries(verdict)) await session.setValue(path, value);
|
|
44
|
+
|
|
45
|
+
await expect(session.submit()).resolves.toMatchObject(verdict);
|
|
46
|
+
await session.close();
|
|
47
|
+
});
|
|
48
|
+
});
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { connect, definePipeline } from "@toloka-platform/sdk";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
imageLink,
|
|
5
|
+
moderationLabel,
|
|
6
|
+
moderationRationale,
|
|
7
|
+
moderationSubcategory,
|
|
8
|
+
productCategory,
|
|
9
|
+
productDescription,
|
|
10
|
+
productId,
|
|
11
|
+
} from "./fields";
|
|
12
|
+
import { collection } from "./nodes/collection/node";
|
|
13
|
+
import { moderateProduct } from "./nodes/moderate-product/node";
|
|
14
|
+
|
|
15
|
+
export const pipeline = definePipeline({
|
|
16
|
+
dataset: "products",
|
|
17
|
+
fields: [
|
|
18
|
+
productId,
|
|
19
|
+
productCategory,
|
|
20
|
+
imageLink,
|
|
21
|
+
productDescription,
|
|
22
|
+
moderationLabel,
|
|
23
|
+
moderationSubcategory,
|
|
24
|
+
moderationRationale,
|
|
25
|
+
],
|
|
26
|
+
nodes: [collection, moderateProduct],
|
|
27
|
+
connections: [
|
|
28
|
+
connect(collection.started, moderateProduct.input),
|
|
29
|
+
connect(moderateProduct.output, collection.completed),
|
|
30
|
+
],
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
export default pipeline;
|
|
@@ -1,9 +1,21 @@
|
|
|
1
1
|
import { field, type JsonSchema } from "@toloka-platform/sdk";
|
|
2
2
|
|
|
3
|
-
const
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
export const metricPickLabels = [
|
|
4
|
+
{ label: "Left", value: "video_1" },
|
|
5
|
+
{ label: "Right", value: "video_2" },
|
|
6
|
+
{ label: "Tie", value: "tie" },
|
|
7
|
+
];
|
|
8
|
+
|
|
9
|
+
export const pairPickLabels = [
|
|
10
|
+
{ label: "First video", value: "video_a" },
|
|
11
|
+
{ label: "Second video", value: "video_b" },
|
|
12
|
+
{ label: "Tie", value: "tie" },
|
|
13
|
+
];
|
|
14
|
+
|
|
15
|
+
const METRIC_PICK = metricPickLabels.map((option) => option.value);
|
|
16
|
+
const METRIC_PICK_OR_NONE = [...METRIC_PICK, "none"];
|
|
17
|
+
const PAIR_PICK = pairPickLabels.map((option) => option.value);
|
|
18
|
+
const PAIR_PICK_OR_NONE = [...PAIR_PICK, "none"];
|
|
7
19
|
|
|
8
20
|
const votes = (values: readonly string[]): JsonSchema => ({
|
|
9
21
|
type: "array",
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import { generation } from "@toloka-platform/sdk";
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
adjudicatedMetrics,
|
|
5
|
+
disagreedMetrics,
|
|
6
|
+
pairPickLabels,
|
|
7
|
+
prompt,
|
|
8
|
+
videoA,
|
|
9
|
+
videoB,
|
|
10
|
+
} from "../../fields";
|
|
4
11
|
|
|
5
12
|
export const adjudicate = generation("adjudicate", {
|
|
6
13
|
about: {
|
|
@@ -64,11 +71,79 @@ export const adjudicate = generation("adjudicate", {
|
|
|
64
71
|
},
|
|
65
72
|
],
|
|
66
73
|
ui: {
|
|
67
|
-
code:
|
|
74
|
+
code: `<View.Flex vertical gap={'1rem'}>
|
|
75
|
+
<View.Card title={'Prompt'} variant={'info'}>
|
|
76
|
+
<View.Text>{actions.get('prompt', '')}</View.Text>
|
|
77
|
+
</View.Card>
|
|
78
|
+
<View.Row gap={'1rem'}>
|
|
79
|
+
<View.Col span={12}>
|
|
80
|
+
<View.Card title={'First video'}>
|
|
81
|
+
<View.Video url={actions.get('video_a', '')} />
|
|
82
|
+
</View.Card>
|
|
83
|
+
</View.Col>
|
|
84
|
+
<View.Col span={12}>
|
|
85
|
+
<View.Card title={'Second video'}>
|
|
86
|
+
<View.Video url={actions.get('video_b', '')} />
|
|
87
|
+
</View.Card>
|
|
88
|
+
</View.Col>
|
|
89
|
+
</View.Row>
|
|
90
|
+
<View.Card title={'Decide the disputed metrics'}>
|
|
91
|
+
<View.Flex vertical gap={'1rem'}>
|
|
92
|
+
{actions.get('disagreed_metrics', []).includes('image_text_alignment') && (
|
|
93
|
+
<Field.RadioGroup
|
|
94
|
+
path={'adjudicated_metrics.image_text_alignment'}
|
|
95
|
+
label={'Imagery matches the prompt'}
|
|
96
|
+
options={CONST.PAIR_PICK}
|
|
97
|
+
validation={[{ required: true }]}
|
|
98
|
+
/>
|
|
99
|
+
)}
|
|
100
|
+
{actions.get('disagreed_metrics', []).includes('motion_text_alignment') && (
|
|
101
|
+
<Field.RadioGroup
|
|
102
|
+
path={'adjudicated_metrics.motion_text_alignment'}
|
|
103
|
+
label={'Motion matches the prompt'}
|
|
104
|
+
options={CONST.PAIR_PICK}
|
|
105
|
+
validation={[{ required: true }]}
|
|
106
|
+
/>
|
|
107
|
+
)}
|
|
108
|
+
{actions.get('disagreed_metrics', []).includes('image_quality') && (
|
|
109
|
+
<Field.RadioGroup
|
|
110
|
+
path={'adjudicated_metrics.image_quality'}
|
|
111
|
+
label={'Image quality'}
|
|
112
|
+
options={CONST.PAIR_PICK}
|
|
113
|
+
validation={[{ required: true }]}
|
|
114
|
+
/>
|
|
115
|
+
)}
|
|
116
|
+
{actions.get('disagreed_metrics', []).includes('motion_quality') && (
|
|
117
|
+
<Field.RadioGroup
|
|
118
|
+
path={'adjudicated_metrics.motion_quality'}
|
|
119
|
+
label={'Motion quality'}
|
|
120
|
+
options={CONST.PAIR_PICK}
|
|
121
|
+
validation={[{ required: true }]}
|
|
122
|
+
/>
|
|
123
|
+
)}
|
|
124
|
+
{actions.get('disagreed_metrics', []).includes('subject_background_consistency') && (
|
|
125
|
+
<Field.RadioGroup
|
|
126
|
+
path={'adjudicated_metrics.subject_background_consistency'}
|
|
127
|
+
label={'Subject and background consistency'}
|
|
128
|
+
options={CONST.PAIR_PICK}
|
|
129
|
+
validation={[{ required: true }]}
|
|
130
|
+
/>
|
|
131
|
+
)}
|
|
132
|
+
{actions.get('disagreed_metrics', []).includes('video_realism') && (
|
|
133
|
+
<Field.RadioGroup
|
|
134
|
+
path={'adjudicated_metrics.video_realism'}
|
|
135
|
+
label={'Realism'}
|
|
136
|
+
options={CONST.PAIR_PICK}
|
|
137
|
+
validation={[{ required: true }]}
|
|
138
|
+
/>
|
|
139
|
+
)}
|
|
140
|
+
</View.Flex>
|
|
141
|
+
</View.Card>
|
|
142
|
+
</View.Flex>`,
|
|
68
143
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4","disagreed_metrics":["image_quality","motion_quality"]}',
|
|
69
144
|
preprocess: "return {}",
|
|
70
145
|
postprocess: "return {};",
|
|
71
|
-
constants:
|
|
146
|
+
constants: JSON.stringify({ PAIR_PICK: pairPickLabels }),
|
|
72
147
|
},
|
|
73
148
|
qualityAssurance: {
|
|
74
149
|
qaMode: "llm_only",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { generation } from "@toloka-platform/sdk";
|
|
2
2
|
|
|
3
|
-
import { opAdjudicatedValue, prompt, videoA, videoB } from "../../fields";
|
|
3
|
+
import { opAdjudicatedValue, pairPickLabels, prompt, videoA, videoB } from "../../fields";
|
|
4
4
|
|
|
5
5
|
export const adjudicateOp = generation("adjudicate_op", {
|
|
6
6
|
about: {
|
|
@@ -34,6 +34,7 @@ export const adjudicateOp = generation("adjudicate_op", {
|
|
|
34
34
|
field: opAdjudicatedValue,
|
|
35
35
|
entityType: "SINGLE_LABEL",
|
|
36
36
|
jsonPath: "$.op_adjudicated_value",
|
|
37
|
+
allowedLabels: pairPickLabels,
|
|
37
38
|
description: "The final overall preference: the first video, the second, or a tie.",
|
|
38
39
|
},
|
|
39
40
|
],
|
|
@@ -50,11 +51,35 @@ export const adjudicateOp = generation("adjudicate_op", {
|
|
|
50
51
|
},
|
|
51
52
|
],
|
|
52
53
|
ui: {
|
|
53
|
-
code:
|
|
54
|
+
code: `<View.Flex vertical gap={'1rem'}>
|
|
55
|
+
<View.Card title={'Prompt'} variant={'info'}>
|
|
56
|
+
<View.Text>{actions.get('prompt', '')}</View.Text>
|
|
57
|
+
</View.Card>
|
|
58
|
+
<View.Row gap={'1rem'}>
|
|
59
|
+
<View.Col span={12}>
|
|
60
|
+
<View.Card title={'First video'}>
|
|
61
|
+
<View.Video url={actions.get('video_a', '')} />
|
|
62
|
+
</View.Card>
|
|
63
|
+
</View.Col>
|
|
64
|
+
<View.Col span={12}>
|
|
65
|
+
<View.Card title={'Second video'}>
|
|
66
|
+
<View.Video url={actions.get('video_b', '')} />
|
|
67
|
+
</View.Card>
|
|
68
|
+
</View.Col>
|
|
69
|
+
</View.Row>
|
|
70
|
+
<View.Card>
|
|
71
|
+
<Field.RadioGroup
|
|
72
|
+
path={'op_adjudicated_value'}
|
|
73
|
+
label={'Which video is better overall?'}
|
|
74
|
+
options={CONST.PAIR_PICK}
|
|
75
|
+
validation={[{ required: true }]}
|
|
76
|
+
/>
|
|
77
|
+
</View.Card>
|
|
78
|
+
</View.Flex>`,
|
|
54
79
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
|
|
55
80
|
preprocess: "return {}",
|
|
56
81
|
postprocess: "return {};",
|
|
57
|
-
constants:
|
|
82
|
+
constants: JSON.stringify({ PAIR_PICK: pairPickLabels }),
|
|
58
83
|
},
|
|
59
84
|
qualityAssurance: {
|
|
60
85
|
qaMode: "llm_only",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { quorum } from "@toloka-platform/sdk";
|
|
2
2
|
|
|
3
|
-
import { overallManualVotes, prompt, videoA, videoB } from "../../fields";
|
|
3
|
+
import { overallManualVotes, pairPickLabels, prompt, videoA, videoB } from "../../fields";
|
|
4
4
|
|
|
5
5
|
export const overallPreferencePick = quorum("overall_preference_pick", {
|
|
6
6
|
about: { name: "overall_preference_pick" },
|
|
@@ -30,6 +30,7 @@ export const overallPreferencePick = quorum("overall_preference_pick", {
|
|
|
30
30
|
field: overallManualVotes,
|
|
31
31
|
entityType: "SINGLE_LABEL",
|
|
32
32
|
jsonPath: "$.overall_manual_votes",
|
|
33
|
+
allowedLabels: pairPickLabels,
|
|
33
34
|
description: "One overall preference per rater, collected as an array.",
|
|
34
35
|
},
|
|
35
36
|
],
|
|
@@ -48,11 +49,35 @@ export const overallPreferencePick = quorum("overall_preference_pick", {
|
|
|
48
49
|
instruction:
|
|
49
50
|
"Watch both videos in full, then pick the one you would ship. This is a single judgment about the whole video, not a sum of its parts. Use tie only when you genuinely have no preference. Unlike the metric rating task, this one shows the pair in its original order, so your pick needs no un-shuffling downstream.",
|
|
50
51
|
ui: {
|
|
51
|
-
code:
|
|
52
|
+
code: `<View.Flex vertical gap={'1rem'}>
|
|
53
|
+
<View.Card title={'Prompt'} variant={'info'}>
|
|
54
|
+
<View.Text>{actions.get('prompt', '')}</View.Text>
|
|
55
|
+
</View.Card>
|
|
56
|
+
<View.Row gap={'1rem'}>
|
|
57
|
+
<View.Col span={12}>
|
|
58
|
+
<View.Card title={'First video'}>
|
|
59
|
+
<View.Video url={actions.get('video_a', '')} />
|
|
60
|
+
</View.Card>
|
|
61
|
+
</View.Col>
|
|
62
|
+
<View.Col span={12}>
|
|
63
|
+
<View.Card title={'Second video'}>
|
|
64
|
+
<View.Video url={actions.get('video_b', '')} />
|
|
65
|
+
</View.Card>
|
|
66
|
+
</View.Col>
|
|
67
|
+
</View.Row>
|
|
68
|
+
<View.Card>
|
|
69
|
+
<Field.RadioGroup
|
|
70
|
+
path={'overall_manual_votes'}
|
|
71
|
+
label={'Which video is better overall?'}
|
|
72
|
+
options={CONST.PAIR_PICK}
|
|
73
|
+
validation={[{ required: true }]}
|
|
74
|
+
/>
|
|
75
|
+
</View.Card>
|
|
76
|
+
</View.Flex>`,
|
|
52
77
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
|
|
53
78
|
preprocess: "return {}",
|
|
54
79
|
postprocess: "return {};",
|
|
55
|
-
constants:
|
|
80
|
+
constants: JSON.stringify({ PAIR_PICK: pairPickLabels }),
|
|
56
81
|
},
|
|
57
82
|
onboarding: {
|
|
58
83
|
enabled: true,
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
imageQuality,
|
|
5
5
|
imageTextAlignment,
|
|
6
6
|
leftVideo,
|
|
7
|
+
metricPickLabels,
|
|
7
8
|
motionQuality,
|
|
8
9
|
motionTextAlignment,
|
|
9
10
|
prompt,
|
|
@@ -39,36 +40,42 @@ export const ratePair = quorum("rate_pair", {
|
|
|
39
40
|
field: imageTextAlignment,
|
|
40
41
|
entityType: "SINGLE_LABEL",
|
|
41
42
|
jsonPath: "$.image_text_alignment",
|
|
43
|
+
allowedLabels: metricPickLabels,
|
|
42
44
|
description: "Which video matches the prompt in its imagery. One element per rater.",
|
|
43
45
|
},
|
|
44
46
|
{
|
|
45
47
|
field: motionTextAlignment,
|
|
46
48
|
entityType: "SINGLE_LABEL",
|
|
47
49
|
jsonPath: "$.motion_text_alignment",
|
|
50
|
+
allowedLabels: metricPickLabels,
|
|
48
51
|
description: "Which video matches the prompt in its motion. One element per rater.",
|
|
49
52
|
},
|
|
50
53
|
{
|
|
51
54
|
field: imageQuality,
|
|
52
55
|
entityType: "SINGLE_LABEL",
|
|
53
56
|
jsonPath: "$.image_quality",
|
|
57
|
+
allowedLabels: metricPickLabels,
|
|
54
58
|
description: "Which video looks better frame by frame. One element per rater.",
|
|
55
59
|
},
|
|
56
60
|
{
|
|
57
61
|
field: motionQuality,
|
|
58
62
|
entityType: "SINGLE_LABEL",
|
|
59
63
|
jsonPath: "$.motion_quality",
|
|
64
|
+
allowedLabels: metricPickLabels,
|
|
60
65
|
description: "Which video moves more plausibly. One element per rater.",
|
|
61
66
|
},
|
|
62
67
|
{
|
|
63
68
|
field: subjectBackgroundConsistency,
|
|
64
69
|
entityType: "SINGLE_LABEL",
|
|
65
70
|
jsonPath: "$.subject_background_consistency",
|
|
71
|
+
allowedLabels: metricPickLabels,
|
|
66
72
|
description: "Which video keeps its subject and background stable. One element per rater.",
|
|
67
73
|
},
|
|
68
74
|
{
|
|
69
75
|
field: videoRealism,
|
|
70
76
|
entityType: "SINGLE_LABEL",
|
|
71
77
|
jsonPath: "$.video_realism",
|
|
78
|
+
allowedLabels: metricPickLabels,
|
|
72
79
|
description: "Which video reads as more real. One element per rater.",
|
|
73
80
|
},
|
|
74
81
|
],
|
|
@@ -96,11 +103,67 @@ export const ratePair = quorum("rate_pair", {
|
|
|
96
103
|
instruction:
|
|
97
104
|
"Watch both videos in full before answering. For each metric, pick the video that is better on that metric alone, or tie when they are equal. The two videos are presented in random order, so left and right carry no meaning. Different metrics can have different winners; that is expected.",
|
|
98
105
|
ui: {
|
|
99
|
-
code:
|
|
106
|
+
code: `<View.Flex vertical gap={'1rem'}>
|
|
107
|
+
<View.Card title={'Prompt'} variant={'info'}>
|
|
108
|
+
<View.Text>{actions.get('prompt', '')}</View.Text>
|
|
109
|
+
</View.Card>
|
|
110
|
+
<View.Row gap={'1rem'}>
|
|
111
|
+
<View.Col span={12}>
|
|
112
|
+
<View.Card title={'Left'}>
|
|
113
|
+
<View.Video url={actions.get('left_video', '')} />
|
|
114
|
+
</View.Card>
|
|
115
|
+
</View.Col>
|
|
116
|
+
<View.Col span={12}>
|
|
117
|
+
<View.Card title={'Right'}>
|
|
118
|
+
<View.Video url={actions.get('right_video', '')} />
|
|
119
|
+
</View.Card>
|
|
120
|
+
</View.Col>
|
|
121
|
+
</View.Row>
|
|
122
|
+
<View.Card title={'Score each metric on its own'}>
|
|
123
|
+
<View.Flex vertical gap={'1rem'}>
|
|
124
|
+
<Field.RadioGroup
|
|
125
|
+
path={'image_text_alignment'}
|
|
126
|
+
label={'Which video matches the prompt in its imagery?'}
|
|
127
|
+
options={CONST.METRIC_PICK}
|
|
128
|
+
validation={[{ required: true }]}
|
|
129
|
+
/>
|
|
130
|
+
<Field.RadioGroup
|
|
131
|
+
path={'motion_text_alignment'}
|
|
132
|
+
label={'Which video matches the prompt in its motion?'}
|
|
133
|
+
options={CONST.METRIC_PICK}
|
|
134
|
+
validation={[{ required: true }]}
|
|
135
|
+
/>
|
|
136
|
+
<Field.RadioGroup
|
|
137
|
+
path={'image_quality'}
|
|
138
|
+
label={'Which video looks better frame by frame?'}
|
|
139
|
+
options={CONST.METRIC_PICK}
|
|
140
|
+
validation={[{ required: true }]}
|
|
141
|
+
/>
|
|
142
|
+
<Field.RadioGroup
|
|
143
|
+
path={'motion_quality'}
|
|
144
|
+
label={'Which video moves more plausibly?'}
|
|
145
|
+
options={CONST.METRIC_PICK}
|
|
146
|
+
validation={[{ required: true }]}
|
|
147
|
+
/>
|
|
148
|
+
<Field.RadioGroup
|
|
149
|
+
path={'subject_background_consistency'}
|
|
150
|
+
label={'Which video keeps its subject and background stable?'}
|
|
151
|
+
options={CONST.METRIC_PICK}
|
|
152
|
+
validation={[{ required: true }]}
|
|
153
|
+
/>
|
|
154
|
+
<Field.RadioGroup
|
|
155
|
+
path={'video_realism'}
|
|
156
|
+
label={'Which video reads as more real?'}
|
|
157
|
+
options={CONST.METRIC_PICK}
|
|
158
|
+
validation={[{ required: true }]}
|
|
159
|
+
/>
|
|
160
|
+
</View.Flex>
|
|
161
|
+
</View.Card>
|
|
162
|
+
</View.Flex>`,
|
|
100
163
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","left_video":"https://example.com/video-left.mp4","right_video":"https://example.com/video-right.mp4"}',
|
|
101
164
|
preprocess: "return {}",
|
|
102
165
|
postprocess: "return {};",
|
|
103
|
-
constants:
|
|
166
|
+
constants: JSON.stringify({ METRIC_PICK: metricPickLabels }),
|
|
104
167
|
},
|
|
105
168
|
onboarding: {
|
|
106
169
|
enabled: true,
|
package/openapi/gen/api.ts
CHANGED
|
@@ -41,6 +41,7 @@ import type {
|
|
|
41
41
|
ListDatasetItems200,
|
|
42
42
|
ListDatasetItemsParams,
|
|
43
43
|
ListDeliveryOrders200,
|
|
44
|
+
ListGenerationModels200,
|
|
44
45
|
ListModelVersions200,
|
|
45
46
|
ListModelVersionsParams,
|
|
46
47
|
ListPipelineChecks200,
|
|
@@ -721,6 +722,27 @@ export const listAudienceSpecializations = async (params?: ListAudienceSpecializ
|
|
|
721
722
|
|
|
722
723
|
|
|
723
724
|
|
|
725
|
+
export const getListGenerationModelsUrl = () => {
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
return `/api/v2-beta/generation-models`
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
export const listGenerationModels = async ( options?: RequestInit): Promise<ListGenerationModels200> => {
|
|
734
|
+
|
|
735
|
+
return potatoFetch<ListGenerationModels200>(getListGenerationModelsUrl(),
|
|
736
|
+
{
|
|
737
|
+
...options,
|
|
738
|
+
method: 'GET'
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
}
|
|
742
|
+
);}
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
|
|
724
746
|
export const getGetCurrentUserUrl = () => {
|
|
725
747
|
|
|
726
748
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { GenerationModelId } from './generationModelId';
|
|
2
|
+
import type { GenerationModelDefaultReasoningEffort } from './generationModelDefaultReasoningEffort';
|
|
3
|
+
import type { GenerationBudget } from './generationBudget';
|
|
4
|
+
|
|
5
|
+
export interface GenerationModel {
|
|
6
|
+
id: GenerationModelId;
|
|
7
|
+
displayName: string;
|
|
8
|
+
description: string;
|
|
9
|
+
recommended: boolean;
|
|
10
|
+
defaultReasoningEffort?: GenerationModelDefaultReasoningEffort;
|
|
11
|
+
budgets: GenerationBudget[];
|
|
12
|
+
}
|
|
@@ -162,6 +162,11 @@ export * from './exportJobResult';
|
|
|
162
162
|
export * from './exportJobResultDownloadUrl';
|
|
163
163
|
export * from './exportJobStatus';
|
|
164
164
|
export * from './exportPipeline200';
|
|
165
|
+
export * from './generationBudget';
|
|
166
|
+
export * from './generationBudgetReasoningEffort';
|
|
167
|
+
export * from './generationModel';
|
|
168
|
+
export * from './generationModelDefaultReasoningEffort';
|
|
169
|
+
export * from './generationModelId';
|
|
165
170
|
export * from './generationNode';
|
|
166
171
|
export * from './generationNodeSettings';
|
|
167
172
|
export * from './generationNodeSettingsAbout';
|
|
@@ -284,6 +289,7 @@ export * from './inferredFieldJsonSchema';
|
|
|
284
289
|
export * from './inferredFieldJsonSchemaAdditionalProperties';
|
|
285
290
|
export * from './inferredFieldJsonSchemaType';
|
|
286
291
|
export * from './inferredFieldJsonSchemaTypeAnyOfItem';
|
|
292
|
+
export * from './invalidFieldSchema';
|
|
287
293
|
export * from './jsonSchema';
|
|
288
294
|
export * from './jsonSchemaAdditionalProperties';
|
|
289
295
|
export * from './jsonSchemaType';
|
|
@@ -296,6 +302,8 @@ export * from './listDatasetFieldsParams';
|
|
|
296
302
|
export * from './listDatasetItems200';
|
|
297
303
|
export * from './listDatasetItemsParams';
|
|
298
304
|
export * from './listDeliveryOrders200';
|
|
305
|
+
export * from './listGenerationModels200';
|
|
306
|
+
export * from './listGenerationModels200DefaultGenerationModelId';
|
|
299
307
|
export * from './listModelVersions200';
|
|
300
308
|
export * from './listModelVersionsParams';
|
|
301
309
|
export * from './listPipelineChecks200';
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ListGenerationModels200DefaultGenerationModelId } from './listGenerationModels200DefaultGenerationModelId';
|
|
2
|
+
import type { GenerationModel } from './generationModel';
|
|
3
|
+
|
|
4
|
+
export type ListGenerationModels200 = {
|
|
5
|
+
defaultGenerationModelId: ListGenerationModels200DefaultGenerationModelId;
|
|
6
|
+
generationModels: GenerationModel[];
|
|
7
|
+
};
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
|
|
2
|
+
export type ListGenerationModels200DefaultGenerationModelId = 'gpt-5.4' | 'gpt-5.4-mini' | 'claude-sonnet-4-6' | 'gemini-3.1-pro-preview' | 'gemini-3.5-flash' | 'gemini-3.7-flash' | 'claude-sonnet-5' | 'claude-opus-5' | 'claude-fable-5' | 'gpt-5.6-terra' | 'gpt-5.6-luna' | 'gpt-5.6-sol';
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { PipelineGraphSnapshotGraph } from './pipelineGraphSnapshotGraph';
|
|
2
2
|
import type { PipelineGraphVersion } from './pipelineGraphVersion';
|
|
3
|
+
import type { InvalidFieldSchema } from './invalidFieldSchema';
|
|
3
4
|
|
|
4
5
|
export interface PipelineGraphSnapshot {
|
|
5
6
|
graph: PipelineGraphSnapshotGraph;
|
|
6
7
|
version: PipelineGraphVersion;
|
|
8
|
+
invalidFieldSchemas: InvalidFieldSchema[];
|
|
7
9
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toloka-platform/sdk",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.3",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Build, inspect, and operate Toloka pipeline projects as TypeScript source. The public API may change between minor versions before 1.0.",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -77,6 +77,6 @@
|
|
|
77
77
|
"bun": ">=1.4.0"
|
|
78
78
|
},
|
|
79
79
|
"tolokaBuild": {
|
|
80
|
-
"commit": "
|
|
80
|
+
"commit": "113d50a57cb089999d1f93567c5c51afff932e67"
|
|
81
81
|
}
|
|
82
82
|
}
|