@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -15
- package/dist/cli/toloka.js +209 -121
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +20 -9
- package/docs/cli-v2.md +3 -3
- package/docs/cli.md +44 -16
- package/docs/getting-started.md +7 -12
- package/docs/limitations.md +6 -6
- package/docs/testing-examples.md +248 -0
- package/docs/testing.md +115 -0
- package/examples/face-image-collection/README.md +7 -1
- package/examples/face-image-collection/fields.ts +20 -8
- package/examples/face-image-collection/nodes/annotate-faces/node.ts +1 -9
- package/examples/face-image-collection/nodes/order-faces/node.ts +3 -5
- package/examples/face-image-collection/pipeline.browser.integration.spec.ts +78 -0
- package/examples/video-pair-rating/README.md +14 -6
- package/examples/video-pair-rating/nodes/adjudicate/node.ts +1 -3
- package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +1 -3
- package/examples/video-pair-rating/nodes/adjudication_match/node.ts +15 -17
- package/examples/video-pair-rating/nodes/assemble/node.ts +25 -27
- package/examples/video-pair-rating/nodes/cascade/node.ts +24 -26
- package/examples/video-pair-rating/nodes/finalize_overall/node.ts +17 -19
- package/examples/video-pair-rating/nodes/merge_adjudication/node.ts +13 -15
- package/examples/video-pair-rating/nodes/merge_op_adjudication/node.ts +12 -14
- package/examples/video-pair-rating/nodes/op_quorum_resolve/node.ts +13 -15
- package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +1 -1
- package/examples/video-pair-rating/nodes/rate_pair/node.ts +1 -1
- package/examples/video-pair-rating/nodes/route_adjudication/node.ts +1 -3
- package/examples/video-pair-rating/nodes/route_op_adjudication/node.ts +5 -7
- package/examples/video-pair-rating/nodes/route_overall/node.ts +1 -3
- package/examples/video-pair-rating/nodes/shuffle_ab/node.ts +11 -13
- package/examples/video-pair-rating/pipeline.browser.integration.spec.ts +267 -0
- package/openapi/gen/api.ts +48 -11
- package/openapi/gen/models/createDatasetExportBody.ts +3 -3
- package/openapi/gen/models/createDatasetExportBodyOneOf.ts +6 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfFormat.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThree.ts +10 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeContract.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeFormat.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilter.ts +9 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/datasetExportReport.ts +9 -0
- package/openapi/gen/models/datasetExportReportErrorsItem.ts +5 -0
- package/openapi/gen/models/datasetExportReportTokens.ts +7 -0
- package/openapi/gen/models/datasetItem.ts +0 -2
- package/openapi/gen/models/exportJob.ts +2 -0
- package/openapi/gen/models/exportJobReport.ts +3 -0
- package/openapi/gen/models/index.ts +48 -1
- package/openapi/gen/models/sampleDatasetItemsBody.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOf.ts +8 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilter.ts +9 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethree.ts +8 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilter.ts +9 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItem.ts +6 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItem.ts +12 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemOperator.ts +2 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
- package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemValue.ts +2 -0
- package/openapi/gen/models/sampleJob.ts +13 -0
- package/openapi/gen/models/sampleJobStatus.ts +2 -0
- package/openapi/gen/models/sampleResult.ts +5 -0
- package/package.json +23 -2
- package/skills/catalog.json +14 -6
- package/skills/toloka-build/SKILL.md +35 -12
- package/skills/toloka-build/references/node-semantics.md +32 -29
- package/skills/toloka-build/references/workflow-and-cli.md +61 -27
- package/skills/toloka-report-issue/references/report-template.md +1 -1
- package/src/authoring/index.ts +27 -26
- package/src/checks/ast.ts +32 -0
- package/src/checks/index.ts +14 -0
- package/src/checks/postprocess-collisions.ts +145 -0
- package/src/checks/task-ui-declarations.generated.ts +19 -0
- package/src/checks/task-ui-typecheck.ts +345 -0
- package/src/checks/typescript-vfs.d.ts +26 -0
- package/src/checks/typescript-vfs.js +5 -0
- package/src/cli/analytics.ts +9 -19
- package/src/cli/auto-update.ts +367 -0
- package/src/cli/boolean-flag.ts +13 -0
- package/src/cli/check.ts +977 -0
- package/src/cli/credentials.ts +3 -0
- package/src/cli/program.ts +184 -20
- package/src/cli/project-context.ts +14 -4
- package/src/cli/runs.ts +21 -11
- package/src/cli/settings.ts +93 -0
- package/src/cli/skill.ts +17 -6
- package/src/importer.ts +12 -16
- package/src/testing/code-node.ts +160 -0
- package/src/testing/index.ts +15 -0
- package/src/testing/task-ui-browser.ts +120 -0
- package/src/testing/task-ui-validation.ts +135 -0
- package/src/testing/task-ui.ts +557 -0
- package/src/testing/test-evidence.ts +170 -0
- package/src/testing/types.ts +141 -0
- package/openapi/gen/models/datasetItemDuplicateOfSubmissionId.ts +0 -2
|
@@ -15,18 +15,16 @@ export const mergeOpAdjudication = code("merge_op_adjudication", {
|
|
|
15
15
|
"Consolidates the overall preference branch: if adjudication ran, emit the adjudicator pick and resolution_method=adjudication; otherwise emit the quorum majority and resolution_method=agreement.",
|
|
16
16
|
},
|
|
17
17
|
envVars: [],
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
],
|
|
29
|
-
},
|
|
18
|
+
input: [
|
|
19
|
+
{ field: opNeedsAdjudication, optional: false },
|
|
20
|
+
{ field: opManualValue, optional: true },
|
|
21
|
+
{ field: opAdjudicatedValue, optional: true },
|
|
22
|
+
],
|
|
23
|
+
output: [
|
|
24
|
+
{ field: finalOpValue, optional: false },
|
|
25
|
+
{ field: finalOpResolutionMethod, optional: false },
|
|
26
|
+
{ field: finalOpAdjudicatorId, optional: false },
|
|
27
|
+
],
|
|
30
28
|
code: {
|
|
31
29
|
code: `from main_types import Inputs, Outputs
|
|
32
30
|
|
|
@@ -38,8 +36,8 @@ async def main(inputs: Inputs) -> Outputs:
|
|
|
38
36
|
"final_op_value": inputs["op_adjudicated_value"],
|
|
39
37
|
"final_op_resolution_method": "adjudication",
|
|
40
38
|
# Platform limitation (mirrors annotator_id_1/2 in assemble): the
|
|
41
|
-
# OP adjudicator id is not surfaced as
|
|
42
|
-
#
|
|
39
|
+
# OP adjudicator id is not surfaced as an output field from the
|
|
40
|
+
# generation node in this pilot. Emit null.
|
|
43
41
|
"final_op_adjudicator_id": None,
|
|
44
42
|
}
|
|
45
43
|
return {
|
|
@@ -18,21 +18,19 @@ export const opQuorumResolve = code("op_quorum_resolve", {
|
|
|
18
18
|
"Resolves three overall preference votes from the quorum into a majority. If fewer than two votes agree, flags for adjudication and packages the disagreed task payload.",
|
|
19
19
|
},
|
|
20
20
|
envVars: [],
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
],
|
|
35
|
-
},
|
|
21
|
+
input: [
|
|
22
|
+
{ field: prompt, optional: false },
|
|
23
|
+
{ field: videoA, optional: false },
|
|
24
|
+
{ field: videoB, optional: false },
|
|
25
|
+
{ field: swap, optional: false },
|
|
26
|
+
{ field: overallManualVotes, optional: false },
|
|
27
|
+
],
|
|
28
|
+
output: [
|
|
29
|
+
{ field: opManualValue, optional: true },
|
|
30
|
+
{ field: opNeedsAdjudication, optional: false },
|
|
31
|
+
{ field: disagreedOpTask, optional: false },
|
|
32
|
+
{ field: raterOpVotes, optional: false },
|
|
33
|
+
],
|
|
36
34
|
code: {
|
|
37
35
|
code: `from main_types import Inputs, Outputs
|
|
38
36
|
|
|
@@ -48,7 +48,7 @@ export const overallPreferencePick = quorum("overall_preference_pick", {
|
|
|
48
48
|
instruction:
|
|
49
49
|
"Watch both videos in full, then pick the one you would ship. This is a single judgment about the whole video, not a sum of its parts. Use tie only when you genuinely have no preference. Unlike the metric rating task, this one shows the pair in its original order, so your pick needs no un-shuffling downstream.",
|
|
50
50
|
ui: {
|
|
51
|
-
code: "<Field.
|
|
51
|
+
code: "<Field.RadioGroup path={'overall_manual_votes'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
|
|
52
52
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
|
|
53
53
|
preprocess: "return {}",
|
|
54
54
|
postprocess: "return {};",
|
|
@@ -96,7 +96,7 @@ export const ratePair = quorum("rate_pair", {
|
|
|
96
96
|
instruction:
|
|
97
97
|
"Watch both videos in full before answering. For each metric, pick the video that is better on that metric alone, or tie when they are equal. The two videos are presented in random order, so left and right carry no meaning. Different metrics can have different winners; that is expected.",
|
|
98
98
|
ui: {
|
|
99
|
-
code: "<View.
|
|
99
|
+
code: "<View.Card><Field.RadioGroup path={'image_text_alignment'} label={'Which video matches the prompt in its imagery?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'motion_text_alignment'} label={'Which video matches the prompt in its motion?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'image_quality'} label={'Which video looks better frame by frame?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'motion_quality'} label={'Which video moves more plausibly?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'subject_background_consistency'} label={'Which video keeps its subject and background stable?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'video_realism'} label={'Which video reads as more real?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /></View.Card>",
|
|
100
100
|
data: '{"prompt":"a red fox crossing a snowy field at dawn","left_video":"https://example.com/video-left.mp4","right_video":"https://example.com/video-right.mp4"}',
|
|
101
101
|
preprocess: "return {}",
|
|
102
102
|
postprocess: "return {};",
|
|
@@ -20,11 +20,9 @@ export const routeOpAdjudication = match("route_op_adjudication", {
|
|
|
20
20
|
],
|
|
21
21
|
},
|
|
22
22
|
],
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
],
|
|
29
|
-
},
|
|
23
|
+
input: [
|
|
24
|
+
{ field: opNeedsAdjudication, optional: false },
|
|
25
|
+
{ field: disagreedOpTask, optional: true },
|
|
26
|
+
{ field: opManualValue, optional: true },
|
|
27
|
+
],
|
|
30
28
|
});
|
|
@@ -8,19 +8,17 @@ export const shuffleAb = code("shuffle_ab", {
|
|
|
8
8
|
"Mints a deterministic record_id from a hash of the prompt and video pair. Swaps the left/right video assignment based on the hash parity to prevent position bias in rater interfaces.",
|
|
9
9
|
},
|
|
10
10
|
envVars: [],
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
],
|
|
23
|
-
},
|
|
11
|
+
input: [
|
|
12
|
+
{ field: f.prompt, optional: false },
|
|
13
|
+
{ field: f.videoA, optional: false },
|
|
14
|
+
{ field: f.videoB, optional: false },
|
|
15
|
+
],
|
|
16
|
+
output: [
|
|
17
|
+
{ field: f.recordId, optional: false },
|
|
18
|
+
{ field: f.leftVideo, optional: false },
|
|
19
|
+
{ field: f.rightVideo, optional: false },
|
|
20
|
+
{ field: f.swap, optional: false },
|
|
21
|
+
],
|
|
24
22
|
code: {
|
|
25
23
|
code: `import hashlib
|
|
26
24
|
from main_types import Inputs, Outputs
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
2
|
+
import { describe } from "vitest";
|
|
3
|
+
|
|
4
|
+
import pipeline from "./pipeline";
|
|
5
|
+
import { adjudicate } from "./nodes/adjudicate/node";
|
|
6
|
+
import { adjudicateOp } from "./nodes/adjudicate_op/node";
|
|
7
|
+
import { adjudicationMatch } from "./nodes/adjudication_match/node";
|
|
8
|
+
import { assemble } from "./nodes/assemble/node";
|
|
9
|
+
import { cascade } from "./nodes/cascade/node";
|
|
10
|
+
import { finalizeOverall } from "./nodes/finalize_overall/node";
|
|
11
|
+
import { mergeAdjudication } from "./nodes/merge_adjudication/node";
|
|
12
|
+
import { mergeOpAdjudication } from "./nodes/merge_op_adjudication/node";
|
|
13
|
+
import { opQuorumResolve } from "./nodes/op_quorum_resolve/node";
|
|
14
|
+
import { overallPreferencePick } from "./nodes/overall_preference_pick/node";
|
|
15
|
+
import { ratePair } from "./nodes/rate_pair/node";
|
|
16
|
+
import { shuffleAb } from "./nodes/shuffle_ab/node";
|
|
17
|
+
|
|
18
|
+
const item = {
|
|
19
|
+
prompt: "a fox crossing a snowy field",
|
|
20
|
+
video_a: "https://example.com/video-a.mp4",
|
|
21
|
+
video_b: "https://example.com/video-b.mp4",
|
|
22
|
+
};
|
|
23
|
+
const metricNames = [
|
|
24
|
+
"image_text_alignment",
|
|
25
|
+
"motion_text_alignment",
|
|
26
|
+
"image_quality",
|
|
27
|
+
"motion_quality",
|
|
28
|
+
"subject_background_consistency",
|
|
29
|
+
"video_realism",
|
|
30
|
+
];
|
|
31
|
+
const canonicalMetrics = Object.fromEntries(metricNames.map((name) => [name, "video_a"]));
|
|
32
|
+
const metricVotes = Object.fromEntries(metricNames.map((name) => [name, ["video_1", "video_1"]]));
|
|
33
|
+
|
|
34
|
+
describe("video pair rating example", () => {
|
|
35
|
+
test.code(pipeline, shuffleAb, "shuffles a pair deterministically", async ({ run }) => {
|
|
36
|
+
const result = await run(item);
|
|
37
|
+
|
|
38
|
+
expect(result).toMatchObject({
|
|
39
|
+
record_id: expect.stringMatching(/^[a-f0-9]{16}$/u),
|
|
40
|
+
swap: expect.any(Boolean),
|
|
41
|
+
});
|
|
42
|
+
expect(new Set([result.left_video, result.right_video])).toEqual(
|
|
43
|
+
new Set([item.video_a, item.video_b]),
|
|
44
|
+
);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test.code(pipeline, cascade, "resolves unanimous metric votes", async ({ run }) => {
|
|
48
|
+
await expect(run({ swap: false, ...metricVotes })).resolves.toMatchObject({
|
|
49
|
+
per_metric_resolved: canonicalMetrics,
|
|
50
|
+
needs_adjudication: "false",
|
|
51
|
+
op_value_or_null: "video_a",
|
|
52
|
+
op_source: "alignment",
|
|
53
|
+
needs_overall_manual: "false",
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test.code(pipeline, mergeAdjudication, "preserves agreed metrics", async ({ run }) => {
|
|
58
|
+
await expect(
|
|
59
|
+
run({
|
|
60
|
+
per_metric_resolved: canonicalMetrics,
|
|
61
|
+
disagreed_metrics: [],
|
|
62
|
+
}),
|
|
63
|
+
).resolves.toMatchObject({
|
|
64
|
+
final_metrics: canonicalMetrics,
|
|
65
|
+
resolution_method: "agreement",
|
|
66
|
+
op_value_or_null: "video_a",
|
|
67
|
+
needs_overall_manual: "false",
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test.code(pipeline, opQuorumResolve, "resolves an overall majority", async ({ run }) => {
|
|
72
|
+
await expect(
|
|
73
|
+
run({
|
|
74
|
+
...item,
|
|
75
|
+
swap: false,
|
|
76
|
+
overall_manual_votes: ["video_a", "video_a", "video_b"],
|
|
77
|
+
}),
|
|
78
|
+
).resolves.toMatchObject({
|
|
79
|
+
op_manual_value: "video_a",
|
|
80
|
+
op_needs_adjudication: "false",
|
|
81
|
+
rater_op_votes: ["video_a", "video_a", "video_b"],
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test.code(pipeline, mergeOpAdjudication, "uses the overall quorum majority", async ({ run }) => {
|
|
86
|
+
await expect(
|
|
87
|
+
run({ op_needs_adjudication: "false", op_manual_value: "video_a" }),
|
|
88
|
+
).resolves.toEqual({
|
|
89
|
+
final_op_value: "video_a",
|
|
90
|
+
final_op_resolution_method: "agreement",
|
|
91
|
+
final_op_adjudicator_id: null,
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test.code(pipeline, finalizeOverall, "finalizes a cascade preference", async ({ run }) => {
|
|
96
|
+
await expect(
|
|
97
|
+
run({
|
|
98
|
+
op_value_or_null: "video_a",
|
|
99
|
+
op_source: "alignment",
|
|
100
|
+
needs_overall_manual: "false",
|
|
101
|
+
final_metrics: canonicalMetrics,
|
|
102
|
+
resolution_method: "agreement",
|
|
103
|
+
annotator_id_adjudicator: null,
|
|
104
|
+
}),
|
|
105
|
+
).resolves.toEqual({
|
|
106
|
+
overall_preference: "video_a",
|
|
107
|
+
overall_preference_source: "alignment",
|
|
108
|
+
resolution_method: "agreement",
|
|
109
|
+
annotator_id_adjudicator: null,
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test.code(
|
|
114
|
+
pipeline,
|
|
115
|
+
adjudicationMatch,
|
|
116
|
+
"grades raters against an adjudicated metric",
|
|
117
|
+
async ({ run }) => {
|
|
118
|
+
await expect(
|
|
119
|
+
run({
|
|
120
|
+
rater_ita_votes: ["video_a", "video_b"],
|
|
121
|
+
rater_iq_votes: ["video_a", "video_a"],
|
|
122
|
+
disagreed_metrics: ["image_text_alignment"],
|
|
123
|
+
final_metrics: canonicalMetrics,
|
|
124
|
+
needs_overall_manual: "false",
|
|
125
|
+
}),
|
|
126
|
+
).resolves.toEqual({
|
|
127
|
+
rater_ita_matches_adjudication: [true, false],
|
|
128
|
+
rater_iq_matches_adjudication: null,
|
|
129
|
+
rater_op_matches_adjudication: null,
|
|
130
|
+
});
|
|
131
|
+
},
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
test.code(pipeline, assemble, "assembles the delivery record", async ({ run }) => {
|
|
135
|
+
await expect(
|
|
136
|
+
run({
|
|
137
|
+
record_id: "record-1",
|
|
138
|
+
...item,
|
|
139
|
+
final_metrics: canonicalMetrics,
|
|
140
|
+
overall_preference: "video_a",
|
|
141
|
+
overall_preference_source: "alignment",
|
|
142
|
+
resolution_method: "agreement",
|
|
143
|
+
annotator_id_adjudicator: null,
|
|
144
|
+
rater_ita_votes: ["video_a", "video_a"],
|
|
145
|
+
rater_mta_votes: ["video_a", "video_a"],
|
|
146
|
+
rater_iq_votes: ["video_a", "video_a"],
|
|
147
|
+
rater_mq_votes: ["video_a", "video_a"],
|
|
148
|
+
rater_sbc_votes: ["video_a", "video_a"],
|
|
149
|
+
rater_vr_votes: ["video_a", "video_a"],
|
|
150
|
+
rater_ita_matches_adjudication: null,
|
|
151
|
+
rater_iq_matches_adjudication: null,
|
|
152
|
+
rater_op_matches_adjudication: null,
|
|
153
|
+
}),
|
|
154
|
+
).resolves.toMatchObject({
|
|
155
|
+
delivery_record: {
|
|
156
|
+
record_id: "record-1",
|
|
157
|
+
video_a_url: item.video_a,
|
|
158
|
+
video_b_url: item.video_b,
|
|
159
|
+
overall_preference: "video_a",
|
|
160
|
+
timestamp: expect.stringMatching(/Z$/u),
|
|
161
|
+
},
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test.ui(pipeline, ratePair, "submits independent metric ratings", async ({ render }) => {
|
|
166
|
+
const session = await render({
|
|
167
|
+
data: { prompt: item.prompt, left_video: item.video_a, right_video: item.video_b },
|
|
168
|
+
});
|
|
169
|
+
for (const metric of metricNames) await session.setValue(metric, "video_1");
|
|
170
|
+
|
|
171
|
+
await expect(session.submit()).resolves.toMatchObject(
|
|
172
|
+
Object.fromEntries(metricNames.map((name) => [name, "video_1"])),
|
|
173
|
+
);
|
|
174
|
+
await session.close();
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test.ui(pipeline, adjudicate, "submits disputed metric decisions", async ({ render }) => {
|
|
178
|
+
const session = await render({
|
|
179
|
+
data: { ...item, disagreed_metrics: ["image_quality", "motion_quality"] },
|
|
180
|
+
});
|
|
181
|
+
await session.setValue("adjudicated_metrics.image_quality", "video_a");
|
|
182
|
+
await session.setValue("adjudicated_metrics.motion_quality", "video_b");
|
|
183
|
+
|
|
184
|
+
await expect(session.submit()).resolves.toMatchObject({
|
|
185
|
+
adjudicated_metrics: { image_quality: "video_a", motion_quality: "video_b" },
|
|
186
|
+
});
|
|
187
|
+
await session.close();
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test.ui(
|
|
191
|
+
pipeline,
|
|
192
|
+
overallPreferencePick,
|
|
193
|
+
"submits an overall preference vote",
|
|
194
|
+
async ({ render }) => {
|
|
195
|
+
const session = await render({ data: item });
|
|
196
|
+
await session.setValue("overall_manual_votes", "video_a");
|
|
197
|
+
|
|
198
|
+
await expect(session.submit()).resolves.toMatchObject({ overall_manual_votes: "video_a" });
|
|
199
|
+
await session.close();
|
|
200
|
+
},
|
|
201
|
+
);
|
|
202
|
+
|
|
203
|
+
test.ui(pipeline, adjudicateOp, "submits an overall adjudication", async ({ render }) => {
|
|
204
|
+
const session = await render({ data: item });
|
|
205
|
+
await session.setValue("op_adjudicated_value", "video_b");
|
|
206
|
+
|
|
207
|
+
await expect(session.submit()).resolves.toMatchObject({ op_adjudicated_value: "video_b" });
|
|
208
|
+
await session.close();
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
test.pipeline(pipeline, "rates and assembles an agreed pair", async ({ renderUi, runCode }) => {
|
|
212
|
+
const shuffled = await runCode(shuffleAb, item);
|
|
213
|
+
const session = await renderUi(ratePair, {
|
|
214
|
+
data: {
|
|
215
|
+
prompt: item.prompt,
|
|
216
|
+
left_video: shuffled.left_video,
|
|
217
|
+
right_video: shuffled.right_video,
|
|
218
|
+
},
|
|
219
|
+
});
|
|
220
|
+
for (const metric of metricNames) await session.setValue(metric, "video_1");
|
|
221
|
+
const rating = await session.submit();
|
|
222
|
+
const votes = Object.fromEntries(
|
|
223
|
+
metricNames.map((name) => [name, [rating[name], rating[name]]]),
|
|
224
|
+
);
|
|
225
|
+
const resolved = await runCode(cascade, { swap: shuffled.swap, ...votes });
|
|
226
|
+
const merged = await runCode(mergeAdjudication, {
|
|
227
|
+
per_metric_resolved: resolved.per_metric_resolved,
|
|
228
|
+
disagreed_metrics: resolved.disagreed_metrics,
|
|
229
|
+
});
|
|
230
|
+
const overall = await runCode(finalizeOverall, {
|
|
231
|
+
op_value_or_null: merged.op_value_or_null,
|
|
232
|
+
op_source: merged.op_source,
|
|
233
|
+
needs_overall_manual: merged.needs_overall_manual,
|
|
234
|
+
final_metrics: merged.final_metrics,
|
|
235
|
+
resolution_method: merged.resolution_method,
|
|
236
|
+
annotator_id_adjudicator: merged.annotator_id_adjudicator,
|
|
237
|
+
});
|
|
238
|
+
const matches = await runCode(adjudicationMatch, {
|
|
239
|
+
rater_ita_votes: resolved.rater_ita_votes,
|
|
240
|
+
rater_iq_votes: resolved.rater_iq_votes,
|
|
241
|
+
disagreed_metrics: resolved.disagreed_metrics,
|
|
242
|
+
final_metrics: merged.final_metrics,
|
|
243
|
+
needs_overall_manual: merged.needs_overall_manual,
|
|
244
|
+
});
|
|
245
|
+
const assembled = await runCode(assemble, {
|
|
246
|
+
record_id: shuffled.record_id,
|
|
247
|
+
...item,
|
|
248
|
+
final_metrics: merged.final_metrics,
|
|
249
|
+
...overall,
|
|
250
|
+
rater_ita_votes: resolved.rater_ita_votes,
|
|
251
|
+
rater_mta_votes: resolved.rater_mta_votes,
|
|
252
|
+
rater_iq_votes: resolved.rater_iq_votes,
|
|
253
|
+
rater_mq_votes: resolved.rater_mq_votes,
|
|
254
|
+
rater_sbc_votes: resolved.rater_sbc_votes,
|
|
255
|
+
rater_vr_votes: resolved.rater_vr_votes,
|
|
256
|
+
...matches,
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
expect(assembled).toMatchObject({
|
|
260
|
+
delivery_record: {
|
|
261
|
+
record_id: shuffled.record_id,
|
|
262
|
+
overall_preference_source: "alignment",
|
|
263
|
+
},
|
|
264
|
+
});
|
|
265
|
+
await session.close();
|
|
266
|
+
});
|
|
267
|
+
});
|
package/openapi/gen/api.ts
CHANGED
|
@@ -71,6 +71,9 @@ import type {
|
|
|
71
71
|
RunItem,
|
|
72
72
|
RunItemId,
|
|
73
73
|
RunJob,
|
|
74
|
+
SampleDatasetItemsBody,
|
|
75
|
+
SampleJob,
|
|
76
|
+
SampleResult,
|
|
74
77
|
SetProjectSecretBody,
|
|
75
78
|
UpdatePipelineBody,
|
|
76
79
|
UpdateProjectBody,
|
|
@@ -564,28 +567,31 @@ export const finalizeDatasetUpload = async (
|
|
|
564
567
|
});
|
|
565
568
|
};
|
|
566
569
|
|
|
567
|
-
export const
|
|
568
|
-
return `/api/v2-beta/uploads/${uploadId}`;
|
|
570
|
+
export const getCancelDatasetUploadUrl = (uploadId: UploadId) => {
|
|
571
|
+
return `/api/v2-beta/uploads/${uploadId}/cancel`;
|
|
569
572
|
};
|
|
570
573
|
|
|
571
|
-
export const
|
|
574
|
+
export const cancelDatasetUpload = async (
|
|
572
575
|
uploadId: UploadId,
|
|
573
576
|
options?: RequestInit,
|
|
574
|
-
): Promise<
|
|
575
|
-
return potatoFetch<
|
|
577
|
+
): Promise<void> => {
|
|
578
|
+
return potatoFetch<void>(getCancelDatasetUploadUrl(uploadId), {
|
|
576
579
|
...options,
|
|
577
|
-
method: "
|
|
580
|
+
method: "POST",
|
|
578
581
|
});
|
|
579
582
|
};
|
|
580
583
|
|
|
581
|
-
export const
|
|
582
|
-
return `/api/v2-beta/uploads/${uploadId}
|
|
584
|
+
export const getGetDatasetUploadUrl = (uploadId: UploadId) => {
|
|
585
|
+
return `/api/v2-beta/uploads/${uploadId}`;
|
|
583
586
|
};
|
|
584
587
|
|
|
585
|
-
export const
|
|
586
|
-
|
|
588
|
+
export const getDatasetUpload = async (
|
|
589
|
+
uploadId: UploadId,
|
|
590
|
+
options?: RequestInit,
|
|
591
|
+
): Promise<UploadJob> => {
|
|
592
|
+
return potatoFetch<UploadJob>(getGetDatasetUploadUrl(uploadId), {
|
|
587
593
|
...options,
|
|
588
|
-
method: "
|
|
594
|
+
method: "GET",
|
|
589
595
|
});
|
|
590
596
|
};
|
|
591
597
|
|
|
@@ -745,6 +751,37 @@ export const cancelPipelineRun = async (runId: RunId, options?: RequestInit): Pr
|
|
|
745
751
|
});
|
|
746
752
|
};
|
|
747
753
|
|
|
754
|
+
export const getSampleDatasetItemsUrl = (datasetId: DatasetId) => {
|
|
755
|
+
return `/api/v2-beta/datasets/${datasetId}/sample`;
|
|
756
|
+
};
|
|
757
|
+
|
|
758
|
+
export const sampleDatasetItems = async (
|
|
759
|
+
datasetId: DatasetId,
|
|
760
|
+
sampleDatasetItemsBody: SampleDatasetItemsBody,
|
|
761
|
+
options?: RequestInit,
|
|
762
|
+
): Promise<SampleResult> => {
|
|
763
|
+
return potatoFetch<SampleResult>(getSampleDatasetItemsUrl(datasetId), {
|
|
764
|
+
...options,
|
|
765
|
+
method: "POST",
|
|
766
|
+
headers: { "Content-Type": "application/json", ...options?.headers },
|
|
767
|
+
body: JSON.stringify(sampleDatasetItemsBody),
|
|
768
|
+
});
|
|
769
|
+
};
|
|
770
|
+
|
|
771
|
+
export const getGetDatasetSampleUrl = (sampleId: string) => {
|
|
772
|
+
return `/api/v2-beta/samples/${sampleId}`;
|
|
773
|
+
};
|
|
774
|
+
|
|
775
|
+
export const getDatasetSample = async (
|
|
776
|
+
sampleId: string,
|
|
777
|
+
options?: RequestInit,
|
|
778
|
+
): Promise<SampleJob> => {
|
|
779
|
+
return potatoFetch<SampleJob>(getGetDatasetSampleUrl(sampleId), {
|
|
780
|
+
...options,
|
|
781
|
+
method: "GET",
|
|
782
|
+
});
|
|
783
|
+
};
|
|
784
|
+
|
|
748
785
|
export const getCreateDatasetExportUrl = (datasetId: DatasetId) => {
|
|
749
786
|
return `/api/v2-beta/datasets/${datasetId}/exports`;
|
|
750
787
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
+
import type { CreateDatasetExportBodyOneOf } from './createDatasetExportBodyOneOf';
|
|
2
|
+
import type { CreateDatasetExportBodyOneOfThree } from './createDatasetExportBodyOneOfThree';
|
|
1
3
|
|
|
2
|
-
export type CreateDatasetExportBody =
|
|
3
|
-
fields: string[];
|
|
4
|
-
};
|
|
4
|
+
export type CreateDatasetExportBody = CreateDatasetExportBodyOneOf | CreateDatasetExportBodyOneOfThree;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { CreateDatasetExportBodyOneOfThreeFormat } from './createDatasetExportBodyOneOfThreeFormat';
|
|
2
|
+
import type { CreateDatasetExportBodyOneOfThreeContract } from './createDatasetExportBodyOneOfThreeContract';
|
|
3
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilter } from './createDatasetExportBodyOneOfThreeItemFilter';
|
|
4
|
+
|
|
5
|
+
export type CreateDatasetExportBodyOneOfThree = {
|
|
6
|
+
format: CreateDatasetExportBodyOneOfThreeFormat;
|
|
7
|
+
contract: CreateDatasetExportBodyOneOfThreeContract;
|
|
8
|
+
itemFilter?: CreateDatasetExportBodyOneOfThreeItemFilter;
|
|
9
|
+
validateOnly?: boolean;
|
|
10
|
+
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItem } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItem';
|
|
2
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem } from './createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem';
|
|
3
|
+
|
|
4
|
+
export type CreateDatasetExportBodyOneOfThreeItemFilter = {
|
|
5
|
+
fieldValues?: CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItem[];
|
|
6
|
+
fieldSources?: CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem[];
|
|
7
|
+
missingRevisionFields?: string[];
|
|
8
|
+
createdBefore?: string;
|
|
9
|
+
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem } from './createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem';
|
|
2
|
+
|
|
3
|
+
export type CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem = {
|
|
4
|
+
fieldId: string;
|
|
5
|
+
sources: CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem[];
|
|
6
|
+
};
|
package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf } from './createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf';
|
|
2
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo } from './createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo';
|
|
3
|
+
|
|
4
|
+
export type CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem = CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf | CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator';
|
|
2
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue';
|
|
3
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem';
|
|
4
|
+
|
|
5
|
+
export type CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItem = {
|
|
6
|
+
jsonPath: string;
|
|
7
|
+
operator: CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator;
|
|
8
|
+
value: CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue;
|
|
9
|
+
fieldId: string;
|
|
10
|
+
latestOnly?: boolean;
|
|
11
|
+
sources?: CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem[];
|
|
12
|
+
};
|
package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf';
|
|
2
|
+
import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo';
|
|
3
|
+
|
|
4
|
+
export type CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem = CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf | CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo;
|