@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +10 -15
  2. package/dist/cli/toloka.js +209 -121
  3. package/dist/cli/toloka.js.jsc +0 -0
  4. package/dist/cli/toloka.js.jsc.sha256 +1 -1
  5. package/dist/cli/toloka.js.sha256 +1 -1
  6. package/docs/authoring.md +20 -9
  7. package/docs/cli-v2.md +3 -3
  8. package/docs/cli.md +44 -16
  9. package/docs/getting-started.md +7 -12
  10. package/docs/limitations.md +6 -6
  11. package/docs/testing-examples.md +248 -0
  12. package/docs/testing.md +115 -0
  13. package/examples/face-image-collection/README.md +7 -1
  14. package/examples/face-image-collection/fields.ts +20 -8
  15. package/examples/face-image-collection/nodes/annotate-faces/node.ts +1 -9
  16. package/examples/face-image-collection/nodes/order-faces/node.ts +3 -5
  17. package/examples/face-image-collection/pipeline.browser.integration.spec.ts +78 -0
  18. package/examples/video-pair-rating/README.md +14 -6
  19. package/examples/video-pair-rating/nodes/adjudicate/node.ts +1 -3
  20. package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +1 -3
  21. package/examples/video-pair-rating/nodes/adjudication_match/node.ts +15 -17
  22. package/examples/video-pair-rating/nodes/assemble/node.ts +25 -27
  23. package/examples/video-pair-rating/nodes/cascade/node.ts +24 -26
  24. package/examples/video-pair-rating/nodes/finalize_overall/node.ts +17 -19
  25. package/examples/video-pair-rating/nodes/merge_adjudication/node.ts +13 -15
  26. package/examples/video-pair-rating/nodes/merge_op_adjudication/node.ts +12 -14
  27. package/examples/video-pair-rating/nodes/op_quorum_resolve/node.ts +13 -15
  28. package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +1 -1
  29. package/examples/video-pair-rating/nodes/rate_pair/node.ts +1 -1
  30. package/examples/video-pair-rating/nodes/route_adjudication/node.ts +1 -3
  31. package/examples/video-pair-rating/nodes/route_op_adjudication/node.ts +5 -7
  32. package/examples/video-pair-rating/nodes/route_overall/node.ts +1 -3
  33. package/examples/video-pair-rating/nodes/shuffle_ab/node.ts +11 -13
  34. package/examples/video-pair-rating/pipeline.browser.integration.spec.ts +267 -0
  35. package/openapi/gen/api.ts +48 -11
  36. package/openapi/gen/models/createDatasetExportBody.ts +3 -3
  37. package/openapi/gen/models/createDatasetExportBodyOneOf.ts +6 -0
  38. package/openapi/gen/models/createDatasetExportBodyOneOfFormat.ts +2 -0
  39. package/openapi/gen/models/createDatasetExportBodyOneOfThree.ts +10 -0
  40. package/openapi/gen/models/createDatasetExportBodyOneOfThreeContract.ts +2 -0
  41. package/openapi/gen/models/createDatasetExportBodyOneOfThreeFormat.ts +2 -0
  42. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilter.ts +9 -0
  43. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem.ts +6 -0
  44. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  45. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  46. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  47. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItem.ts +12 -0
  48. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator.ts +2 -0
  49. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
  50. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  51. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  52. package/openapi/gen/models/createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue.ts +2 -0
  53. package/openapi/gen/models/datasetExportReport.ts +9 -0
  54. package/openapi/gen/models/datasetExportReportErrorsItem.ts +5 -0
  55. package/openapi/gen/models/datasetExportReportTokens.ts +7 -0
  56. package/openapi/gen/models/datasetItem.ts +0 -2
  57. package/openapi/gen/models/exportJob.ts +2 -0
  58. package/openapi/gen/models/exportJobReport.ts +3 -0
  59. package/openapi/gen/models/index.ts +48 -1
  60. package/openapi/gen/models/sampleDatasetItemsBody.ts +4 -0
  61. package/openapi/gen/models/sampleDatasetItemsBodyAnyOf.ts +8 -0
  62. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilter.ts +9 -0
  63. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItem.ts +6 -0
  64. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  65. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  66. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  67. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItem.ts +12 -0
  68. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemOperator.ts +2 -0
  69. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItem.ts +4 -0
  70. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  71. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  72. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfItemFilterFieldValuesItemValue.ts +2 -0
  73. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethree.ts +8 -0
  74. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilter.ts +9 -0
  75. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItem.ts +6 -0
  76. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItem.ts +4 -0
  77. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOf.ts +5 -0
  78. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldSourcesItemSourcesItemOneOfTwo.ts +5 -0
  79. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItem.ts +12 -0
  80. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemOperator.ts +2 -0
  81. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItem.ts +4 -0
  82. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOf.ts +5 -0
  83. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemSourcesItemOneOfTwo.ts +5 -0
  84. package/openapi/gen/models/sampleDatasetItemsBodyAnyOfOnethreeItemFilterFieldValuesItemValue.ts +2 -0
  85. package/openapi/gen/models/sampleJob.ts +13 -0
  86. package/openapi/gen/models/sampleJobStatus.ts +2 -0
  87. package/openapi/gen/models/sampleResult.ts +5 -0
  88. package/package.json +23 -2
  89. package/skills/catalog.json +14 -6
  90. package/skills/toloka-build/SKILL.md +35 -12
  91. package/skills/toloka-build/references/node-semantics.md +32 -29
  92. package/skills/toloka-build/references/workflow-and-cli.md +61 -27
  93. package/skills/toloka-report-issue/references/report-template.md +1 -1
  94. package/src/authoring/index.ts +27 -26
  95. package/src/checks/ast.ts +32 -0
  96. package/src/checks/index.ts +14 -0
  97. package/src/checks/postprocess-collisions.ts +145 -0
  98. package/src/checks/task-ui-declarations.generated.ts +19 -0
  99. package/src/checks/task-ui-typecheck.ts +345 -0
  100. package/src/checks/typescript-vfs.d.ts +26 -0
  101. package/src/checks/typescript-vfs.js +5 -0
  102. package/src/cli/analytics.ts +9 -19
  103. package/src/cli/auto-update.ts +367 -0
  104. package/src/cli/boolean-flag.ts +13 -0
  105. package/src/cli/check.ts +977 -0
  106. package/src/cli/credentials.ts +3 -0
  107. package/src/cli/program.ts +184 -20
  108. package/src/cli/project-context.ts +14 -4
  109. package/src/cli/runs.ts +21 -11
  110. package/src/cli/settings.ts +93 -0
  111. package/src/cli/skill.ts +17 -6
  112. package/src/importer.ts +12 -16
  113. package/src/testing/code-node.ts +160 -0
  114. package/src/testing/index.ts +15 -0
  115. package/src/testing/task-ui-browser.ts +120 -0
  116. package/src/testing/task-ui-validation.ts +135 -0
  117. package/src/testing/task-ui.ts +557 -0
  118. package/src/testing/test-evidence.ts +170 -0
  119. package/src/testing/types.ts +141 -0
  120. package/openapi/gen/models/datasetItemDuplicateOfSubmissionId.ts +0 -2
@@ -15,18 +15,16 @@ export const mergeOpAdjudication = code("merge_op_adjudication", {
15
15
  "Consolidates the overall preference branch: if adjudication ran, emit the adjudicator pick and resolution_method=adjudication; otherwise emit the quorum majority and resolution_method=agreement.",
16
16
  },
17
17
  envVars: [],
18
- fieldProxies: {
19
- input: [
20
- { field: opNeedsAdjudication, optional: false },
21
- { field: opManualValue, optional: true },
22
- { field: opAdjudicatedValue, optional: true },
23
- ],
24
- output: [
25
- { field: finalOpValue, optional: false },
26
- { field: finalOpResolutionMethod, optional: false },
27
- { field: finalOpAdjudicatorId, optional: false },
28
- ],
29
- },
18
+ input: [
19
+ { field: opNeedsAdjudication, optional: false },
20
+ { field: opManualValue, optional: true },
21
+ { field: opAdjudicatedValue, optional: true },
22
+ ],
23
+ output: [
24
+ { field: finalOpValue, optional: false },
25
+ { field: finalOpResolutionMethod, optional: false },
26
+ { field: finalOpAdjudicatorId, optional: false },
27
+ ],
30
28
  code: {
31
29
  code: `from main_types import Inputs, Outputs
32
30
 
@@ -38,8 +36,8 @@ async def main(inputs: Inputs) -> Outputs:
38
36
  "final_op_value": inputs["op_adjudicated_value"],
39
37
  "final_op_resolution_method": "adjudication",
40
38
  # Platform limitation (mirrors annotator_id_1/2 in assemble): the
41
- # OP adjudicator id is not surfaced as a field proxy from the
42
- # generation_sft node in this pilot. Emit null.
39
+ # OP adjudicator id is not surfaced as an output field from the
40
+ # generation node in this pilot. Emit null.
43
41
  "final_op_adjudicator_id": None,
44
42
  }
45
43
  return {
@@ -18,21 +18,19 @@ export const opQuorumResolve = code("op_quorum_resolve", {
18
18
  "Resolves three overall preference votes from the quorum into a majority. If fewer than two votes agree, flags for adjudication and packages the disagreed task payload.",
19
19
  },
20
20
  envVars: [],
21
- fieldProxies: {
22
- input: [
23
- { field: prompt, optional: false },
24
- { field: videoA, optional: false },
25
- { field: videoB, optional: false },
26
- { field: swap, optional: false },
27
- { field: overallManualVotes, optional: false },
28
- ],
29
- output: [
30
- { field: opManualValue, optional: true },
31
- { field: opNeedsAdjudication, optional: false },
32
- { field: disagreedOpTask, optional: false },
33
- { field: raterOpVotes, optional: false },
34
- ],
35
- },
21
+ input: [
22
+ { field: prompt, optional: false },
23
+ { field: videoA, optional: false },
24
+ { field: videoB, optional: false },
25
+ { field: swap, optional: false },
26
+ { field: overallManualVotes, optional: false },
27
+ ],
28
+ output: [
29
+ { field: opManualValue, optional: true },
30
+ { field: opNeedsAdjudication, optional: false },
31
+ { field: disagreedOpTask, optional: false },
32
+ { field: raterOpVotes, optional: false },
33
+ ],
36
34
  code: {
37
35
  code: `from main_types import Inputs, Outputs
38
36
 
@@ -48,7 +48,7 @@ export const overallPreferencePick = quorum("overall_preference_pick", {
48
48
  instruction:
49
49
  "Watch both videos in full, then pick the one you would ship. This is a single judgment about the whole video, not a sum of its parts. Use tie only when you genuinely have no preference. Unlike the metric rating task, this one shows the pair in its original order, so your pick needs no un-shuffling downstream.",
50
50
  ui: {
51
- code: "<Field.SelectGroup path={'overall_manual_votes'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
51
+ code: "<Field.RadioGroup path={'overall_manual_votes'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
52
52
  data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
53
53
  preprocess: "return {}",
54
54
  postprocess: "return {};",
@@ -96,7 +96,7 @@ export const ratePair = quorum("rate_pair", {
96
96
  instruction:
97
97
  "Watch both videos in full before answering. For each metric, pick the video that is better on that metric alone, or tie when they are equal. The two videos are presented in random order, so left and right carry no meaning. Different metrics can have different winners; that is expected.",
98
98
  ui: {
99
- code: "<View.Group><Field.SelectGroup path={'image_text_alignment'} label={'Which video matches the prompt in its imagery?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'motion_text_alignment'} label={'Which video matches the prompt in its motion?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'image_quality'} label={'Which video looks better frame by frame?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'motion_quality'} label={'Which video moves more plausibly?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'subject_background_consistency'} label={'Which video keeps its subject and background stable?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'video_realism'} label={'Which video reads as more real?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /></View.Group>",
99
+ code: "<View.Card><Field.RadioGroup path={'image_text_alignment'} label={'Which video matches the prompt in its imagery?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'motion_text_alignment'} label={'Which video matches the prompt in its motion?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'image_quality'} label={'Which video looks better frame by frame?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'motion_quality'} label={'Which video moves more plausibly?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'subject_background_consistency'} label={'Which video keeps its subject and background stable?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'video_realism'} label={'Which video reads as more real?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /></View.Card>",
100
100
  data: '{"prompt":"a red fox crossing a snowy field at dawn","left_video":"https://example.com/video-left.mp4","right_video":"https://example.com/video-right.mp4"}',
101
101
  preprocess: "return {}",
102
102
  postprocess: "return {};",
@@ -20,7 +20,5 @@ export const routeAdjudication = match("route_adjudication", {
20
20
  ],
21
21
  },
22
22
  ],
23
- fieldProxies: {
24
- input: [{ field: needsAdjudication, optional: false }],
25
- },
23
+ input: [{ field: needsAdjudication, optional: false }],
26
24
  });
@@ -20,11 +20,9 @@ export const routeOpAdjudication = match("route_op_adjudication", {
20
20
  ],
21
21
  },
22
22
  ],
23
- fieldProxies: {
24
- input: [
25
- { field: opNeedsAdjudication, optional: false },
26
- { field: disagreedOpTask, optional: true },
27
- { field: opManualValue, optional: true },
28
- ],
29
- },
23
+ input: [
24
+ { field: opNeedsAdjudication, optional: false },
25
+ { field: disagreedOpTask, optional: true },
26
+ { field: opManualValue, optional: true },
27
+ ],
30
28
  });
@@ -20,7 +20,5 @@ export const routeOverall = match("route_overall", {
20
20
  ],
21
21
  },
22
22
  ],
23
- fieldProxies: {
24
- input: [{ field: needsOverallManual, optional: false }],
25
- },
23
+ input: [{ field: needsOverallManual, optional: false }],
26
24
  });
@@ -8,19 +8,17 @@ export const shuffleAb = code("shuffle_ab", {
8
8
  "Mints a deterministic record_id from a hash of the prompt and video pair. Swaps the left/right video assignment based on the hash parity to prevent position bias in rater interfaces.",
9
9
  },
10
10
  envVars: [],
11
- fieldProxies: {
12
- input: [
13
- { field: f.prompt, optional: false },
14
- { field: f.videoA, optional: false },
15
- { field: f.videoB, optional: false },
16
- ],
17
- output: [
18
- { field: f.recordId, optional: false },
19
- { field: f.leftVideo, optional: false },
20
- { field: f.rightVideo, optional: false },
21
- { field: f.swap, optional: false },
22
- ],
23
- },
11
+ input: [
12
+ { field: f.prompt, optional: false },
13
+ { field: f.videoA, optional: false },
14
+ { field: f.videoB, optional: false },
15
+ ],
16
+ output: [
17
+ { field: f.recordId, optional: false },
18
+ { field: f.leftVideo, optional: false },
19
+ { field: f.rightVideo, optional: false },
20
+ { field: f.swap, optional: false },
21
+ ],
24
22
  code: {
25
23
  code: `import hashlib
26
24
  from main_types import Inputs, Outputs
@@ -0,0 +1,267 @@
1
+ import { expect, test } from "@toloka-platform/sdk/testing";
2
+ import { describe } from "vitest";
3
+
4
+ import pipeline from "./pipeline";
5
+ import { adjudicate } from "./nodes/adjudicate/node";
6
+ import { adjudicateOp } from "./nodes/adjudicate_op/node";
7
+ import { adjudicationMatch } from "./nodes/adjudication_match/node";
8
+ import { assemble } from "./nodes/assemble/node";
9
+ import { cascade } from "./nodes/cascade/node";
10
+ import { finalizeOverall } from "./nodes/finalize_overall/node";
11
+ import { mergeAdjudication } from "./nodes/merge_adjudication/node";
12
+ import { mergeOpAdjudication } from "./nodes/merge_op_adjudication/node";
13
+ import { opQuorumResolve } from "./nodes/op_quorum_resolve/node";
14
+ import { overallPreferencePick } from "./nodes/overall_preference_pick/node";
15
+ import { ratePair } from "./nodes/rate_pair/node";
16
+ import { shuffleAb } from "./nodes/shuffle_ab/node";
17
+
18
+ const item = {
19
+ prompt: "a fox crossing a snowy field",
20
+ video_a: "https://example.com/video-a.mp4",
21
+ video_b: "https://example.com/video-b.mp4",
22
+ };
23
+ const metricNames = [
24
+ "image_text_alignment",
25
+ "motion_text_alignment",
26
+ "image_quality",
27
+ "motion_quality",
28
+ "subject_background_consistency",
29
+ "video_realism",
30
+ ];
31
+ const canonicalMetrics = Object.fromEntries(metricNames.map((name) => [name, "video_a"]));
32
+ const metricVotes = Object.fromEntries(metricNames.map((name) => [name, ["video_1", "video_1"]]));
33
+
34
+ describe("video pair rating example", () => {
35
+ test.code(pipeline, shuffleAb, "shuffles a pair deterministically", async ({ run }) => {
36
+ const result = await run(item);
37
+
38
+ expect(result).toMatchObject({
39
+ record_id: expect.stringMatching(/^[a-f0-9]{16}$/u),
40
+ swap: expect.any(Boolean),
41
+ });
42
+ expect(new Set([result.left_video, result.right_video])).toEqual(
43
+ new Set([item.video_a, item.video_b]),
44
+ );
45
+ });
46
+
47
+ test.code(pipeline, cascade, "resolves unanimous metric votes", async ({ run }) => {
48
+ await expect(run({ swap: false, ...metricVotes })).resolves.toMatchObject({
49
+ per_metric_resolved: canonicalMetrics,
50
+ needs_adjudication: "false",
51
+ op_value_or_null: "video_a",
52
+ op_source: "alignment",
53
+ needs_overall_manual: "false",
54
+ });
55
+ });
56
+
57
+ test.code(pipeline, mergeAdjudication, "preserves agreed metrics", async ({ run }) => {
58
+ await expect(
59
+ run({
60
+ per_metric_resolved: canonicalMetrics,
61
+ disagreed_metrics: [],
62
+ }),
63
+ ).resolves.toMatchObject({
64
+ final_metrics: canonicalMetrics,
65
+ resolution_method: "agreement",
66
+ op_value_or_null: "video_a",
67
+ needs_overall_manual: "false",
68
+ });
69
+ });
70
+
71
+ test.code(pipeline, opQuorumResolve, "resolves an overall majority", async ({ run }) => {
72
+ await expect(
73
+ run({
74
+ ...item,
75
+ swap: false,
76
+ overall_manual_votes: ["video_a", "video_a", "video_b"],
77
+ }),
78
+ ).resolves.toMatchObject({
79
+ op_manual_value: "video_a",
80
+ op_needs_adjudication: "false",
81
+ rater_op_votes: ["video_a", "video_a", "video_b"],
82
+ });
83
+ });
84
+
85
+ test.code(pipeline, mergeOpAdjudication, "uses the overall quorum majority", async ({ run }) => {
86
+ await expect(
87
+ run({ op_needs_adjudication: "false", op_manual_value: "video_a" }),
88
+ ).resolves.toEqual({
89
+ final_op_value: "video_a",
90
+ final_op_resolution_method: "agreement",
91
+ final_op_adjudicator_id: null,
92
+ });
93
+ });
94
+
95
+ test.code(pipeline, finalizeOverall, "finalizes a cascade preference", async ({ run }) => {
96
+ await expect(
97
+ run({
98
+ op_value_or_null: "video_a",
99
+ op_source: "alignment",
100
+ needs_overall_manual: "false",
101
+ final_metrics: canonicalMetrics,
102
+ resolution_method: "agreement",
103
+ annotator_id_adjudicator: null,
104
+ }),
105
+ ).resolves.toEqual({
106
+ overall_preference: "video_a",
107
+ overall_preference_source: "alignment",
108
+ resolution_method: "agreement",
109
+ annotator_id_adjudicator: null,
110
+ });
111
+ });
112
+
113
+ test.code(
114
+ pipeline,
115
+ adjudicationMatch,
116
+ "grades raters against an adjudicated metric",
117
+ async ({ run }) => {
118
+ await expect(
119
+ run({
120
+ rater_ita_votes: ["video_a", "video_b"],
121
+ rater_iq_votes: ["video_a", "video_a"],
122
+ disagreed_metrics: ["image_text_alignment"],
123
+ final_metrics: canonicalMetrics,
124
+ needs_overall_manual: "false",
125
+ }),
126
+ ).resolves.toEqual({
127
+ rater_ita_matches_adjudication: [true, false],
128
+ rater_iq_matches_adjudication: null,
129
+ rater_op_matches_adjudication: null,
130
+ });
131
+ },
132
+ );
133
+
134
+ test.code(pipeline, assemble, "assembles the delivery record", async ({ run }) => {
135
+ await expect(
136
+ run({
137
+ record_id: "record-1",
138
+ ...item,
139
+ final_metrics: canonicalMetrics,
140
+ overall_preference: "video_a",
141
+ overall_preference_source: "alignment",
142
+ resolution_method: "agreement",
143
+ annotator_id_adjudicator: null,
144
+ rater_ita_votes: ["video_a", "video_a"],
145
+ rater_mta_votes: ["video_a", "video_a"],
146
+ rater_iq_votes: ["video_a", "video_a"],
147
+ rater_mq_votes: ["video_a", "video_a"],
148
+ rater_sbc_votes: ["video_a", "video_a"],
149
+ rater_vr_votes: ["video_a", "video_a"],
150
+ rater_ita_matches_adjudication: null,
151
+ rater_iq_matches_adjudication: null,
152
+ rater_op_matches_adjudication: null,
153
+ }),
154
+ ).resolves.toMatchObject({
155
+ delivery_record: {
156
+ record_id: "record-1",
157
+ video_a_url: item.video_a,
158
+ video_b_url: item.video_b,
159
+ overall_preference: "video_a",
160
+ timestamp: expect.stringMatching(/Z$/u),
161
+ },
162
+ });
163
+ });
164
+
165
+ test.ui(pipeline, ratePair, "submits independent metric ratings", async ({ render }) => {
166
+ const session = await render({
167
+ data: { prompt: item.prompt, left_video: item.video_a, right_video: item.video_b },
168
+ });
169
+ for (const metric of metricNames) await session.setValue(metric, "video_1");
170
+
171
+ await expect(session.submit()).resolves.toMatchObject(
172
+ Object.fromEntries(metricNames.map((name) => [name, "video_1"])),
173
+ );
174
+ await session.close();
175
+ });
176
+
177
+ test.ui(pipeline, adjudicate, "submits disputed metric decisions", async ({ render }) => {
178
+ const session = await render({
179
+ data: { ...item, disagreed_metrics: ["image_quality", "motion_quality"] },
180
+ });
181
+ await session.setValue("adjudicated_metrics.image_quality", "video_a");
182
+ await session.setValue("adjudicated_metrics.motion_quality", "video_b");
183
+
184
+ await expect(session.submit()).resolves.toMatchObject({
185
+ adjudicated_metrics: { image_quality: "video_a", motion_quality: "video_b" },
186
+ });
187
+ await session.close();
188
+ });
189
+
190
+ test.ui(
191
+ pipeline,
192
+ overallPreferencePick,
193
+ "submits an overall preference vote",
194
+ async ({ render }) => {
195
+ const session = await render({ data: item });
196
+ await session.setValue("overall_manual_votes", "video_a");
197
+
198
+ await expect(session.submit()).resolves.toMatchObject({ overall_manual_votes: "video_a" });
199
+ await session.close();
200
+ },
201
+ );
202
+
203
+ test.ui(pipeline, adjudicateOp, "submits an overall adjudication", async ({ render }) => {
204
+ const session = await render({ data: item });
205
+ await session.setValue("op_adjudicated_value", "video_b");
206
+
207
+ await expect(session.submit()).resolves.toMatchObject({ op_adjudicated_value: "video_b" });
208
+ await session.close();
209
+ });
210
+
211
+ test.pipeline(pipeline, "rates and assembles an agreed pair", async ({ renderUi, runCode }) => {
212
+ const shuffled = await runCode(shuffleAb, item);
213
+ const session = await renderUi(ratePair, {
214
+ data: {
215
+ prompt: item.prompt,
216
+ left_video: shuffled.left_video,
217
+ right_video: shuffled.right_video,
218
+ },
219
+ });
220
+ for (const metric of metricNames) await session.setValue(metric, "video_1");
221
+ const rating = await session.submit();
222
+ const votes = Object.fromEntries(
223
+ metricNames.map((name) => [name, [rating[name], rating[name]]]),
224
+ );
225
+ const resolved = await runCode(cascade, { swap: shuffled.swap, ...votes });
226
+ const merged = await runCode(mergeAdjudication, {
227
+ per_metric_resolved: resolved.per_metric_resolved,
228
+ disagreed_metrics: resolved.disagreed_metrics,
229
+ });
230
+ const overall = await runCode(finalizeOverall, {
231
+ op_value_or_null: merged.op_value_or_null,
232
+ op_source: merged.op_source,
233
+ needs_overall_manual: merged.needs_overall_manual,
234
+ final_metrics: merged.final_metrics,
235
+ resolution_method: merged.resolution_method,
236
+ annotator_id_adjudicator: merged.annotator_id_adjudicator,
237
+ });
238
+ const matches = await runCode(adjudicationMatch, {
239
+ rater_ita_votes: resolved.rater_ita_votes,
240
+ rater_iq_votes: resolved.rater_iq_votes,
241
+ disagreed_metrics: resolved.disagreed_metrics,
242
+ final_metrics: merged.final_metrics,
243
+ needs_overall_manual: merged.needs_overall_manual,
244
+ });
245
+ const assembled = await runCode(assemble, {
246
+ record_id: shuffled.record_id,
247
+ ...item,
248
+ final_metrics: merged.final_metrics,
249
+ ...overall,
250
+ rater_ita_votes: resolved.rater_ita_votes,
251
+ rater_mta_votes: resolved.rater_mta_votes,
252
+ rater_iq_votes: resolved.rater_iq_votes,
253
+ rater_mq_votes: resolved.rater_mq_votes,
254
+ rater_sbc_votes: resolved.rater_sbc_votes,
255
+ rater_vr_votes: resolved.rater_vr_votes,
256
+ ...matches,
257
+ });
258
+
259
+ expect(assembled).toMatchObject({
260
+ delivery_record: {
261
+ record_id: shuffled.record_id,
262
+ overall_preference_source: "alignment",
263
+ },
264
+ });
265
+ await session.close();
266
+ });
267
+ });
@@ -71,6 +71,9 @@ import type {
71
71
  RunItem,
72
72
  RunItemId,
73
73
  RunJob,
74
+ SampleDatasetItemsBody,
75
+ SampleJob,
76
+ SampleResult,
74
77
  SetProjectSecretBody,
75
78
  UpdatePipelineBody,
76
79
  UpdateProjectBody,
@@ -564,28 +567,31 @@ export const finalizeDatasetUpload = async (
564
567
  });
565
568
  };
566
569
 
567
- export const getGetDatasetUploadUrl = (uploadId: UploadId) => {
568
- return `/api/v2-beta/uploads/${uploadId}`;
570
+ export const getCancelDatasetUploadUrl = (uploadId: UploadId) => {
571
+ return `/api/v2-beta/uploads/${uploadId}/cancel`;
569
572
  };
570
573
 
571
- export const getDatasetUpload = async (
574
+ export const cancelDatasetUpload = async (
572
575
  uploadId: UploadId,
573
576
  options?: RequestInit,
574
- ): Promise<UploadJob> => {
575
- return potatoFetch<UploadJob>(getGetDatasetUploadUrl(uploadId), {
577
+ ): Promise<void> => {
578
+ return potatoFetch<void>(getCancelDatasetUploadUrl(uploadId), {
576
579
  ...options,
577
- method: "GET",
580
+ method: "POST",
578
581
  });
579
582
  };
580
583
 
581
- export const getCancelDatasetUploadUrl = (uploadId: UploadId) => {
582
- return `/api/v2-beta/uploads/${uploadId}/cancel`;
584
+ export const getGetDatasetUploadUrl = (uploadId: UploadId) => {
585
+ return `/api/v2-beta/uploads/${uploadId}`;
583
586
  };
584
587
 
585
- export const cancelDatasetUpload = async (uploadId: UploadId, options?: RequestInit): Promise<void> => {
586
- return potatoFetch<void>(getCancelDatasetUploadUrl(uploadId), {
588
+ export const getDatasetUpload = async (
589
+ uploadId: UploadId,
590
+ options?: RequestInit,
591
+ ): Promise<UploadJob> => {
592
+ return potatoFetch<UploadJob>(getGetDatasetUploadUrl(uploadId), {
587
593
  ...options,
588
- method: "POST",
594
+ method: "GET",
589
595
  });
590
596
  };
591
597
 
@@ -745,6 +751,37 @@ export const cancelPipelineRun = async (runId: RunId, options?: RequestInit): Pr
745
751
  });
746
752
  };
747
753
 
754
+ export const getSampleDatasetItemsUrl = (datasetId: DatasetId) => {
755
+ return `/api/v2-beta/datasets/${datasetId}/sample`;
756
+ };
757
+
758
+ export const sampleDatasetItems = async (
759
+ datasetId: DatasetId,
760
+ sampleDatasetItemsBody: SampleDatasetItemsBody,
761
+ options?: RequestInit,
762
+ ): Promise<SampleResult> => {
763
+ return potatoFetch<SampleResult>(getSampleDatasetItemsUrl(datasetId), {
764
+ ...options,
765
+ method: "POST",
766
+ headers: { "Content-Type": "application/json", ...options?.headers },
767
+ body: JSON.stringify(sampleDatasetItemsBody),
768
+ });
769
+ };
770
+
771
+ export const getGetDatasetSampleUrl = (sampleId: string) => {
772
+ return `/api/v2-beta/samples/${sampleId}`;
773
+ };
774
+
775
+ export const getDatasetSample = async (
776
+ sampleId: string,
777
+ options?: RequestInit,
778
+ ): Promise<SampleJob> => {
779
+ return potatoFetch<SampleJob>(getGetDatasetSampleUrl(sampleId), {
780
+ ...options,
781
+ method: "GET",
782
+ });
783
+ };
784
+
748
785
  export const getCreateDatasetExportUrl = (datasetId: DatasetId) => {
749
786
  return `/api/v2-beta/datasets/${datasetId}/exports`;
750
787
  };
@@ -1,4 +1,4 @@
1
+ import type { CreateDatasetExportBodyOneOf } from './createDatasetExportBodyOneOf';
2
+ import type { CreateDatasetExportBodyOneOfThree } from './createDatasetExportBodyOneOfThree';
1
3
 
2
- export type CreateDatasetExportBody = {
3
- fields: string[];
4
- };
4
+ export type CreateDatasetExportBody = CreateDatasetExportBodyOneOf | CreateDatasetExportBodyOneOfThree;
@@ -0,0 +1,6 @@
1
+ import type { CreateDatasetExportBodyOneOfFormat } from './createDatasetExportBodyOneOfFormat';
2
+
3
+ export type CreateDatasetExportBodyOneOf = {
4
+ format?: CreateDatasetExportBodyOneOfFormat;
5
+ fields: string[];
6
+ };
@@ -0,0 +1,2 @@
1
+
2
+ export type CreateDatasetExportBodyOneOfFormat = 'json-array';
@@ -0,0 +1,10 @@
1
+ import type { CreateDatasetExportBodyOneOfThreeFormat } from './createDatasetExportBodyOneOfThreeFormat';
2
+ import type { CreateDatasetExportBodyOneOfThreeContract } from './createDatasetExportBodyOneOfThreeContract';
3
+ import type { CreateDatasetExportBodyOneOfThreeItemFilter } from './createDatasetExportBodyOneOfThreeItemFilter';
4
+
5
+ export type CreateDatasetExportBodyOneOfThree = {
6
+ format: CreateDatasetExportBodyOneOfThreeFormat;
7
+ contract: CreateDatasetExportBodyOneOfThreeContract;
8
+ itemFilter?: CreateDatasetExportBodyOneOfThreeItemFilter;
9
+ validateOnly?: boolean;
10
+ };
@@ -0,0 +1,2 @@
1
+
2
+ export type CreateDatasetExportBodyOneOfThreeContract = 'openai-chat-v1';
@@ -0,0 +1,2 @@
1
+
2
+ export type CreateDatasetExportBodyOneOfThreeFormat = 'training-jsonl';
@@ -0,0 +1,9 @@
1
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItem } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItem';
2
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem } from './createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem';
3
+
4
+ export type CreateDatasetExportBodyOneOfThreeItemFilter = {
5
+ fieldValues?: CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItem[];
6
+ fieldSources?: CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem[];
7
+ missingRevisionFields?: string[];
8
+ createdBefore?: string;
9
+ };
@@ -0,0 +1,6 @@
1
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem } from './createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem';
2
+
3
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItem = {
4
+ fieldId: string;
5
+ sources: CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem[];
6
+ };
@@ -0,0 +1,4 @@
1
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf } from './createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf';
2
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo } from './createDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo';
3
+
4
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItem = CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf | CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo;
@@ -0,0 +1,5 @@
1
+
2
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOf = {
3
+ type: 'user';
4
+ userId: string;
5
+ };
@@ -0,0 +1,5 @@
1
+
2
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldSourcesItemSourcesItemOneOfTwo = {
3
+ type: 'system';
4
+ source: string;
5
+ };
@@ -0,0 +1,12 @@
1
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator';
2
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue';
3
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem';
4
+
5
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItem = {
6
+ jsonPath: string;
7
+ operator: CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator;
8
+ value: CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemValue;
9
+ fieldId: string;
10
+ latestOnly?: boolean;
11
+ sources?: CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem[];
12
+ };
@@ -0,0 +1,2 @@
1
+
2
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemOperator = 'text_eq' | 'text_neq';
@@ -0,0 +1,4 @@
1
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf';
2
+ import type { CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo } from './createDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo';
3
+
4
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItem = CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf | CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo;
@@ -0,0 +1,5 @@
1
+
2
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOf = {
3
+ type: 'user';
4
+ userId: string;
5
+ };
@@ -0,0 +1,5 @@
1
+
2
+ export type CreateDatasetExportBodyOneOfThreeItemFilterFieldValuesItemSourcesItemOneOfTwo = {
3
+ type: 'system';
4
+ source: string;
5
+ };