@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.7

package/examples/face-image-collection/README.md

@@ -17,7 +17,13 @@ toloka import pp.0190abcd-abcd-7abc-8abc-abcdefabcdef
 # Imported project pp.0190abcd-abcd-7abc-8abc-abcdefabcdef with 1 pipeline and 1 dataset into ...
 ```
 
-Edit a node under `pipelines/face-image-collection/nodes/`, then inspect and apply the project-wide reconciliation:
+Edit a node under `pipelines/face-image-collection/nodes/`, then run the shipped typed code, UI, and connected-flow scenarios before inspecting any platform changes:
+
+```sh
+toloka check face-image-collection
+```
+
+After the local gate passes, inspect and apply the project-wide reconciliation:
 
 ```sh
 toloka plan

package/examples/face-image-collection/fields.ts

@@ -5,19 +5,31 @@ const faceAnnotationsSchema: Parameters<typeof field>[1] = {
   items: {
     type: "object",
     properties: {
-      boundingBox: {
+      id: { type: "string" },
+      type: { type: "string" },
+      geometry: {
         type: "object",
         properties: {
-          x: { type: "number" },
-          y: { type: "number" },
-          width: { type: "number" },
-          height: { type: "number" },
+          bounds: {
+            type: "object",
+            properties: {
+              minX: { type: "number" },
+              minY: { type: "number" },
+              maxX: { type: "number" },
+              maxY: { type: "number" },
+            },
+            required: ["minX", "minY", "maxX", "maxY"],
+          },
         },
-        required: ["x", "y", "width", "height"],
+        required: ["bounds"],
+      },
+      properties: {
+        type: "object",
+        properties: { value: { type: "string", const: "face" } },
+        required: ["value"],
       },
-      label: { type: "string", const: "face" },
     },
-    required: ["boundingBox", "label"],
+    required: ["id", "type", "geometry", "properties"],
   },
 };
 

package/examples/face-image-collection/nodes/annotate-faces/node.ts

@@ -55,7 +55,7 @@ export const annotateFaces = generation("annotate_faces", {
   instruction:
     "Draw one tight rectangular bounding box around every visible human face. Do not annotate faces shown only in photographs, posters, or reflections.",
   ui: {
-    code: "<Field.ImageAnnotation path={'draft_faces'} image={actions.get('image_url', '')} tools={['rectangle']} labels={[{ label: 'Face', value: 'face' }]} />",
+    code: "<Field.ImageAnnotation path={'draft_faces'} image={actions.get('image_url')} tools={['rectangle']} labels={[{ label: 'Face', value: 'face', color: 'blue' }]} navigator={true} />",
     data: '{"image_url":"https://example.com/face.jpg"}',
     preprocess: "return {}",
     postprocess: "return {};",
@@ -131,12 +131,4 @@ export const annotateFaces = generation("annotate_faces", {
       successCriteria: "Every face has one tight bounding box",
     },
   },
-  exam: {
-    enabled: false,
-    gradedEntities: [],
-    tasksPerExpert: 1,
-    passThreshold: 1,
-  },
-  reuseAudienceWithNodeId: null,
-  smartSearch: null,
 });

package/examples/face-image-collection/nodes/order-faces/node.ts

@@ -19,7 +19,7 @@ export const orderFaces = code("order_faces", {
 async def main(inputs: Inputs) -> Outputs:
     ordered_faces = sorted(
         inputs["draft_faces"],
-        key=lambda face: (face["boundingBox"]["y"], face["boundingBox"]["x"]),
+        key=lambda face: (face["geometry"]["bounds"]["minY"], face["geometry"]["bounds"]["minX"]),
     )
     return {"faces": ordered_faces}`,
   },

package/examples/face-image-collection/pipeline.browser.integration.spec.ts

@@ -0,0 +1,78 @@
+import { expect, test } from "@toloka-platform/sdk/testing";
+import { describe } from "vitest";
+import { assert } from "es-toolkit";
+import type { TaskUiSession } from "@toloka-platform/sdk/testing";
+
+import pipeline from "./pipeline";
+import { annotateFaces } from "./nodes/annotate-faces/node";
+import { orderFaces } from "./nodes/order-faces/node";
+const upperFace = {
+  id: "upper-face",
+  type: "RECTANGLE",
+  geometry: { x: 40, y: 10, w: 20, h: 20, bounds: { minX: 40, minY: 10, maxX: 60, maxY: 30 } },
+  properties: { value: "face" },
+};
+const lowerFace = {
+  id: "lower-face",
+  type: "RECTANGLE",
+  geometry: { x: 10, y: 60, w: 30, h: 30, bounds: { minX: 10, minY: 60, maxX: 40, maxY: 90 } },
+  properties: { value: "face" },
+};
+const imageUrl =
+  "https://tlkfrontprod.azureedge.net/frontend-testing/static/gzarip/cats/cat_001.jpg";
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const documentedFaceAnnotations = async (
+  session: TaskUiSession,
+): Promise<Record<string, unknown>[]> => {
+  const documentation = await session.componentDocumentation(["Field.ImageAnnotation"]);
+  const output = documentation["Field.ImageAnnotation"]?.example?.output;
+  assert(output, "Field.ImageAnnotation has no documented example output");
+  const example: unknown = JSON.parse(output);
+  assert(isRecord(example), "Field.ImageAnnotation example output is not an object");
+  const annotations = example.roi;
+  assert(Array.isArray(annotations), "Field.ImageAnnotation example output has no roi array");
+  return annotations.map((annotation) => {
+    assert(isRecord(annotation), "Field.ImageAnnotation example contains a non-object value");
+    assert(
+      isRecord(annotation.properties),
+      "Field.ImageAnnotation example annotation has no properties",
+    );
+    return {
+      ...annotation,
+      properties: { ...annotation.properties, value: "face" },
+    };
+  });
+};
+
+describe("face image collection example", () => {
+  test.code(pipeline, orderFaces, "orders faces from top to bottom", async ({ run }) => {
+    await expect(run({ draft_faces: [lowerFace, upperFace] })).resolves.toEqual({
+      faces: [upperFace, lowerFace],
+    });
+  });
+
+  test.ui(pipeline, annotateFaces, "submits face annotations", async ({ render }) => {
+    const session = await render({ data: { image_url: imageUrl, draft_faces: [] } });
+    const annotations = await documentedFaceAnnotations(session);
+    await session.setValue("draft_faces", annotations);
+
+    expect(await session.getValue("draft_faces")).toEqual(annotations);
+    await expect(session.submit()).resolves.toMatchObject({ draft_faces: annotations });
+    await session.close();
+  });
+
+  test.pipeline(pipeline, "collects and orders face annotations", async ({ renderUi, runCode }) => {
+    const session = await renderUi(annotateFaces, {
+      data: { image_url: imageUrl, draft_faces: [] },
+    });
+    await session.setValue("draft_faces", [lowerFace, upperFace]);
+    const annotation = await session.submit();
+    const result = await runCode(orderFaces, { draft_faces: annotation.draft_faces });
+
+    expect(result).toEqual({ faces: [upperFace, lowerFace] });
+    await session.close();
+  });
+});

package/examples/video-pair-rating/README.md

@@ -33,12 +33,12 @@ Sixteen nodes: one root, eight `code`, three `match`, two `generation`, two `quo
 
 ## What to read first
 
-| file | why |
-|---|---|
-| `pipeline.ts` | the whole topology on one screen — every edge is explicit |
-| `nodes/cascade/node.ts` | the deterministic core that resolves every metric from rater arrays |
-| `nodes/rate_pair/node.ts` | a quorum node with real quality criteria and a Training course |
-| `nodes/route_overall/node.ts` | predicate routing, and what the `default` branch is for |
+| file                          | why                                                                 |
+| ----------------------------- | ------------------------------------------------------------------- |
+| `pipeline.ts`                 | the whole topology on one screen — every edge is explicit           |
+| `nodes/cascade/node.ts`       | the deterministic core that resolves every metric from rater arrays |
+| `nodes/rate_pair/node.ts`     | a quorum node with real quality criteria and a Training course      |
+| `nodes/route_overall/node.ts` | predicate routing, and what the `default` branch is for             |
 
 ## Conventions this example demonstrates
 
@@ -52,6 +52,14 @@ Sixteen nodes: one root, eight `code`, three `match`, two `generation`, two `quo
 
 **Adjudication is a branch, not a loop.** The graph stays acyclic. Redo, appeal, and payment retry live inside the human-work nodes; only the tie-breaking hand-off is a graph edge.
 
+## Check locally
+
+The shipped typed scenarios cover all eight code nodes, all four Task UIs, and an agreed-pair connected flow:
+
+```sh
+toloka check video-pair-rating
+```
+
 ## Provenance
 
 The topology, the resolution cascade, and the per-node layout are adapted from a design spike by @hexode. Media URLs, sample payloads, prompts, and prose here are synthetic: no customer data, project identifiers, or asset URLs from the original are carried over.

package/examples/video-pair-rating/nodes/adjudicate/node.ts

@@ -64,7 +64,7 @@ export const adjudicate = generation("adjudicate", {
     },
   ],
   ui: {
-    code: "<View.Group><If condition={actions.get('disagreed_metrics', []).includes('image_text_alignment')}><Field.SelectGroup path={'adjudicated_metrics.image_text_alignment'} label={'Imagery matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('motion_text_alignment')}><Field.SelectGroup path={'adjudicated_metrics.motion_text_alignment'} label={'Motion matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('image_quality')}><Field.SelectGroup path={'adjudicated_metrics.image_quality'} label={'Image quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('motion_quality')}><Field.SelectGroup path={'adjudicated_metrics.motion_quality'} label={'Motion quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('subject_background_consistency')}><Field.SelectGroup path={'adjudicated_metrics.subject_background_consistency'} label={'Subject and background consistency'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If><If condition={actions.get('disagreed_metrics', []).includes('video_realism')}><Field.SelectGroup path={'adjudicated_metrics.video_realism'} label={'Realism'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} /></If></View.Group>",
+    code: "<View.Card>{actions.get('disagreed_metrics', []).includes('image_text_alignment') && <Field.RadioGroup path={'adjudicated_metrics.image_text_alignment'} label={'Imagery matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('motion_text_alignment') && <Field.RadioGroup path={'adjudicated_metrics.motion_text_alignment'} label={'Motion matches the prompt'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('image_quality') && <Field.RadioGroup path={'adjudicated_metrics.image_quality'} label={'Image quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('motion_quality') && <Field.RadioGroup path={'adjudicated_metrics.motion_quality'} label={'Motion quality'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('subject_background_consistency') && <Field.RadioGroup path={'adjudicated_metrics.subject_background_consistency'} label={'Subject and background consistency'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}{actions.get('disagreed_metrics', []).includes('video_realism') && <Field.RadioGroup path={'adjudicated_metrics.video_realism'} label={'Realism'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />}</View.Card>",
     data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4","disagreed_metrics":["image_quality","motion_quality"]}',
     preprocess: "return {}",
     postprocess: "return {};",
@@ -157,6 +157,4 @@ export const adjudicate = generation("adjudicate", {
     perExpertCapEnabled: false,
     perExpertCapValue: 1,
   },
-  reuseAudienceWithNodeId: null,
-  smartSearch: null,
 });

package/examples/video-pair-rating/nodes/adjudicate_op/node.ts

@@ -50,7 +50,7 @@ export const adjudicateOp = generation("adjudicate_op", {
     },
   ],
   ui: {
-    code: "<Field.SelectGroup path={'op_adjudicated_value'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
+    code: "<Field.RadioGroup path={'op_adjudicated_value'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
     data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
     preprocess: "return {}",
     postprocess: "return {};",
@@ -147,6 +147,4 @@ export const adjudicateOp = generation("adjudicate_op", {
     perExpertCapEnabled: false,
     perExpertCapValue: 1,
   },
-  reuseAudienceWithNodeId: null,
-  smartSearch: null,
 });

package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts

@@ -48,7 +48,7 @@ export const overallPreferencePick = quorum("overall_preference_pick", {
   instruction:
     "Watch both videos in full, then pick the one you would ship. This is a single judgment about the whole video, not a sum of its parts. Use tie only when you genuinely have no preference. Unlike the metric rating task, this one shows the pair in its original order, so your pick needs no un-shuffling downstream.",
   ui: {
-    code: "<Field.SelectGroup path={'overall_manual_votes'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
+    code: "<Field.RadioGroup path={'overall_manual_votes'} options={[{ label: 'First video', value: 'video_a' }, { label: 'Second video', value: 'video_b' }, { label: 'Tie', value: 'tie' }]} />",
     data: '{"prompt":"a red fox crossing a snowy field at dawn","video_a":"https://example.com/video-a.mp4","video_b":"https://example.com/video-b.mp4"}',
     preprocess: "return {}",
     postprocess: "return {};",

package/examples/video-pair-rating/nodes/rate_pair/node.ts

@@ -96,7 +96,7 @@ export const ratePair = quorum("rate_pair", {
   instruction:
     "Watch both videos in full before answering. For each metric, pick the video that is better on that metric alone, or tie when they are equal. The two videos are presented in random order, so left and right carry no meaning. Different metrics can have different winners; that is expected.",
   ui: {
-    code: "<View.Group><Field.SelectGroup path={'image_text_alignment'} label={'Which video matches the prompt in its imagery?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'motion_text_alignment'} label={'Which video matches the prompt in its motion?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'image_quality'} label={'Which video looks better frame by frame?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'motion_quality'} label={'Which video moves more plausibly?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'subject_background_consistency'} label={'Which video keeps its subject and background stable?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.SelectGroup path={'video_realism'} label={'Which video reads as more real?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /></View.Group>",
+    code: "<View.Card><Field.RadioGroup path={'image_text_alignment'} label={'Which video matches the prompt in its imagery?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'motion_text_alignment'} label={'Which video matches the prompt in its motion?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'image_quality'} label={'Which video looks better frame by frame?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'motion_quality'} label={'Which video moves more plausibly?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'subject_background_consistency'} label={'Which video keeps its subject and background stable?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /><Field.RadioGroup path={'video_realism'} label={'Which video reads as more real?'} options={[{ label: 'Left', value: 'video_1' }, { label: 'Right', value: 'video_2' }, { label: 'Tie', value: 'tie' }]} /></View.Card>",
     data: '{"prompt":"a red fox crossing a snowy field at dawn","left_video":"https://example.com/video-left.mp4","right_video":"https://example.com/video-right.mp4"}',
     preprocess: "return {}",
     postprocess: "return {};",

package/examples/video-pair-rating/pipeline.browser.integration.spec.ts

@@ -0,0 +1,267 @@
+import { expect, test } from "@toloka-platform/sdk/testing";
+import { describe } from "vitest";
+
+import pipeline from "./pipeline";
+import { adjudicate } from "./nodes/adjudicate/node";
+import { adjudicateOp } from "./nodes/adjudicate_op/node";
+import { adjudicationMatch } from "./nodes/adjudication_match/node";
+import { assemble } from "./nodes/assemble/node";
+import { cascade } from "./nodes/cascade/node";
+import { finalizeOverall } from "./nodes/finalize_overall/node";
+import { mergeAdjudication } from "./nodes/merge_adjudication/node";
+import { mergeOpAdjudication } from "./nodes/merge_op_adjudication/node";
+import { opQuorumResolve } from "./nodes/op_quorum_resolve/node";
+import { overallPreferencePick } from "./nodes/overall_preference_pick/node";
+import { ratePair } from "./nodes/rate_pair/node";
+import { shuffleAb } from "./nodes/shuffle_ab/node";
+
+const item = {
+  prompt: "a fox crossing a snowy field",
+  video_a: "https://example.com/video-a.mp4",
+  video_b: "https://example.com/video-b.mp4",
+};
+const metricNames = [
+  "image_text_alignment",
+  "motion_text_alignment",
+  "image_quality",
+  "motion_quality",
+  "subject_background_consistency",
+  "video_realism",
+];
+const canonicalMetrics = Object.fromEntries(metricNames.map((name) => [name, "video_a"]));
+const metricVotes = Object.fromEntries(metricNames.map((name) => [name, ["video_1", "video_1"]]));
+
+describe("video pair rating example", () => {
+  test.code(pipeline, shuffleAb, "shuffles a pair deterministically", async ({ run }) => {
+    const result = await run(item);
+
+    expect(result).toMatchObject({
+      record_id: expect.stringMatching(/^[a-f0-9]{16}$/u),
+      swap: expect.any(Boolean),
+    });
+    expect(new Set([result.left_video, result.right_video])).toEqual(
+      new Set([item.video_a, item.video_b]),
+    );
+  });
+
+  test.code(pipeline, cascade, "resolves unanimous metric votes", async ({ run }) => {
+    await expect(run({ swap: false, ...metricVotes })).resolves.toMatchObject({
+      per_metric_resolved: canonicalMetrics,
+      needs_adjudication: "false",
+      op_value_or_null: "video_a",
+      op_source: "alignment",
+      needs_overall_manual: "false",
+    });
+  });
+
+  test.code(pipeline, mergeAdjudication, "preserves agreed metrics", async ({ run }) => {
+    await expect(
+      run({
+        per_metric_resolved: canonicalMetrics,
+        disagreed_metrics: [],
+      }),
+    ).resolves.toMatchObject({
+      final_metrics: canonicalMetrics,
+      resolution_method: "agreement",
+      op_value_or_null: "video_a",
+      needs_overall_manual: "false",
+    });
+  });
+
+  test.code(pipeline, opQuorumResolve, "resolves an overall majority", async ({ run }) => {
+    await expect(
+      run({
+        ...item,
+        swap: false,
+        overall_manual_votes: ["video_a", "video_a", "video_b"],
+      }),
+    ).resolves.toMatchObject({
+      op_manual_value: "video_a",
+      op_needs_adjudication: "false",
+      rater_op_votes: ["video_a", "video_a", "video_b"],
+    });
+  });
+
+  test.code(pipeline, mergeOpAdjudication, "uses the overall quorum majority", async ({ run }) => {
+    await expect(
+      run({ op_needs_adjudication: "false", op_manual_value: "video_a" }),
+    ).resolves.toEqual({
+      final_op_value: "video_a",
+      final_op_resolution_method: "agreement",
+      final_op_adjudicator_id: null,
+    });
+  });
+
+  test.code(pipeline, finalizeOverall, "finalizes a cascade preference", async ({ run }) => {
+    await expect(
+      run({
+        op_value_or_null: "video_a",
+        op_source: "alignment",
+        needs_overall_manual: "false",
+        final_metrics: canonicalMetrics,
+        resolution_method: "agreement",
+        annotator_id_adjudicator: null,
+      }),
+    ).resolves.toEqual({
+      overall_preference: "video_a",
+      overall_preference_source: "alignment",
+      resolution_method: "agreement",
+      annotator_id_adjudicator: null,
+    });
+  });
+
+  test.code(
+    pipeline,
+    adjudicationMatch,
+    "grades raters against an adjudicated metric",
+    async ({ run }) => {
+      await expect(
+        run({
+          rater_ita_votes: ["video_a", "video_b"],
+          rater_iq_votes: ["video_a", "video_a"],
+          disagreed_metrics: ["image_text_alignment"],
+          final_metrics: canonicalMetrics,
+          needs_overall_manual: "false",
+        }),
+      ).resolves.toEqual({
+        rater_ita_matches_adjudication: [true, false],
+        rater_iq_matches_adjudication: null,
+        rater_op_matches_adjudication: null,
+      });
+    },
+  );
+
+  test.code(pipeline, assemble, "assembles the delivery record", async ({ run }) => {
+    await expect(
+      run({
+        record_id: "record-1",
+        ...item,
+        final_metrics: canonicalMetrics,
+        overall_preference: "video_a",
+        overall_preference_source: "alignment",
+        resolution_method: "agreement",
+        annotator_id_adjudicator: null,
+        rater_ita_votes: ["video_a", "video_a"],
+        rater_mta_votes: ["video_a", "video_a"],
+        rater_iq_votes: ["video_a", "video_a"],
+        rater_mq_votes: ["video_a", "video_a"],
+        rater_sbc_votes: ["video_a", "video_a"],
+        rater_vr_votes: ["video_a", "video_a"],
+        rater_ita_matches_adjudication: null,
+        rater_iq_matches_adjudication: null,
+        rater_op_matches_adjudication: null,
+      }),
+    ).resolves.toMatchObject({
+      delivery_record: {
+        record_id: "record-1",
+        video_a_url: item.video_a,
+        video_b_url: item.video_b,
+        overall_preference: "video_a",
+        timestamp: expect.stringMatching(/Z$/u),
+      },
+    });
+  });
+
+  test.ui(pipeline, ratePair, "submits independent metric ratings", async ({ render }) => {
+    const session = await render({
+      data: { prompt: item.prompt, left_video: item.video_a, right_video: item.video_b },
+    });
+    for (const metric of metricNames) await session.setValue(metric, "video_1");
+
+    await expect(session.submit()).resolves.toMatchObject(
+      Object.fromEntries(metricNames.map((name) => [name, "video_1"])),
+    );
+    await session.close();
+  });
+
+  test.ui(pipeline, adjudicate, "submits disputed metric decisions", async ({ render }) => {
+    const session = await render({
+      data: { ...item, disagreed_metrics: ["image_quality", "motion_quality"] },
+    });
+    await session.setValue("adjudicated_metrics.image_quality", "video_a");
+    await session.setValue("adjudicated_metrics.motion_quality", "video_b");
+
+    await expect(session.submit()).resolves.toMatchObject({
+      adjudicated_metrics: { image_quality: "video_a", motion_quality: "video_b" },
+    });
+    await session.close();
+  });
+
+  test.ui(
+    pipeline,
+    overallPreferencePick,
+    "submits an overall preference vote",
+    async ({ render }) => {
+      const session = await render({ data: item });
+      await session.setValue("overall_manual_votes", "video_a");
+
+      await expect(session.submit()).resolves.toMatchObject({ overall_manual_votes: "video_a" });
+      await session.close();
+    },
+  );
+
+  test.ui(pipeline, adjudicateOp, "submits an overall adjudication", async ({ render }) => {
+    const session = await render({ data: item });
+    await session.setValue("op_adjudicated_value", "video_b");
+
+    await expect(session.submit()).resolves.toMatchObject({ op_adjudicated_value: "video_b" });
+    await session.close();
+  });
+
+  test.pipeline(pipeline, "rates and assembles an agreed pair", async ({ renderUi, runCode }) => {
+    const shuffled = await runCode(shuffleAb, item);
+    const session = await renderUi(ratePair, {
+      data: {
+        prompt: item.prompt,
+        left_video: shuffled.left_video,
+        right_video: shuffled.right_video,
+      },
+    });
+    for (const metric of metricNames) await session.setValue(metric, "video_1");
+    const rating = await session.submit();
+    const votes = Object.fromEntries(
+      metricNames.map((name) => [name, [rating[name], rating[name]]]),
+    );
+    const resolved = await runCode(cascade, { swap: shuffled.swap, ...votes });
+    const merged = await runCode(mergeAdjudication, {
+      per_metric_resolved: resolved.per_metric_resolved,
+      disagreed_metrics: resolved.disagreed_metrics,
+    });
+    const overall = await runCode(finalizeOverall, {
+      op_value_or_null: merged.op_value_or_null,
+      op_source: merged.op_source,
+      needs_overall_manual: merged.needs_overall_manual,
+      final_metrics: merged.final_metrics,
+      resolution_method: merged.resolution_method,
+      annotator_id_adjudicator: merged.annotator_id_adjudicator,
+    });
+    const matches = await runCode(adjudicationMatch, {
+      rater_ita_votes: resolved.rater_ita_votes,
+      rater_iq_votes: resolved.rater_iq_votes,
+      disagreed_metrics: resolved.disagreed_metrics,
+      final_metrics: merged.final_metrics,
+      needs_overall_manual: merged.needs_overall_manual,
+    });
+    const assembled = await runCode(assemble, {
+      record_id: shuffled.record_id,
+      ...item,
+      final_metrics: merged.final_metrics,
+      ...overall,
+      rater_ita_votes: resolved.rater_ita_votes,
+      rater_mta_votes: resolved.rater_mta_votes,
+      rater_iq_votes: resolved.rater_iq_votes,
+      rater_mq_votes: resolved.rater_mq_votes,
+      rater_sbc_votes: resolved.rater_sbc_votes,
+      rater_vr_votes: resolved.rater_vr_votes,
+      ...matches,
+    });
+
+    expect(assembled).toMatchObject({
+      delivery_record: {
+        record_id: shuffled.record_id,
+        overall_preference_source: "alignment",
+      },
+    });
+    await session.close();
+  });
+});

package/openapi/gen/api.ts

@@ -71,6 +71,9 @@ import type {
   RunItem,
   RunItemId,
   RunJob,
+  SampleDatasetItemsBody,
+  SampleJob,
+  SampleResult,
   SetProjectSecretBody,
   UpdatePipelineBody,
   UpdateProjectBody,
@@ -564,28 +567,31 @@ export const finalizeDatasetUpload = async (
   });
 };
 
-export const getGetDatasetUploadUrl = (uploadId: UploadId) => {
-  return `/api/v2-beta/uploads/${uploadId}`;
+export const getCancelDatasetUploadUrl = (uploadId: UploadId) => {
+  return `/api/v2-beta/uploads/${uploadId}/cancel`;
 };
 
-export const getDatasetUpload = async (
+export const cancelDatasetUpload = async (
   uploadId: UploadId,
   options?: RequestInit,
-): Promise<UploadJob> => {
-  return potatoFetch<UploadJob>(getGetDatasetUploadUrl(uploadId), {
+): Promise<void> => {
+  return potatoFetch<void>(getCancelDatasetUploadUrl(uploadId), {
     ...options,
-    method: "GET",
+    method: "POST",
   });
 };
 
-export const getCancelDatasetUploadUrl = (uploadId: UploadId) => {
-  return `/api/v2-beta/uploads/${uploadId}/cancel`;
+export const getGetDatasetUploadUrl = (uploadId: UploadId) => {
+  return `/api/v2-beta/uploads/${uploadId}`;
 };
 
-export const cancelDatasetUpload = async (uploadId: UploadId, options?: RequestInit): Promise<void> => {
-  return potatoFetch<void>(getCancelDatasetUploadUrl(uploadId), {
+export const getDatasetUpload = async (
+  uploadId: UploadId,
+  options?: RequestInit,
+): Promise<UploadJob> => {
+  return potatoFetch<UploadJob>(getGetDatasetUploadUrl(uploadId), {
     ...options,
-    method: "POST",
+    method: "GET",
   });
 };
 
@@ -745,6 +751,37 @@ export const cancelPipelineRun = async (runId: RunId, options?: RequestInit): Pr
   });
 };
 
+export const getSampleDatasetItemsUrl = (datasetId: DatasetId) => {
+  return `/api/v2-beta/datasets/${datasetId}/sample`;
+};
+
+export const sampleDatasetItems = async (
+  datasetId: DatasetId,
+  sampleDatasetItemsBody: SampleDatasetItemsBody,
+  options?: RequestInit,
+): Promise<SampleResult> => {
+  return potatoFetch<SampleResult>(getSampleDatasetItemsUrl(datasetId), {
+    ...options,
+    method: "POST",
+    headers: { "Content-Type": "application/json", ...options?.headers },
+    body: JSON.stringify(sampleDatasetItemsBody),
+  });
+};
+
+export const getGetDatasetSampleUrl = (sampleId: string) => {
+  return `/api/v2-beta/samples/${sampleId}`;
+};
+
+export const getDatasetSample = async (
+  sampleId: string,
+  options?: RequestInit,
+): Promise<SampleJob> => {
+  return potatoFetch<SampleJob>(getGetDatasetSampleUrl(sampleId), {
+    ...options,
+    method: "GET",
+  });
+};
+
 export const getCreateDatasetExportUrl = (datasetId: DatasetId) => {
   return `/api/v2-beta/datasets/${datasetId}/exports`;
 };

package/openapi/gen/models/createDatasetExportBody.ts

@@ -1,4 +1,4 @@
+import type { CreateDatasetExportBodyOneOf } from './createDatasetExportBodyOneOf';
+import type { CreateDatasetExportBodyOneOfThree } from './createDatasetExportBodyOneOfThree';
 
-export type CreateDatasetExportBody = {
-  fields: string[];
-};
+export type CreateDatasetExportBody = CreateDatasetExportBodyOneOf | CreateDatasetExportBodyOneOfThree;

package/openapi/gen/models/createDatasetExportBodyOneOf.ts

@@ -0,0 +1,6 @@
+import type { CreateDatasetExportBodyOneOfFormat } from './createDatasetExportBodyOneOfFormat';
+
+export type CreateDatasetExportBodyOneOf = {
+  format?: CreateDatasetExportBodyOneOfFormat;
+  fields: string[];
+};

package/openapi/gen/models/createDatasetExportBodyOneOfFormat.ts

@@ -0,0 +1,2 @@
+
+export type CreateDatasetExportBodyOneOfFormat = 'json-array';