@toloka-platform/sdk 0.1.0-preview.5 → 0.1.0-preview.7

package/examples/video-pair-rating/pipeline.browser.integration.spec.ts

@@ -0,0 +1,267 @@
+import { expect, test } from "@toloka-platform/sdk/testing";
+import { describe } from "vitest";
+
+import pipeline from "./pipeline";
+import { adjudicate } from "./nodes/adjudicate/node";
+import { adjudicateOp } from "./nodes/adjudicate_op/node";
+import { adjudicationMatch } from "./nodes/adjudication_match/node";
+import { assemble } from "./nodes/assemble/node";
+import { cascade } from "./nodes/cascade/node";
+import { finalizeOverall } from "./nodes/finalize_overall/node";
+import { mergeAdjudication } from "./nodes/merge_adjudication/node";
+import { mergeOpAdjudication } from "./nodes/merge_op_adjudication/node";
+import { opQuorumResolve } from "./nodes/op_quorum_resolve/node";
+import { overallPreferencePick } from "./nodes/overall_preference_pick/node";
+import { ratePair } from "./nodes/rate_pair/node";
+import { shuffleAb } from "./nodes/shuffle_ab/node";
+
+const item = {
+  prompt: "a fox crossing a snowy field",
+  video_a: "https://example.com/video-a.mp4",
+  video_b: "https://example.com/video-b.mp4",
+};
+const metricNames = [
+  "image_text_alignment",
+  "motion_text_alignment",
+  "image_quality",
+  "motion_quality",
+  "subject_background_consistency",
+  "video_realism",
+];
+const canonicalMetrics = Object.fromEntries(metricNames.map((name) => [name, "video_a"]));
+const metricVotes = Object.fromEntries(metricNames.map((name) => [name, ["video_1", "video_1"]]));
+
+describe("video pair rating example", () => {
+  test.code(pipeline, shuffleAb, "shuffles a pair deterministically", async ({ run }) => {
+    const result = await run(item);
+
+    expect(result).toMatchObject({
+      record_id: expect.stringMatching(/^[a-f0-9]{16}$/u),
+      swap: expect.any(Boolean),
+    });
+    expect(new Set([result.left_video, result.right_video])).toEqual(
+      new Set([item.video_a, item.video_b]),
+    );
+  });
+
+  test.code(pipeline, cascade, "resolves unanimous metric votes", async ({ run }) => {
+    await expect(run({ swap: false, ...metricVotes })).resolves.toMatchObject({
+      per_metric_resolved: canonicalMetrics,
+      needs_adjudication: "false",
+      op_value_or_null: "video_a",
+      op_source: "alignment",
+      needs_overall_manual: "false",
+    });
+  });
+
+  test.code(pipeline, mergeAdjudication, "preserves agreed metrics", async ({ run }) => {
+    await expect(
+      run({
+        per_metric_resolved: canonicalMetrics,
+        disagreed_metrics: [],
+      }),
+    ).resolves.toMatchObject({
+      final_metrics: canonicalMetrics,
+      resolution_method: "agreement",
+      op_value_or_null: "video_a",
+      needs_overall_manual: "false",
+    });
+  });
+
+  test.code(pipeline, opQuorumResolve, "resolves an overall majority", async ({ run }) => {
+    await expect(
+      run({
+        ...item,
+        swap: false,
+        overall_manual_votes: ["video_a", "video_a", "video_b"],
+      }),
+    ).resolves.toMatchObject({
+      op_manual_value: "video_a",
+      op_needs_adjudication: "false",
+      rater_op_votes: ["video_a", "video_a", "video_b"],
+    });
+  });
+
+  test.code(pipeline, mergeOpAdjudication, "uses the overall quorum majority", async ({ run }) => {
+    await expect(
+      run({ op_needs_adjudication: "false", op_manual_value: "video_a" }),
+    ).resolves.toEqual({
+      final_op_value: "video_a",
+      final_op_resolution_method: "agreement",
+      final_op_adjudicator_id: null,
+    });
+  });
+
+  test.code(pipeline, finalizeOverall, "finalizes a cascade preference", async ({ run }) => {
+    await expect(
+      run({
+        op_value_or_null: "video_a",
+        op_source: "alignment",
+        needs_overall_manual: "false",
+        final_metrics: canonicalMetrics,
+        resolution_method: "agreement",
+        annotator_id_adjudicator: null,
+      }),
+    ).resolves.toEqual({
+      overall_preference: "video_a",
+      overall_preference_source: "alignment",
+      resolution_method: "agreement",
+      annotator_id_adjudicator: null,
+    });
+  });
+
+  test.code(
+    pipeline,
+    adjudicationMatch,
+    "grades raters against an adjudicated metric",
+    async ({ run }) => {
+      await expect(
+        run({
+          rater_ita_votes: ["video_a", "video_b"],
+          rater_iq_votes: ["video_a", "video_a"],
+          disagreed_metrics: ["image_text_alignment"],
+          final_metrics: canonicalMetrics,
+          needs_overall_manual: "false",
+        }),
+      ).resolves.toEqual({
+        rater_ita_matches_adjudication: [true, false],
+        rater_iq_matches_adjudication: null,
+        rater_op_matches_adjudication: null,
+      });
+    },
+  );
+
+  test.code(pipeline, assemble, "assembles the delivery record", async ({ run }) => {
+    await expect(
+      run({
+        record_id: "record-1",
+        ...item,
+        final_metrics: canonicalMetrics,
+        overall_preference: "video_a",
+        overall_preference_source: "alignment",
+        resolution_method: "agreement",
+        annotator_id_adjudicator: null,
+        rater_ita_votes: ["video_a", "video_a"],
+        rater_mta_votes: ["video_a", "video_a"],
+        rater_iq_votes: ["video_a", "video_a"],
+        rater_mq_votes: ["video_a", "video_a"],
+        rater_sbc_votes: ["video_a", "video_a"],
+        rater_vr_votes: ["video_a", "video_a"],
+        rater_ita_matches_adjudication: null,
+        rater_iq_matches_adjudication: null,
+        rater_op_matches_adjudication: null,
+      }),
+    ).resolves.toMatchObject({
+      delivery_record: {
+        record_id: "record-1",
+        video_a_url: item.video_a,
+        video_b_url: item.video_b,
+        overall_preference: "video_a",
+        timestamp: expect.stringMatching(/Z$/u),
+      },
+    });
+  });
+
+  test.ui(pipeline, ratePair, "submits independent metric ratings", async ({ render }) => {
+    const session = await render({
+      data: { prompt: item.prompt, left_video: item.video_a, right_video: item.video_b },
+    });
+    for (const metric of metricNames) await session.setValue(metric, "video_1");
+
+    await expect(session.submit()).resolves.toMatchObject(
+      Object.fromEntries(metricNames.map((name) => [name, "video_1"])),
+    );
+    await session.close();
+  });
+
+  test.ui(pipeline, adjudicate, "submits disputed metric decisions", async ({ render }) => {
+    const session = await render({
+      data: { ...item, disagreed_metrics: ["image_quality", "motion_quality"] },
+    });
+    await session.setValue("adjudicated_metrics.image_quality", "video_a");
+    await session.setValue("adjudicated_metrics.motion_quality", "video_b");
+
+    await expect(session.submit()).resolves.toMatchObject({
+      adjudicated_metrics: { image_quality: "video_a", motion_quality: "video_b" },
+    });
+    await session.close();
+  });
+
+  test.ui(
+    pipeline,
+    overallPreferencePick,
+    "submits an overall preference vote",
+    async ({ render }) => {
+      const session = await render({ data: item });
+      await session.setValue("overall_manual_votes", "video_a");
+
+      await expect(session.submit()).resolves.toMatchObject({ overall_manual_votes: "video_a" });
+      await session.close();
+    },
+  );
+
+  test.ui(pipeline, adjudicateOp, "submits an overall adjudication", async ({ render }) => {
+    const session = await render({ data: item });
+    await session.setValue("op_adjudicated_value", "video_b");
+
+    await expect(session.submit()).resolves.toMatchObject({ op_adjudicated_value: "video_b" });
+    await session.close();
+  });
+
+  test.pipeline(pipeline, "rates and assembles an agreed pair", async ({ renderUi, runCode }) => {
+    const shuffled = await runCode(shuffleAb, item);
+    const session = await renderUi(ratePair, {
+      data: {
+        prompt: item.prompt,
+        left_video: shuffled.left_video,
+        right_video: shuffled.right_video,
+      },
+    });
+    for (const metric of metricNames) await session.setValue(metric, "video_1");
+    const rating = await session.submit();
+    const votes = Object.fromEntries(
+      metricNames.map((name) => [name, [rating[name], rating[name]]]),
+    );
+    const resolved = await runCode(cascade, { swap: shuffled.swap, ...votes });
+    const merged = await runCode(mergeAdjudication, {
+      per_metric_resolved: resolved.per_metric_resolved,
+      disagreed_metrics: resolved.disagreed_metrics,
+    });
+    const overall = await runCode(finalizeOverall, {
+      op_value_or_null: merged.op_value_or_null,
+      op_source: merged.op_source,
+      needs_overall_manual: merged.needs_overall_manual,
+      final_metrics: merged.final_metrics,
+      resolution_method: merged.resolution_method,
+      annotator_id_adjudicator: merged.annotator_id_adjudicator,
+    });
+    const matches = await runCode(adjudicationMatch, {
+      rater_ita_votes: resolved.rater_ita_votes,
+      rater_iq_votes: resolved.rater_iq_votes,
+      disagreed_metrics: resolved.disagreed_metrics,
+      final_metrics: merged.final_metrics,
+      needs_overall_manual: merged.needs_overall_manual,
+    });
+    const assembled = await runCode(assemble, {
+      record_id: shuffled.record_id,
+      ...item,
+      final_metrics: merged.final_metrics,
+      ...overall,
+      rater_ita_votes: resolved.rater_ita_votes,
+      rater_mta_votes: resolved.rater_mta_votes,
+      rater_iq_votes: resolved.rater_iq_votes,
+      rater_mq_votes: resolved.rater_mq_votes,
+      rater_sbc_votes: resolved.rater_sbc_votes,
+      rater_vr_votes: resolved.rater_vr_votes,
+      ...matches,
+    });
+
+    expect(assembled).toMatchObject({
+      delivery_record: {
+        record_id: shuffled.record_id,
+        overall_preference_source: "alignment",
+      },
+    });
+    await session.close();
+  });
+});