@toloka-platform/sdk 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/toloka.js +157 -154
- package/dist/cli/toloka.js.jsc +0 -0
- package/dist/cli/toloka.js.jsc.sha256 +1 -1
- package/dist/cli/toloka.js.sha256 +1 -1
- package/docs/authoring.md +6 -4
- package/docs/cli.md +30 -11
- package/docs/limitations.md +2 -2
- package/docs/testing.md +4 -2
- package/examples/README.md +11 -0
- package/examples/app-crowdtesting/README.md +41 -0
- package/examples/app-crowdtesting/fields.ts +6 -0
- package/examples/app-crowdtesting/nodes/app-flow-test/node.ts +381 -0
- package/examples/app-crowdtesting/nodes/collection/node.ts +8 -0
- package/examples/app-crowdtesting/pipeline.browser.integration.spec.ts +77 -0
- package/examples/app-crowdtesting/pipeline.ts +17 -0
- package/examples/egocentric-video-collection/README.md +35 -0
- package/examples/egocentric-video-collection/fields.ts +24 -0
- package/examples/egocentric-video-collection/nodes/collection/node.ts +5 -0
- package/examples/egocentric-video-collection/nodes/record-activity/node.ts +760 -0
- package/examples/egocentric-video-collection/pipeline.browser.integration.spec.ts +65 -0
- package/examples/egocentric-video-collection/pipeline.ts +17 -0
- package/examples/product-moderation/README.md +49 -0
- package/examples/product-moderation/fields.ts +37 -0
- package/examples/product-moderation/nodes/collection/node.ts +16 -0
- package/examples/product-moderation/nodes/moderate-product/instruction.ts +77 -0
- package/examples/product-moderation/nodes/moderate-product/node.ts +368 -0
- package/examples/product-moderation/pipeline.browser.integration.spec.ts +48 -0
- package/examples/product-moderation/pipeline.ts +33 -0
- package/examples/video-pair-rating/fields.ts +16 -4
- package/examples/video-pair-rating/nodes/adjudicate/node.ts +78 -3
- package/examples/video-pair-rating/nodes/adjudicate_op/node.ts +28 -3
- package/examples/video-pair-rating/nodes/overall_preference_pick/node.ts +28 -3
- package/examples/video-pair-rating/nodes/rate_pair/node.ts +65 -2
- package/openapi/gen/api.ts +22 -0
- package/openapi/gen/models/generationBudget.ts +8 -0
- package/openapi/gen/models/generationBudgetReasoningEffort.ts +2 -0
- package/openapi/gen/models/generationModel.ts +12 -0
- package/openapi/gen/models/generationModelDefaultReasoningEffort.ts +2 -0
- package/openapi/gen/models/generationModelId.ts +2 -0
- package/openapi/gen/models/index.ts +8 -0
- package/openapi/gen/models/invalidFieldSchema.ts +6 -0
- package/openapi/gen/models/listGenerationModels200.ts +7 -0
- package/openapi/gen/models/listGenerationModels200DefaultGenerationModelId.ts +2 -0
- package/openapi/gen/models/pipelineGraphSnapshot.ts +2 -0
- package/package.json +2 -2
- package/skills/catalog.json +16 -12
- package/skills/toloka-build/SKILL.md +44 -58
- package/skills/toloka-build/references/human-work.md +156 -0
- package/skills/toloka-build/references/local-setup.md +136 -0
- package/skills/toloka-build/references/node-checks.md +11 -0
- package/skills/toloka-build/references/node-semantics.md +22 -213
- package/skills/toloka-build/references/task-ui.md +61 -0
- package/skills/toloka-build/references/workflow-and-cli.md +29 -229
- package/skills/toloka-fine-tuning/SKILL.md +1 -1
- package/skills/toloka-get-started/SKILL.md +1 -1
- package/skills/toloka-gisting/SKILL.md +1 -1
- package/src/authoring/index.ts +26 -7
- package/src/checks/task-ui-typecheck.ts +3 -3
- package/src/cli/analytics.ts +1 -0
- package/src/cli/check.ts +83 -12
- package/src/cli/commands/generation-model.ts +18 -0
- package/src/cli/commands/pipeline.ts +1 -1
- package/src/cli/commands/source.ts +14 -3
- package/src/cli/generation-models.ts +46 -0
- package/src/cli/program.ts +2 -0
- package/src/cli/project.ts +288 -138
- package/src/cli/trainings.ts +12 -0
- package/src/cli/ui.ts +3 -1
- package/src/client.ts +5 -0
- package/src/graph-rules.json +11 -0
- package/src/importer.ts +46 -40
- package/src/index.ts +4 -0
- package/src/validate.ts +183 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { expect, test } from "@toloka-platform/sdk/testing";
|
|
2
|
+
import { describe } from "vitest";
|
|
3
|
+
import { assert } from "es-toolkit";
|
|
4
|
+
import type { TaskUiSession } from "@toloka-platform/sdk/testing";
|
|
5
|
+
|
|
6
|
+
import pipeline from "./pipeline";
|
|
7
|
+
import { recordActivity } from "./nodes/record-activity/node";
|
|
8
|
+
|
|
9
|
+
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
10
|
+
typeof value === "object" && value !== null && !Array.isArray(value);
|
|
11
|
+
|
|
12
|
+
const documentedRecording = async (session: TaskUiSession): Promise<Record<string, unknown>> => {
|
|
13
|
+
const documentation = await session.componentDocumentation(["Field.VideoRecord"]);
|
|
14
|
+
const output = documentation["Field.VideoRecord"]?.example?.output;
|
|
15
|
+
assert(output, "Field.VideoRecord has no documented example output");
|
|
16
|
+
const example: unknown = JSON.parse(output);
|
|
17
|
+
assert(isRecord(example), "Field.VideoRecord example output is not an object");
|
|
18
|
+
const [recording] = Object.values(example);
|
|
19
|
+
assert(Array.isArray(recording), "Field.VideoRecord example output is not an array");
|
|
20
|
+
const [take] = recording;
|
|
21
|
+
assert(isRecord(take), "Field.VideoRecord example contains a non-object value");
|
|
22
|
+
assert(take.status === "done", "Field.VideoRecord example take has not finished uploading");
|
|
23
|
+
assert(typeof take.sasLink === "string", "Field.VideoRecord example take has no sasLink");
|
|
24
|
+
return take;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
describe("egocentric video collection example", () => {
|
|
28
|
+
test.ui(pipeline, recordActivity, "submits a recording with its category", async ({ render }) => {
|
|
29
|
+
const session = await render({ data: { id: 6 }, allowPartialCoverage: true });
|
|
30
|
+
const take = await documentedRecording(session);
|
|
31
|
+
await session.setValue("task_category", "cleaning");
|
|
32
|
+
await session.setValue("video_file", [take]);
|
|
33
|
+
|
|
34
|
+
expect(await session.getValue("video_file")).toEqual([take]);
|
|
35
|
+
await expect(session.submit()).resolves.toMatchObject({
|
|
36
|
+
id: 6,
|
|
37
|
+
task_category: "cleaning",
|
|
38
|
+
video_file: [take],
|
|
39
|
+
});
|
|
40
|
+
await session.close();
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test.pipeline(
|
|
44
|
+
pipeline,
|
|
45
|
+
"writes the recording link and category back to the slot",
|
|
46
|
+
async ({ renderUi }) => {
|
|
47
|
+
const session = await renderUi(recordActivity, {
|
|
48
|
+
data: { id: 7 },
|
|
49
|
+
allowPartialCoverage: true,
|
|
50
|
+
});
|
|
51
|
+
const take = await documentedRecording(session);
|
|
52
|
+
await session.setValue("task_category", "food_preparation");
|
|
53
|
+
await session.setValue("video_file", [take]);
|
|
54
|
+
const annotation = await session.submit();
|
|
55
|
+
|
|
56
|
+
const [submittedTake] = Array.isArray(annotation.video_file) ? annotation.video_file : [];
|
|
57
|
+
assert(isRecord(submittedTake), "The submission has no recorded take");
|
|
58
|
+
expect({
|
|
59
|
+
recorded_video: submittedTake.sasLink,
|
|
60
|
+
task_category: annotation.task_category,
|
|
61
|
+
}).toEqual({ recorded_video: take.sasLink, task_category: "food_preparation" });
|
|
62
|
+
await session.close();
|
|
63
|
+
},
|
|
64
|
+
);
|
|
65
|
+
});
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { connect, definePipeline } from "@toloka-platform/sdk";
|
|
2
|
+
|
|
3
|
+
import { recordedVideo, slotId, taskCategory } from "./fields";
|
|
4
|
+
import { collection } from "./nodes/collection/node";
|
|
5
|
+
import { recordActivity } from "./nodes/record-activity/node";
|
|
6
|
+
|
|
7
|
+
export const pipeline = definePipeline({
|
|
8
|
+
dataset: "egocentric-videos",
|
|
9
|
+
fields: [slotId, recordedVideo, taskCategory],
|
|
10
|
+
nodes: [collection, recordActivity],
|
|
11
|
+
connections: [
|
|
12
|
+
connect(collection.started, recordActivity.input),
|
|
13
|
+
connect(recordActivity.output, collection.completed),
|
|
14
|
+
],
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
export default pipeline;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Product moderation
|
|
2
|
+
|
|
3
|
+
The simplest complete human-work pipeline: one `generation` node in which an expert looks at a product image and description and returns a label, a subcategory, and a short rationale. Start here for any "show an item, collect a judgment" task such as classification, moderation, or rating.
|
|
4
|
+
|
|
5
|
+
## Shape
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
root.started product_id, product_category (optional), image_link, product_description
|
|
9
|
+
└─> moderate_product generation label + subcategory + rationale
|
|
10
|
+
└─> root.completed moderation_label, moderation_subcategory, moderation_rationale
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## What to read first
|
|
14
|
+
|
|
15
|
+
| file | why |
|
|
16
|
+
| --------------------------------------- | ------------------------------------------------------------------------------------- |
|
|
17
|
+
| `fields.ts` | label lists declared once and reused by the field `enum`, `allowedLabels`, and the UI |
|
|
18
|
+
| `nodes/moderate-product/node.ts` | every setting a human generation node needs, and nothing more |
|
|
19
|
+
| `nodes/moderate-product/instruction.ts` | a long guideline kept out of the node file |
|
|
20
|
+
| `pipeline.browser.integration.spec.ts` | a UI scenario that checks required answers and the submitted verdict |
|
|
21
|
+
|
|
22
|
+
## Patterns this example demonstrates
|
|
23
|
+
|
|
24
|
+
**Classification outputs.** `moderation_label` and `moderation_subcategory` are `SINGLE_LABEL` entities whose `allowedLabels` come from the same lists as the field `enum` and the Task UI options (`ui.constants`, read as `CONST.LABELS`). `moderation_rationale` is an `EXPLANATION` that references the label.
|
|
25
|
+
|
|
26
|
+
**Criteria per output.** Each quality criterion targets one output: the verdict, the consistency of the subcategory with the verdict, and a rationale grounded in what is actually shown.
|
|
27
|
+
|
|
28
|
+
**A two-column screen.** The product on the left with its image height capped, the answers on the right, every answer `required`, and the free-text answer limited with `max`.
|
|
29
|
+
|
|
30
|
+
**Stored checks.** Six `annotation_qa` cases pair a product with the expected verdict: three acceptable products (patriotic identity twice and a plain logo), a flag with a slogan that takes a side, a faith slogan, and an insult.
|
|
31
|
+
|
|
32
|
+
## Variants
|
|
33
|
+
|
|
34
|
+
**Synthetic generation.** Replace `audienceAndPricing` with a synthetic block, drop the `onboarding` block (synthetic generation does not use Training) and any human QA mode, and add `synthgen` cases next to `annotation_qa`; the entities, criteria, instruction, and Task UI stay. Pick the model, reasoning effort, and budget from `toloka generation-model list`:
|
|
35
|
+
|
|
36
|
+
```ts
|
|
37
|
+
audienceAndPricing: {
|
|
38
|
+
audienceType: "synthetic",
|
|
39
|
+
generationModelId: "gpt-5.6-terra",
|
|
40
|
+
reasoningEffort: "medium",
|
|
41
|
+
llmInstruction:
|
|
42
|
+
"Look carefully at the product image, including any text printed on the product, and read the description. Apply the guidelines exactly, including the precedence rule.",
|
|
43
|
+
maxCostPerAttempt: 0.7,
|
|
44
|
+
},
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
A `synthgen` case holds only the input: `{ id, name, input: { input: JSON.stringify(product) } }`.
|
|
48
|
+
|
|
49
|
+
**Several experts per product.** Use `quorum` with an `overlap` instead of `generation`. Each output then becomes an array with one element per expert, so a single final verdict needs a code node that resolves the votes; see `video-pair-rating` (`rate_pair` → `cascade`).
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { field } from "@toloka-platform/sdk";
|
|
2
|
+
|
|
3
|
+
export const moderationLabels = [
|
|
4
|
+
{ label: "Opinionated", value: "opinionated" },
|
|
5
|
+
{ label: "Offensive", value: "offensive" },
|
|
6
|
+
{ label: "Acceptable", value: "acceptable" },
|
|
7
|
+
];
|
|
8
|
+
|
|
9
|
+
export const moderationSubcategories = [
|
|
10
|
+
{ label: "None (acceptable)", value: "none" },
|
|
11
|
+
{ label: "Political", value: "political" },
|
|
12
|
+
{ label: "Religious", value: "religious" },
|
|
13
|
+
{ label: "Drug reference", value: "drug_reference" },
|
|
14
|
+
{ label: "Implied violence", value: "implied_violence" },
|
|
15
|
+
{ label: "Racist", value: "racist" },
|
|
16
|
+
{ label: "Sexual implicit", value: "sexual_implicit" },
|
|
17
|
+
{ label: "Profanity", value: "profanity" },
|
|
18
|
+
{ label: "Derogatory words", value: "derogatory" },
|
|
19
|
+
{ label: "Outright offensive", value: "outright_offensive" },
|
|
20
|
+
{ label: "Gore", value: "gore" },
|
|
21
|
+
{ label: "Unacceptable edgy", value: "unacceptable_edgy" },
|
|
22
|
+
{ label: "Negative sports team", value: "negative_sports_team" },
|
|
23
|
+
];
|
|
24
|
+
|
|
25
|
+
export const productId = field("product_id", { type: "integer" });
|
|
26
|
+
export const productCategory = field("product_category", { type: "string" });
|
|
27
|
+
export const imageLink = field("image_link", { type: "string", format: "uri" });
|
|
28
|
+
export const productDescription = field("product_description", { type: "string" });
|
|
29
|
+
export const moderationLabel = field("moderation_label", {
|
|
30
|
+
type: "string",
|
|
31
|
+
enum: moderationLabels.map((label) => label.value),
|
|
32
|
+
});
|
|
33
|
+
export const moderationSubcategory = field("moderation_subcategory", {
|
|
34
|
+
type: "string",
|
|
35
|
+
enum: moderationSubcategories.map((subcategory) => subcategory.value),
|
|
36
|
+
});
|
|
37
|
+
export const moderationRationale = field("moderation_rationale", { type: "string" });
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { root } from "@toloka-platform/sdk";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
imageLink,
|
|
5
|
+
moderationLabel,
|
|
6
|
+
moderationRationale,
|
|
7
|
+
moderationSubcategory,
|
|
8
|
+
productCategory,
|
|
9
|
+
productDescription,
|
|
10
|
+
productId,
|
|
11
|
+
} from "../../fields";
|
|
12
|
+
|
|
13
|
+
export const collection = root({
|
|
14
|
+
started: [productId, { field: productCategory, optional: true }, imageLink, productDescription],
|
|
15
|
+
completed: [moderationLabel, moderationSubcategory, moderationRationale],
|
|
16
|
+
});
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
export const instruction = `# Product content moderation
|
|
2
|
+
|
|
3
|
+
Review the product image, including any text visible on the product, together with the product category and description. Then choose one label, one subcategory, and write a short rationale.
|
|
4
|
+
|
|
5
|
+
## Labels and precedence
|
|
6
|
+
|
|
7
|
+
- **Opinionated**: political or religious advocacy and explicit references to taking illegal drugs, as defined below.
|
|
8
|
+
- **Offensive**: other content we moderate: outright offensive content, implied violence, racism, sexual innuendo, profanity, derogatory words, gore, unacceptable edgy content, and negative sports-team merchandise.
|
|
9
|
+
- **Acceptable**: everything else, including acceptable edgy content.
|
|
10
|
+
- If a product is both offensive and opinionated, label it **Opinionated**. Flagging opinionated content is the primary goal.
|
|
11
|
+
- Be conservative with political and religious content: when you are unsure whether a design asserts a viewpoint or urges change, label it Opinionated.
|
|
12
|
+
|
|
13
|
+
## Subcategory
|
|
14
|
+
|
|
15
|
+
- Opinionated: political, religious, drug_reference.
|
|
16
|
+
- Offensive: implied_violence, racist, sexual_implicit, profanity, derogatory, outright_offensive, gore, unacceptable_edgy, negative_sports_team.
|
|
17
|
+
- Acceptable: none.
|
|
18
|
+
|
|
19
|
+
## Political (Opinionated)
|
|
20
|
+
|
|
21
|
+
Flag images or visible text that advocate for or against political parties, candidates, public policies, or protected groups (race, gender identity, sexual orientation, religion, immigration status, special-interest groups); that ask people to act (vote, donate, protest); or that promote extremist, violent, or hateful ideologies. Also flag satire that keeps a clear political message, merchandise of advocacy movements, and symbols adopted by a political movement when they convey an opinion. Symbols of hate groups and calls for harm are always Opinionated.
|
|
22
|
+
|
|
23
|
+
Identity is not advocacy. A symbol or label that only states who someone is stays Acceptable; the same symbol paired with a slogan that takes a side becomes Opinionated.
|
|
24
|
+
|
|
25
|
+
- Flags: a standalone national flag is Acceptable, even for a country at war. A flag with a slogan that takes a side ("Victory for Freedonia") is Opinionated.
|
|
26
|
+
- Patriotism and service: "Proud to be American", "Support our veterans", a cap with the name of a military branch, and a photo of a tank are Acceptable. "Bring our troops home" and "You should salute when a marine walks into a room!" are Opinionated.
|
|
27
|
+
- Identity: a standalone identity flag or gender symbol, a couple of any kind, "Vegan", and "Gun Owner" are Acceptable. "Go Vegan", a gender symbol with a raised fist, a slogan that states a position on a contested social issue, and "GUN OWNER" printed over a national flag are Opinionated.
|
|
28
|
+
- Causes: the Red Cross, animal shelters, "Support our firefighters", and certifications such as Halal, Kosher, or Fair Trade are Acceptable. Merchandise of a party or an advocacy campaign ("Vote Harbor Party", "Stop the Riverside Dam") is Opinionated.
|
|
29
|
+
- People: well-known historical figures on currency are Acceptable. A current political or world leader on a product, or a politician edited onto a banknote, is Opinionated.
|
|
30
|
+
- Books: factual history books and memoirs are Acceptable, even about wars or atrocities. Books written by leaders of hate or genocidal movements are Opinionated.
|
|
31
|
+
- Parody: a parody of a campaign slogan keeps its political meaning and is Opinionated.
|
|
32
|
+
- Guns: a standalone photo of a gun, holsters, and a logo with bullets are Acceptable. A gun pointed at a person or a gun over a national flag is Opinionated.
|
|
33
|
+
- Single words that take a side (resist, boycott, protest, endorse) are Opinionated when the target is political. Neutral words ("Believe", "Freedom") are Acceptable.
|
|
34
|
+
|
|
35
|
+
## Religious (Opinionated)
|
|
36
|
+
|
|
37
|
+
Neutral religious symbols state identity, not advocacy: a cross or Star of David necklace without text, a Buddha statue, an image of a deity, incense for meditation, and crystals sold without claims are Acceptable. Label as Opinionated:
|
|
38
|
+
|
|
39
|
+
- text that professes or promotes belief or asks people to convert, anywhere on the product or in the description ("Faith will save you", "Show your faith with this necklace", scripture verses, "FAITH" written on a cross);
|
|
40
|
+
- religious texts and ritual tools: prayer beads, a rosary even when sold as "decorative beads", spirit boards, oils sold to ward off evil spirits;
|
|
41
|
+
- metaphysical claims ("aligns your chakras", "cleanses your aura");
|
|
42
|
+
- mock religions used as social commentary, and content that denigrates believers.
|
|
43
|
+
|
|
44
|
+
Fiction is fine: games and stories with clearly fictional gods, magic, or dark cults are Acceptable unless they present a real living tradition as fantasy or as evil. A cauldron is Acceptable unless it is marketed for spells.
|
|
45
|
+
|
|
46
|
+
## Offensive
|
|
47
|
+
|
|
48
|
+
- outright_offensive: exploiting tragedies, body shaming, mocking mental health or disabilities, cultural appropriation.
|
|
49
|
+
- implied_violence: wording or imagery that implies violence toward people or places. Violence in a political or religious context, or against a protected group, is Opinionated instead.
|
|
50
|
+
- racist: negative racial stereotypes or hostility toward a racial group. When tied to a political group it is Opinionated.
|
|
51
|
+
- sexual_implicit: depicted sexual acts or slogans with a sexual double meaning.
|
|
52
|
+
- profanity: explicit swear words. "AF", "heck", "darn", and "crap" do not count.
|
|
53
|
+
- derogatory: insulting words, even when playful ("idiot", "moron").
|
|
54
|
+
- negative_sports_team: a negative message about a team ("Riverside FC are the worst!").
|
|
55
|
+
|
|
56
|
+
## Edgy
|
|
57
|
+
|
|
58
|
+
Acceptable edgy content is ironic, irreverent, or darkly funny about universal experiences, with mild profanity or self-deprecation and no attack on people: "It's 5 o'clock somewhere!", "Nothing like a cold beer after a cold beer", "Thick AF", "Dead inside but thriving!".
|
|
59
|
+
|
|
60
|
+
Unacceptable edgy content is Offensive (unacceptable_edgy): jokes about suicide or self-harm, instructions for illegal acts ("Sell drugs, make money"), harassment, humiliation, and shock without satire.
|
|
61
|
+
|
|
62
|
+
## Drugs and nicotine
|
|
63
|
+
|
|
64
|
+
Alcohol and nicotine are Acceptable edgy content. Psychedelic art and ambiguous phrases such as "mind expanding" are Acceptable. Explicit references to taking illegal drugs ("Shrooms", "Acid") are Opinionated (drug_reference).
|
|
65
|
+
|
|
66
|
+
## Skulls, zombies, and pop culture
|
|
67
|
+
|
|
68
|
+
- Skulls and skeletons are edgy, not offensive. Judge the scene, not the skull: a skeleton firefighter is Acceptable; a dead soldier lying on a battlefield is Offensive.
|
|
69
|
+
- Cartoon zombies are Acceptable. Realistic corpses and visible gore are Offensive (gore). If a case is borderline, flag it.
|
|
70
|
+
- Band and song names with religious words ("Lamb of God") are Acceptable; an album title that professes faith is Opinionated. A flag or religious symbol being burned is Opinionated.
|
|
71
|
+
- Hopes and identity words ("Long Live Rock", "Queen", "Divine"), fictional dark characters (the Grim Reaper, a plague doctor), and brand names with a country ("Harbor USA") are Acceptable. Territorial claims ("Freedonia owns Sylvania") are Opinionated.
|
|
72
|
+
- To separate offensive from edgy, ask: is it punching down, provoking a protected group, or vulgar? Be lenient with edgy content that stays short of offensive.
|
|
73
|
+
|
|
74
|
+
## Rationale
|
|
75
|
+
|
|
76
|
+
In one to three sentences, name the specific element (visible text, symbol, imagery, or a phrase in the description) that decided the label and the rule you applied. For Acceptable items, say briefly why nothing is flagged, for example: "A standalone American flag and 'Live Free' are patriotic identity statements."
|
|
77
|
+
`;
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
import { generation } from "@toloka-platform/sdk";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
imageLink,
|
|
5
|
+
moderationLabel,
|
|
6
|
+
moderationLabels,
|
|
7
|
+
moderationRationale,
|
|
8
|
+
moderationSubcategories,
|
|
9
|
+
moderationSubcategory,
|
|
10
|
+
productCategory,
|
|
11
|
+
productDescription,
|
|
12
|
+
} from "../../fields";
|
|
13
|
+
import { instruction } from "./instruction";
|
|
14
|
+
|
|
15
|
+
const teeCategory = "Apparel & Accessories > Clothing > Clothing Tops > T-Shirts";
|
|
16
|
+
const hairTieCategory =
|
|
17
|
+
"Apparel & Accessories > Clothing Accessories > Hair Accessories > Ponytail Holders";
|
|
18
|
+
|
|
19
|
+
const reviewCase = (
|
|
20
|
+
id: string,
|
|
21
|
+
name: string,
|
|
22
|
+
product: { image_link: string; product_category: string; product_description: string },
|
|
23
|
+
verdict: {
|
|
24
|
+
moderation_label: string;
|
|
25
|
+
moderation_subcategory: string;
|
|
26
|
+
moderation_rationale: string;
|
|
27
|
+
},
|
|
28
|
+
) => ({
|
|
29
|
+
id,
|
|
30
|
+
name,
|
|
31
|
+
input: { taskInput: JSON.stringify(product), taskOutput: JSON.stringify(verdict) },
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
export const moderateProduct = generation("moderate_product", {
|
|
35
|
+
about: {
|
|
36
|
+
description:
|
|
37
|
+
"Moderate a product image and description: Opinionated (political or religious), Offensive, or Acceptable.",
|
|
38
|
+
projectType: "general",
|
|
39
|
+
},
|
|
40
|
+
publicName: "Product content moderation",
|
|
41
|
+
audienceAndPricing: {
|
|
42
|
+
audienceType: "human",
|
|
43
|
+
verifiedExperts: true,
|
|
44
|
+
audience: {
|
|
45
|
+
languages: { type: "and", values: [{ value: "English" }] },
|
|
46
|
+
specializations: { type: "and", values: [] },
|
|
47
|
+
countries: { type: "or", values: [] },
|
|
48
|
+
},
|
|
49
|
+
pricePerTask: 0.2,
|
|
50
|
+
timeForTask: 120,
|
|
51
|
+
perExpertCapEnabled: false,
|
|
52
|
+
perExpertCapValue: 1,
|
|
53
|
+
},
|
|
54
|
+
entities: {
|
|
55
|
+
input: [
|
|
56
|
+
{
|
|
57
|
+
field: imageLink,
|
|
58
|
+
entityType: "IMAGE",
|
|
59
|
+
description: "Product image, including any visible text or graphics.",
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
field: productDescription,
|
|
63
|
+
entityType: "CONTEXT",
|
|
64
|
+
description: "Seller's product description.",
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
field: productCategory,
|
|
68
|
+
entityType: "CONTEXT",
|
|
69
|
+
optional: true,
|
|
70
|
+
description: "Product taxonomy category.",
|
|
71
|
+
},
|
|
72
|
+
],
|
|
73
|
+
output: [
|
|
74
|
+
{
|
|
75
|
+
field: moderationLabel,
|
|
76
|
+
entityType: "SINGLE_LABEL",
|
|
77
|
+
jsonPath: "$.moderation_label",
|
|
78
|
+
description: "Moderation verdict; Opinionated takes precedence over Offensive.",
|
|
79
|
+
allowedLabels: moderationLabels,
|
|
80
|
+
relations: [
|
|
81
|
+
{ field: imageLink, rel_type: "evaluates" },
|
|
82
|
+
{ field: productDescription, rel_type: "evaluates" },
|
|
83
|
+
],
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
field: moderationSubcategory,
|
|
87
|
+
entityType: "SINGLE_LABEL",
|
|
88
|
+
jsonPath: "$.moderation_subcategory",
|
|
89
|
+
description: "Specific reason for the verdict; 'none' for Acceptable.",
|
|
90
|
+
allowedLabels: moderationSubcategories,
|
|
91
|
+
relations: [{ field: moderationLabel, rel_type: "depends_on" }],
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
field: moderationRationale,
|
|
95
|
+
entityType: "EXPLANATION",
|
|
96
|
+
jsonPath: "$.moderation_rationale",
|
|
97
|
+
description:
|
|
98
|
+
"One to three sentences naming the deciding element and the guideline rule applied.",
|
|
99
|
+
relations: [{ field: moderationLabel, rel_type: "references" }],
|
|
100
|
+
},
|
|
101
|
+
],
|
|
102
|
+
},
|
|
103
|
+
qualityCriteria: [
|
|
104
|
+
{
|
|
105
|
+
id: "c4d320e7-955f-4877-9b5e-7d99859e3da1",
|
|
106
|
+
qualityElement: "Correct verdict",
|
|
107
|
+
thresholds: {
|
|
108
|
+
pass: "The label follows the guidelines for the visible image content and the description: political or religious advocacy, ritual tools, metaphysical claims, and explicit drug-taking references are Opinionated, also when they are offensive; other moderated content is Offensive; neutral identity, patriotic, standalone symbols, and acceptable edgy content are Acceptable. When political or religious meaning is genuinely ambiguous, Opinionated is chosen.",
|
|
109
|
+
fail: "The label contradicts the guidelines: advocacy, ritual tools, metaphysical claims, or offensive content are marked Acceptable; content that is both offensive and opinionated is marked Offensive; or neutral identity, patriotic, standalone-symbol, or acceptable edgy content is flagged.",
|
|
110
|
+
},
|
|
111
|
+
targetEntity: moderationLabel,
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
id: "c053e226-2106-49c1-8f24-1c1bd8071040",
|
|
115
|
+
qualityElement: "Consistent subcategory",
|
|
116
|
+
thresholds: {
|
|
117
|
+
pass: "The subcategory is 'none' for Acceptable; political, religious, or drug_reference for Opinionated; one of the offensive subcategories for Offensive; and it names the main reason given in the rationale.",
|
|
118
|
+
fail: "The subcategory does not belong to the chosen label, or names a different reason than the rationale.",
|
|
119
|
+
},
|
|
120
|
+
targetEntity: moderationSubcategory,
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
id: "24133e9c-7be8-4533-9e02-3036ce023099",
|
|
124
|
+
qualityElement: "Grounded rationale",
|
|
125
|
+
thresholds: {
|
|
126
|
+
pass: "The rationale names a specific element actually present in the image or description (text, symbol, imagery, or phrase) and the guideline rule that decided the label.",
|
|
127
|
+
fail: "The rationale is generic, cites elements not present in the product, or does not explain the label.",
|
|
128
|
+
},
|
|
129
|
+
targetEntity: moderationRationale,
|
|
130
|
+
},
|
|
131
|
+
],
|
|
132
|
+
instruction,
|
|
133
|
+
ui: {
|
|
134
|
+
code: `<View.Flex vertical gap={'1rem'}>
|
|
135
|
+
<View.Row gap={'1rem'} align={'stretch'}>
|
|
136
|
+
<View.Col span={12}>
|
|
137
|
+
<View.Card title={'Product'}>
|
|
138
|
+
<View.Flex vertical gap={'0.75rem'}>
|
|
139
|
+
<View.Image
|
|
140
|
+
src={actions.get('image_link', '')}
|
|
141
|
+
style={{ maxHeight: '60vh', objectFit: 'contain' }}
|
|
142
|
+
/>
|
|
143
|
+
<View.Text variant={'small'} color={'secondary'}>{actions.get('product_category', '')}</View.Text>
|
|
144
|
+
<View.Text>{actions.get('product_description', '')}</View.Text>
|
|
145
|
+
</View.Flex>
|
|
146
|
+
</View.Card>
|
|
147
|
+
</View.Col>
|
|
148
|
+
<View.Col span={12}>
|
|
149
|
+
<View.Card title={'Moderation'}>
|
|
150
|
+
<View.Flex vertical gap={'1rem'}>
|
|
151
|
+
<Field.RadioGroup
|
|
152
|
+
path={'moderation_label'}
|
|
153
|
+
label={'Label'}
|
|
154
|
+
options={CONST.LABELS}
|
|
155
|
+
validation={[{ required: true, message: 'Choose a label' }]}
|
|
156
|
+
/>
|
|
157
|
+
<Field.Select
|
|
158
|
+
path={'moderation_subcategory'}
|
|
159
|
+
label={'Subcategory'}
|
|
160
|
+
placeholder={'Select a subcategory'}
|
|
161
|
+
options={CONST.SUBCATEGORIES}
|
|
162
|
+
validation={[{ required: true, message: 'Choose a subcategory' }]}
|
|
163
|
+
/>
|
|
164
|
+
<Field.Textarea
|
|
165
|
+
path={'moderation_rationale'}
|
|
166
|
+
label={'Rationale'}
|
|
167
|
+
rows={4}
|
|
168
|
+
validation={[
|
|
169
|
+
{ required: true, message: 'Explain the label' },
|
|
170
|
+
{ max: 1000, message: 'Keep the rationale under 1000 characters' },
|
|
171
|
+
]}
|
|
172
|
+
/>
|
|
173
|
+
</View.Flex>
|
|
174
|
+
</View.Card>
|
|
175
|
+
</View.Col>
|
|
176
|
+
</View.Row>
|
|
177
|
+
</View.Flex>`,
|
|
178
|
+
data: JSON.stringify({
|
|
179
|
+
image_link: "https://example.com/products/flag-tee.jpg",
|
|
180
|
+
product_category: teeCategory,
|
|
181
|
+
product_description:
|
|
182
|
+
'Women\'s American flag tee made in the USA. Our popular "Live Free" logo is printed on the left chest with the American flag on the back.',
|
|
183
|
+
}),
|
|
184
|
+
preprocess: "return {}",
|
|
185
|
+
postprocess: "return {};",
|
|
186
|
+
constants: JSON.stringify({ LABELS: moderationLabels, SUBCATEGORIES: moderationSubcategories }),
|
|
187
|
+
},
|
|
188
|
+
onboarding: {
|
|
189
|
+
enabled: true,
|
|
190
|
+
steps: [
|
|
191
|
+
{
|
|
192
|
+
id: "step-1",
|
|
193
|
+
blocks: [
|
|
194
|
+
{
|
|
195
|
+
id: "labels",
|
|
196
|
+
type: "text",
|
|
197
|
+
title: "Opinionated, Offensive, Acceptable",
|
|
198
|
+
content: [
|
|
199
|
+
{
|
|
200
|
+
type: "markdown",
|
|
201
|
+
value:
|
|
202
|
+
"Opinionated covers political and religious advocacy, ritual tools, metaphysical claims, and explicit drug references. Offensive covers other moderated content. If a product is both, label it Opinionated.",
|
|
203
|
+
},
|
|
204
|
+
],
|
|
205
|
+
},
|
|
206
|
+
],
|
|
207
|
+
},
|
|
208
|
+
{
|
|
209
|
+
id: "step-2",
|
|
210
|
+
blocks: [
|
|
211
|
+
{
|
|
212
|
+
id: "identity",
|
|
213
|
+
type: "text",
|
|
214
|
+
title: "Identity is not advocacy",
|
|
215
|
+
content: [
|
|
216
|
+
{
|
|
217
|
+
type: "markdown",
|
|
218
|
+
value:
|
|
219
|
+
"Standalone flags, plain crosses, an identity flag alone, and patriotic designs are Acceptable. The same symbols paired with a slogan that takes a side ('Victory for Freedonia', 'Faith will save you') are Opinionated.",
|
|
220
|
+
},
|
|
221
|
+
],
|
|
222
|
+
},
|
|
223
|
+
],
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
id: "step-3",
|
|
227
|
+
blocks: [
|
|
228
|
+
{
|
|
229
|
+
id: "quiz",
|
|
230
|
+
type: "quiz",
|
|
231
|
+
title: "Moderation check",
|
|
232
|
+
questions: [
|
|
233
|
+
{
|
|
234
|
+
id: "question-1",
|
|
235
|
+
type: "single-choice",
|
|
236
|
+
question:
|
|
237
|
+
"A tee shows a national flag with the slogan 'Victory for Freedonia'. Which label applies?",
|
|
238
|
+
options: moderationLabels,
|
|
239
|
+
correctAnswer: "opinionated",
|
|
240
|
+
correctAnswerFeedback:
|
|
241
|
+
"Correct. A flag paired with a slogan that takes a side is political advocacy.",
|
|
242
|
+
incorrectAnswerFeedback:
|
|
243
|
+
"A standalone flag is Acceptable, but a flag with a slogan that takes a side is Opinionated.",
|
|
244
|
+
},
|
|
245
|
+
{
|
|
246
|
+
id: "question-2",
|
|
247
|
+
type: "single-choice",
|
|
248
|
+
question:
|
|
249
|
+
"A mug reads 'Nothing like a cold beer after a cold beer'. Which label applies?",
|
|
250
|
+
options: moderationLabels,
|
|
251
|
+
correctAnswer: "acceptable",
|
|
252
|
+
correctAnswerFeedback: "Correct. Alcohol humor is acceptable edgy content.",
|
|
253
|
+
incorrectAnswerFeedback:
|
|
254
|
+
"Alcohol humor that attacks nobody is acceptable edgy content, not Offensive.",
|
|
255
|
+
},
|
|
256
|
+
],
|
|
257
|
+
},
|
|
258
|
+
],
|
|
259
|
+
},
|
|
260
|
+
],
|
|
261
|
+
info: {
|
|
262
|
+
estimatedTime: "5 minutes",
|
|
263
|
+
role: "Product content moderator",
|
|
264
|
+
goal: "Flag opinionated and offensive products",
|
|
265
|
+
successCriteria: "Labels follow the guidelines and the precedence rule",
|
|
266
|
+
},
|
|
267
|
+
},
|
|
268
|
+
checks: {
|
|
269
|
+
annotation_qa: [
|
|
270
|
+
reviewCase(
|
|
271
|
+
"freedom-hair-tie",
|
|
272
|
+
"Red, white and blue 'freedom' hair tie (Acceptable)",
|
|
273
|
+
{
|
|
274
|
+
image_link: "https://example.com/products/hair-tie.png",
|
|
275
|
+
product_category: hairTieCategory,
|
|
276
|
+
product_description:
|
|
277
|
+
"Meet our Freedom Trail tangle-free tie! Let your freedom flag fly with this red, white, and blue hair tie.",
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
moderation_label: "acceptable",
|
|
281
|
+
moderation_subcategory: "none",
|
|
282
|
+
moderation_rationale:
|
|
283
|
+
"Red, white, and blue colors and the word 'freedom' are patriotic identity with no advocacy.",
|
|
284
|
+
},
|
|
285
|
+
),
|
|
286
|
+
reviewCase(
|
|
287
|
+
"logo-graphic-tee",
|
|
288
|
+
"Streetwear tee with a brand logo (Acceptable)",
|
|
289
|
+
{
|
|
290
|
+
image_link: "https://example.com/products/logo-tee.webp",
|
|
291
|
+
product_category: teeCategory,
|
|
292
|
+
product_description:
|
|
293
|
+
"An urban streetwear t-shirt for boys in white, made from organic cotton, with a printed brand logo and a relaxed fit.",
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
moderation_label: "acceptable",
|
|
297
|
+
moderation_subcategory: "none",
|
|
298
|
+
moderation_rationale:
|
|
299
|
+
"A brand logo on a plain tee; nothing in the image or the description takes a side or offends.",
|
|
300
|
+
},
|
|
301
|
+
),
|
|
302
|
+
reviewCase(
|
|
303
|
+
"flag-live-free-tee",
|
|
304
|
+
"American flag 'Live Free' tee (Acceptable)",
|
|
305
|
+
{
|
|
306
|
+
image_link: "https://example.com/products/flag-tee.jpg",
|
|
307
|
+
product_category: teeCategory,
|
|
308
|
+
product_description:
|
|
309
|
+
'Women\'s American flag tee made in the USA. Our popular "Live Free" logo is printed on the left chest with the American flag on the back.',
|
|
310
|
+
},
|
|
311
|
+
{
|
|
312
|
+
moderation_label: "acceptable",
|
|
313
|
+
moderation_subcategory: "none",
|
|
314
|
+
moderation_rationale:
|
|
315
|
+
"A standalone American flag and 'Live Free' are patriotic identity statements.",
|
|
316
|
+
},
|
|
317
|
+
),
|
|
318
|
+
reviewCase(
|
|
319
|
+
"flag-tee-victory-slogan",
|
|
320
|
+
"Flag tee with 'Victory for Freedonia' (Opinionated, political)",
|
|
321
|
+
{
|
|
322
|
+
image_link: "https://example.com/products/flag-tee.jpg",
|
|
323
|
+
product_category: teeCategory,
|
|
324
|
+
product_description:
|
|
325
|
+
'Women\'s flag tee with "Victory for Freedonia" printed on the chest. Show where you stand!',
|
|
326
|
+
},
|
|
327
|
+
{
|
|
328
|
+
moderation_label: "opinionated",
|
|
329
|
+
moderation_subcategory: "political",
|
|
330
|
+
moderation_rationale:
|
|
331
|
+
"The flag is paired with 'Victory for Freedonia', a slogan that takes a side; a flag with a support slogan is political advocacy.",
|
|
332
|
+
},
|
|
333
|
+
),
|
|
334
|
+
reviewCase(
|
|
335
|
+
"tee-faith-slogan",
|
|
336
|
+
"Tee with 'Faith will save you' (Opinionated, religious)",
|
|
337
|
+
{
|
|
338
|
+
image_link: "https://example.com/products/logo-tee.webp",
|
|
339
|
+
product_category: teeCategory,
|
|
340
|
+
product_description:
|
|
341
|
+
"Graphic t-shirt with 'Faith will save you' across the chest. Put your faith on display.",
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
moderation_label: "opinionated",
|
|
345
|
+
moderation_subcategory: "religious",
|
|
346
|
+
moderation_rationale:
|
|
347
|
+
"'Faith will save you' promotes religious belief; text that professes faith is Opinionated.",
|
|
348
|
+
},
|
|
349
|
+
),
|
|
350
|
+
reviewCase(
|
|
351
|
+
"hair-tie-insult",
|
|
352
|
+
"Hair tie with an insulting slogan (Offensive, derogatory)",
|
|
353
|
+
{
|
|
354
|
+
image_link: "https://example.com/products/hair-tie.png",
|
|
355
|
+
product_category: hairTieCategory,
|
|
356
|
+
product_description:
|
|
357
|
+
"Our 'Only idiots wear their hair down' tangle-free hair tie. Red, white, and blue.",
|
|
358
|
+
},
|
|
359
|
+
{
|
|
360
|
+
moderation_label: "offensive",
|
|
361
|
+
moderation_subcategory: "derogatory",
|
|
362
|
+
moderation_rationale:
|
|
363
|
+
"The slogan calls people 'idiots', a derogatory word, with no political or religious context.",
|
|
364
|
+
},
|
|
365
|
+
),
|
|
366
|
+
],
|
|
367
|
+
},
|
|
368
|
+
});
|