@intellectif/lk-core 0.20.0 → 0.22.0

package/vectors/scoring.json

@@ -1,7 +1,7 @@
 {
   "$comment": "Grade-stability corpus for @intellectif/lk-core. Generated by scripts/generate-vectors.mjs from the built package; never edit expectations by hand. Replay with vectors/replay.mjs. See vectors/README.md.",
   "corpusVersion": 2,
-  "frozenFrom": "@intellectif/lk-core@0.17.0",
+  "frozenFrom": "@intellectif/lk-core@0.20.0",
   "vectors": [
     {
       "args": [
@@ -32855,6 +32855,2132 @@
       "const": "SPEECH_ASSESSMENT_MAX_WORDS",
       "expect": 1000,
       "id": "const/SPEECH_ASSESSMENT_MAX_WORDS"
+    },
+    {
+      "const": "DEFAULT_ITEM_SCORING_POLICY",
+      "expect": {
+        "counts": "first",
+        "hintPenalty": 0,
+        "retries": 0,
+        "retryPenalty": 0
+      },
+      "id": "const/DEFAULT_ITEM_SCORING_POLICY"
+    },
+    {
+      "const": "ITEM_SCORING_MAX_RETRIES",
+      "expect": 10,
+      "id": "const/ITEM_SCORING_MAX_RETRIES"
+    },
+    {
+      "args": [null],
+      "expect": {
+        "counts": "first",
+        "hintPenalty": 0,
+        "retries": 0,
+        "retryPenalty": 0
+      },
+      "fn": "resolveItemScoringPolicy",
+      "id": "resolveItemScoringPolicy/none-is-scoring-before-policies",
+      "note": "No policy is one try, nothing costing anything, the first try counting: exactly how every question was scored before policies existed."
+    },
+    {
+      "args": [
+        {
+          "counts": null,
+          "hintPenalty": 0.1,
+          "retries": null,
+          "retryPenalty": null
+        }
+      ],
+      "expect": {
+        "counts": "first",
+        "hintPenalty": 0.1,
+        "retries": 0,
+        "retryPenalty": 0
+      },
+      "fn": "resolveItemScoringPolicy",
+      "id": "resolveItemScoringPolicy/settings-left-null-are-defaults"
+    },
+    {
+      "args": [
+        {
+          "hintPenalty": "0.1"
+        }
+      ],
+      "expect": {
+        "$throws": "RangeError"
+      },
+      "fn": "resolveItemScoringPolicy",
+      "id": "resolveItemScoringPolicy/refuses-a-cost-it-cannot-read",
+      "note": "A grade-moving setting has no safe reading: a cost misread as 0 raises grades, as 1 lowers them. It throws, where a delivery policy would restrict."
+    },
+    {
+      "args": [
+        {
+          "retries": 2,
+          "retryPenalty": 0.25
+        }
+      ],
+      "expect": {
+        "$throws": "RangeError"
+      },
+      "fn": "resolveItemScoringPolicy",
+      "id": "resolveItemScoringPolicy/refuses-a-retry-cost-never-charged",
+      "note": "The first try counts by default, so a retry cost would never be charged: a school that set one believes retries cost something."
+    },
+    {
+      "args": [
+        {
+          "counts": "last",
+          "hintPenalty": 1,
+          "retries": 10,
+          "retryPenalty": 1
+        }
+      ],
+      "expect": {
+        "counts": "last",
+        "hintPenalty": 1,
+        "retries": 10,
+        "retryPenalty": 1
+      },
+      "fn": "resolveItemScoringPolicy",
+      "id": "resolveItemScoringPolicy/accepts-every-setting-at-its-limit",
+      "note": "A cost of the whole mark and ten retries are policies a school may set; refusing them would fail every paper that uses them."
+    },
+    {
+      "args": [3],
+      "expect": {
+        "$throws": "RangeError"
+      },
+      "fn": "resolveItemScoringPolicy",
+      "id": "resolveItemScoringPolicy/refuses-a-policy-that-is-not-an-object",
+      "note": "A number where a policy belongs is refused, not read as the default policy: it names no setting, and a stored column that holds one is broken."
+    },
+    {
+      "args": [null],
+      "expect": {
+        "data": {},
+        "success": true
+      },
+      "fn": "validateItemScoringPolicy",
+      "id": "validateItemScoringPolicy/no-policy-is-valid"
+    },
+    {
+      "args": [
+        {
+          "retries": 11
+        }
+      ],
+      "expect": {
+        "$throws": "RangeError"
+      },
+      "fn": "resolveItemScoringPolicy",
+      "id": "resolveItemScoringPolicy/refuses-retries-past-the-limit"
+    },
+    {
+      "args": [
+        {
+          "counts": "average",
+          "hintPenalties": 0.1,
+          "hintPenalty": 1.5,
+          "retries": 1.5
+        }
+      ],
+      "expect": {
+        "issues": [
+          {
+            "message": "\"hintPenalties\" is not an item scoring setting. The settings are hintPenalty, retries, retryPenalty and counts.",
+            "path": "hintPenalties"
+          },
+          {
+            "message": "\"hintPenalty\" must be a number from 0 to 1: the fraction of a question's marks it costs.",
+            "path": "hintPenalty"
+          },
+          {
+            "message": "\"retries\" must be a whole number from 0 to 10.",
+            "path": "retries"
+          },
+          {
+            "message": "\"counts\" must be \"first\", \"best\" or \"last\".",
+            "path": "counts"
+          }
+        ],
+        "success": false
+      },
+      "fn": "validateItemScoringPolicy",
+      "id": "validateItemScoringPolicy/names-each-setting-it-refuses",
+      "ignore": ["message"]
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 3,
+            "maxScore": 1,
+            "score": 0.5
+          },
+          {
+            "maxScore": 1,
+            "score": 1
+          }
+        ],
+        null
+      ],
+      "expect": {
+        "counted": 0,
+        "maxScore": 1,
+        "score": 0.5,
+        "tries": [
+          {
+            "hintsRevealed": 3,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0.5,
+            "scored": 0.5
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/no-policy-is-the-first-answer-as-it-scored",
+      "note": "Hints reported and later tries made, and no policy: the question scores what its first answer scored, to the bit."
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 2,
+            "maxScore": 1,
+            "score": 1
+          }
+        ],
+        {
+          "hintPenalty": 0.25
+        }
+      ],
+      "expect": {
+        "counted": 0,
+        "maxScore": 1,
+        "score": 0.5,
+        "tries": [
+          {
+            "hintsRevealed": 2,
+            "maxScore": 1,
+            "penalty": 0.5,
+            "score": 1,
+            "scored": 0.5
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/hint-cost-is-a-fraction-of-the-marks"
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 1,
+            "maxScore": 1,
+            "score": 0.5
+          }
+        ],
+        {
+          "hintPenalty": 0.25
+        }
+      ],
+      "expect": {
+        "counted": 0,
+        "maxScore": 1,
+        "score": 0.25,
+        "tries": [
+          {
+            "hintsRevealed": 1,
+            "maxScore": 1,
+            "penalty": 0.25,
+            "score": 0.5,
+            "scored": 0.25
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/hint-cost-on-a-partial-answer-subtracts",
+      "note": "Subtracted from what the answer scored, not multiplied into it: half marks less a quarter is a quarter, not three eighths."
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 3,
+            "maxScore": 1,
+            "score": 0.5
+          }
+        ],
+        {
+          "hintPenalty": 0.25
+        }
+      ],
+      "expect": {
+        "counted": 0,
+        "maxScore": 1,
+        "score": 0,
+        "tries": [
+          {
+            "hintsRevealed": 3,
+            "maxScore": 1,
+            "penalty": 0.75,
+            "score": 0.5,
+            "scored": 0
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/cost-never-takes-a-try-below-zero"
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 2,
+            "maxScore": 1,
+            "score": 0.7
+          }
+        ],
+        {
+          "hintPenalty": 0.1
+        }
+      ],
+      "expect": {
+        "counted": 0,
+        "maxScore": 1,
+        "score": 0.5,
+        "tries": [
+          {
+            "hintsRevealed": 2,
+            "maxScore": 1,
+            "penalty": 0.2,
+            "score": 0.7,
+            "scored": 0.5
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/no-floating-point-residue-under-a-pass-line",
+      "note": "0.7 - 0.2 is 0.49999999999999994 in floating point, which fails a 0.5 pass line the learner met. A costed score is snapped to twelve decimal places."
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 1.5,
+            "maxScore": 1,
+            "score": 1
+          },
+          {
+            "hintsRevealed": -2,
+            "maxScore": 1,
+            "score": 1
+          }
+        ],
+        {
+          "counts": "last",
+          "hintPenalty": 0.5,
+          "retries": 1
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "maxScore": 1,
+        "score": 1,
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 1,
+            "scored": 1
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 1,
+            "scored": 1
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/a-hint-count-it-cannot-read-is-none"
+    },
+    {
+      "args": [
+        [
+          {
+            "maxScore": 1,
+            "score": 0.5
+          },
+          {
+            "maxScore": 1,
+            "score": 1
+          }
+        ],
+        {
+          "retries": 1
+        }
+      ],
+      "expect": {
+        "counted": 0,
+        "maxScore": 1,
+        "score": 0.5,
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0.5,
+            "scored": 0.5
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 1,
+            "scored": 1
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/first-counts-by-default",
+      "note": "The default: tries after the first are for learning. Switching tries on moves no grade until a school chooses best or last."
+    },
+    {
+      "args": [
+        [
+          {
+            "maxScore": 1,
+            "score": 0.5
+          },
+          {
+            "maxScore": 1,
+            "score": 0.5
+          },
+          {
+            "maxScore": 1,
+            "score": 1
+          },
+          {
+            "hintsRevealed": 1,
+            "maxScore": 1,
+            "score": 1
+          }
+        ],
+        {
+          "counts": "best",
+          "hintPenalty": 0.1,
+          "retries": 3,
+          "retryPenalty": 0.25
+        }
+      ],
+      "expect": {
+        "counted": 0,
+        "maxScore": 1,
+        "score": 0.5,
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0.5,
+            "scored": 0.5
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0.25,
+            "score": 0.5,
+            "scored": 0.25
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0.5,
+            "score": 1,
+            "scored": 0.5
+          },
+          {
+            "hintsRevealed": 1,
+            "maxScore": 1,
+            "penalty": 0.85,
+            "score": 1,
+            "scored": 0.15
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/best-after-costs-earliest-of-equals",
+      "note": "The third try scores 1 less two retries at a quarter: 0.5, equal to the first. The earliest of equal tries counts."
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 10,
+            "score": 5
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "score": 0.6
+          }
+        ],
+        {
+          "counts": "best",
+          "retries": 1
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "maxScore": 1,
+        "score": 0.6,
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 10,
+            "penalty": 0,
+            "score": 5,
+            "scored": 5
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0.6,
+            "scored": 0.6
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/best-compares-fractions-of-the-marks",
+      "note": "Best is the highest share of the marks: 0.6 out of 1 (60%) beats 5 out of 10 (50%), though 5 is more points."
+    },
+    {
+      "args": [
+        [
+          {
+            "maxScore": 1,
+            "score": 0
+          },
+          {
+            "maxScore": 1,
+            "score": 1
+          }
+        ],
+        {
+          "counts": "best",
+          "hintPenalty": 0.1,
+          "retries": 2,
+          "retryPenalty": 0.25
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "maxScore": 1,
+        "score": 0.75,
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0,
+            "scored": 0
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0.25,
+            "score": 1,
+            "scored": 0.75
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/best-a-later-right-answer"
+    },
+    {
+      "args": [
+        [
+          {
+            "maxScore": 1,
+            "score": 1
+          },
+          {
+            "maxScore": 1,
+            "score": 0
+          }
+        ],
+        {
+          "counts": "last",
+          "retries": 1
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "maxScore": 1,
+        "score": 0,
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 1,
+            "scored": 1
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0,
+            "scored": 0
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/last-counts-even-when-lower"
+    },
+    {
+      "args": [
+        [
+          {
+            "maxScore": 1,
+            "score": 0
+          },
+          {
+            "maxScore": 1,
+            "score": 0.5
+          },
+          {
+            "maxScore": 1,
+            "score": 1
+          }
+        ],
+        {
+          "counts": "last",
+          "retries": 1
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "maxScore": 1,
+        "score": 0.5,
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0,
+            "scored": 0
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0.5,
+            "scored": 0.5
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/no-try-past-the-allowance-is-believed",
+      "note": "Two tries allowed and three recorded: the third cannot have come from a learner under this policy, and is not read."
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 1,
+            "maxScore": 1,
+            "score": 0
+          },
+          {
+            "hintsRevealed": 2,
+            "maxScore": 1,
+            "score": 1
+          }
+        ],
+        {
+          "counts": "last",
+          "hintPenalty": 0.1,
+          "retries": 1,
+          "retryPenalty": 0.2
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "maxScore": 1,
+        "score": 0.6,
+        "tries": [
+          {
+            "hintsRevealed": 1,
+            "maxScore": 1,
+            "penalty": 0.1,
+            "score": 0,
+            "scored": 0
+          },
+          {
+            "hintsRevealed": 2,
+            "maxScore": 1,
+            "penalty": 0.4,
+            "score": 1,
+            "scored": 0.6
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/later-hint-count-includes-earlier-hints"
+    },
+    {
+      "args": [
+        [
+          {
+            "hintsRevealed": 1,
+            "maxScore": 10,
+            "score": 8
+          }
+        ],
+        {
+          "hintPenalty": 0.1
+        }
+      ],
+      "expect": {
+        "counted": 0,
+        "maxScore": 10,
+        "score": 7,
+        "tries": [
+          {
+            "hintsRevealed": 1,
+            "maxScore": 10,
+            "penalty": 0.1,
+            "score": 8,
+            "scored": 7
+          }
+        ]
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/cost-out-of-any-maximum"
+    },
+    {
+      "args": [[], null],
+      "expect": {
+        "$throws": "RangeError"
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/throws-on-no-try"
+    },
+    {
+      "args": [
+        [
+          {
+            "maxScore": 1,
+            "score": 85
+          }
+        ],
+        {
+          "hintPenalty": 0.1
+        }
+      ],
+      "expect": {
+        "$throws": "RangeError"
+      },
+      "fn": "scoreTries",
+      "id": "scoreTries/throws-on-a-try-that-is-not-a-grade"
+    },
+    {
+      "args": [
+        {
+          "id": "mc-capitals",
+          "mode": "multi",
+          "options": [
+            {
+              "id": "a",
+              "isCorrect": true,
+              "text": "Lisbon"
+            },
+            {
+              "id": "b",
+              "isCorrect": false,
+              "text": "Porto"
+            },
+            {
+              "id": "c",
+              "isCorrect": true,
+              "text": "Madrid"
+            },
+            {
+              "id": "d",
+              "isCorrect": false,
+              "text": "Seville"
+            }
+          ],
+          "question": "Which of these are capital cities?",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "all-or-nothing",
+          "title": "Capitals",
+          "type": "multiple-choice"
+        },
+        {
+          "hintsRevealed": 2,
+          "selectedOptionIds": ["a", "c"],
+          "type": "multiple-choice"
+        },
+        {
+          "scoring": {
+            "hintPenalty": 0.2
+          }
+        }
+      ],
+      "expect": {
+        "details": [
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "a",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "b",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "c",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "d",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          }
+        ],
+        "feedback": null,
+        "maxScore": 1,
+        "passed": false,
+        "score": 0.6,
+        "status": "scored"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/mc/hint-cost-reads-the-pass-line-after-it",
+      "note": "A right answer after two hints at a fifth: 0.6, below the default 0.7 pass line. The details and the feedback stay the answer's."
+    },
+    {
+      "args": [
+        {
+          "id": "mc-capitals",
+          "mode": "multi",
+          "options": [
+            {
+              "id": "a",
+              "isCorrect": true,
+              "text": "Lisbon"
+            },
+            {
+              "id": "b",
+              "isCorrect": false,
+              "text": "Porto"
+            },
+            {
+              "id": "c",
+              "isCorrect": true,
+              "text": "Madrid"
+            },
+            {
+              "id": "d",
+              "isCorrect": false,
+              "text": "Seville"
+            }
+          ],
+          "question": "Which of these are capital cities?",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "all-or-nothing",
+          "title": "Capitals",
+          "type": "multiple-choice"
+        },
+        {
+          "hintsRevealed": 2,
+          "selectedOptionIds": ["a", "c"],
+          "type": "multiple-choice"
+        },
+        {
+          "scoring": {}
+        }
+      ],
+      "expect": {
+        "details": [
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "a",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "b",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "c",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "d",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          }
+        ],
+        "feedback": null,
+        "maxScore": 1,
+        "passed": true,
+        "score": 1,
+        "status": "scored"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/mc/hint-cost-policy-empty-changes-nothing"
+    },
+    {
+      "args": [
+        {
+          "blanks": [
+            {
+              "acceptedAnswers": ["a"],
+              "id": "b1"
+            },
+            {
+              "acceptedAnswers": ["the"],
+              "id": "b2"
+            },
+            {
+              "acceptedAnswers": ["the"],
+              "id": "b3"
+            }
+          ],
+          "id": "fib-articles",
+          "passage": "We bought {{b1}} lamp for {{b2}} study and {{b3}} hall.",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "partial",
+          "title": "Articles",
+          "type": "fill-in-the-blanks"
+        },
+        {
+          "answers": {
+            "b1": "a",
+            "b2": "the",
+            "b3": "wrong"
+          },
+          "hintsRevealed": 1,
+          "type": "fill-in-the-blanks"
+        },
+        {
+          "scoring": {
+            "hintPenalty": 0.1
+          }
+        }
+      ],
+      "expect": {
+        "details": [
+          {
+            "correct": true,
+            "correctResponse": ["a"],
+            "itemId": "b1",
+            "learnerResponse": ["a"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["the"],
+            "itemId": "b2",
+            "learnerResponse": ["the"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": false,
+            "correctResponse": ["the"],
+            "itemId": "b3",
+            "learnerResponse": ["wrong"],
+            "outcome": "incorrect",
+            "weight": 1
+          }
+        ],
+        "feedback": null,
+        "maxScore": 1,
+        "passed": false,
+        "score": 0.566666666667,
+        "status": "scored"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/fib/hint-cost-on-partial-credit"
+    },
+    {
+      "args": [
+        {
+          "banks": [
+            {
+              "choices": [
+                {
+                  "id": "of",
+                  "text": "of"
+                },
+                {
+                  "id": "from",
+                  "text": "from"
+                },
+                {
+                  "id": "to",
+                  "text": "to"
+                },
+                {
+                  "id": "on",
+                  "text": "on"
+                }
+              ],
+              "id": "prep"
+            }
+          ],
+          "gaps": [
+            {
+              "bankId": "prep",
+              "correctChoiceId": "from",
+              "id": "a"
+            },
+            {
+              "bankId": "prep",
+              "correctChoiceId": "from",
+              "id": "b"
+            }
+          ],
+          "id": "gs-prepositions",
+          "passage": "Where are you {{a}}? I'm {{b}} Spain.",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "partial",
+          "title": "Prepositions",
+          "type": "gap-select"
+        },
+        {
+          "hintsRevealed": 1,
+          "selections": {
+            "a": "from",
+            "b": "from"
+          },
+          "type": "gap-select"
+        },
+        {
+          "scoring": {
+            "hintPenalty": 0.3
+          }
+        }
+      ],
+      "expect": {
+        "details": [
+          {
+            "correct": true,
+            "correctResponse": ["from"],
+            "itemId": "a",
+            "learnerResponse": ["from"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["from"],
+            "itemId": "b",
+            "learnerResponse": ["from"],
+            "outcome": "correct",
+            "weight": 1
+          }
+        ],
+        "feedback": null,
+        "maxScore": 1,
+        "passed": true,
+        "score": 0.7,
+        "status": "scored"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/gs/hint-cost"
+    },
+    {
+      "args": [
+        {
+          "id": "dc-lisbon",
+          "media": {
+            "alt": "Recording",
+            "type": "audio",
+            "url": "https://cdn.example/lisbon.mp3"
+          },
+          "schemaVersion": "1.0",
+          "title": "Listen and type the sentence",
+          "transcript": "It isn't raining in Lisbon today.",
+          "type": "dictation"
+        },
+        {
+          "hintsRevealed": 3,
+          "text": "It isn't raining in Lisbon today.",
+          "type": "dictation"
+        },
+        {
+          "scoring": {
+            "hintPenalty": 0.05
+          }
+        }
+      ],
+      "expect": {
+        "details": [
+          {
+            "correct": true,
+            "correctResponse": "it",
+            "itemId": "w1",
+            "learnerResponse": "it",
+            "outcome": "correct",
+            "score": 1,
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": "isn't",
+            "itemId": "w2",
+            "learnerResponse": "isn't",
+            "outcome": "correct",
+            "score": 1,
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": "raining",
+            "itemId": "w3",
+            "learnerResponse": "raining",
+            "outcome": "correct",
+            "score": 1,
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": "in",
+            "itemId": "w4",
+            "learnerResponse": "in",
+            "outcome": "correct",
+            "score": 1,
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": "lisbon",
+            "itemId": "w5",
+            "learnerResponse": "lisbon",
+            "outcome": "correct",
+            "score": 1,
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": "today",
+            "itemId": "w6",
+            "learnerResponse": "today",
+            "outcome": "correct",
+            "score": 1,
+            "weight": 1
+          }
+        ],
+        "feedback": null,
+        "maxScore": 1,
+        "passed": true,
+        "score": 0.85,
+        "status": "scored"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/dc/hint-cost-on-the-word-hints",
+      "note": "A dictation has always reported the hint words it showed; a policy now charges for them."
+    },
+    {
+      "args": [
+        {
+          "id": "mc-capitals",
+          "mode": "multi",
+          "options": [
+            {
+              "id": "a",
+              "isCorrect": true,
+              "text": "Lisbon"
+            },
+            {
+              "id": "b",
+              "isCorrect": false,
+              "text": "Porto"
+            },
+            {
+              "id": "c",
+              "isCorrect": true,
+              "text": "Madrid"
+            },
+            {
+              "id": "d",
+              "isCorrect": false,
+              "text": "Seville"
+            }
+          ],
+          "passThreshold": 0.5,
+          "question": "Which of these are capital cities?",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "all-or-nothing",
+          "title": "Capitals",
+          "type": "multiple-choice"
+        },
+        {
+          "hintsRevealed": 2,
+          "selectedOptionIds": ["a", "c"],
+          "type": "multiple-choice"
+        },
+        {
+          "rounding": {
+            "dp": 2,
+            "mode": "half-up"
+          },
+          "scoring": {
+            "hintPenalty": 0.25
+          }
+        }
+      ],
+      "expect": {
+        "details": [
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "a",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "b",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "c",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "d",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          }
+        ],
+        "feedback": null,
+        "maxScore": 1,
+        "passed": true,
+        "score": 0.5,
+        "status": "scored"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/mc/hint-cost-with-rounding-meets-the-line"
+    },
+    {
+      "args": [
+        {
+          "id": "mc-capitals",
+          "mode": "multi",
+          "options": [
+            {
+              "id": "a",
+              "isCorrect": true,
+              "text": "Lisbon"
+            },
+            {
+              "id": "b",
+              "isCorrect": false,
+              "text": "Porto"
+            },
+            {
+              "id": "c",
+              "isCorrect": true,
+              "text": "Madrid"
+            },
+            {
+              "id": "d",
+              "isCorrect": false,
+              "text": "Seville"
+            }
+          ],
+          "passThreshold": 0.7,
+          "question": "Which of these are capital cities?",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "all-or-nothing",
+          "title": "Capitals",
+          "type": "multiple-choice"
+        },
+        {
+          "hintsRevealed": 1,
+          "selectedOptionIds": ["a", "c"],
+          "type": "multiple-choice"
+        },
+        {
+          "rounding": {
+            "dp": 2,
+            "mode": "half-up"
+          },
+          "scoring": {
+            "hintPenalty": 0.305
+          }
+        }
+      ],
+      "expect": {
+        "details": [
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "a",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "b",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "c",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "d",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          }
+        ],
+        "feedback": null,
+        "maxScore": 1,
+        "passed": true,
+        "score": 0.695,
+        "status": "scored"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/mc/hint-cost-rounded-pass-line",
+      "note": "A right answer less one hint at 0.305 is 0.695: a fail against 0.7 raw, a pass once both sides are rounded half-up to two places. The pass line after a cost is read with the rounding the call was given."
+    },
+    {
+      "args": [
+        {
+          "id": "mc-capitals",
+          "mode": "multi",
+          "options": [
+            {
+              "id": "a",
+              "isCorrect": true,
+              "text": "Lisbon"
+            },
+            {
+              "id": "b",
+              "isCorrect": false,
+              "text": "Porto"
+            },
+            {
+              "id": "c",
+              "isCorrect": true,
+              "text": "Madrid"
+            },
+            {
+              "id": "d",
+              "isCorrect": false,
+              "text": "Seville"
+            }
+          ],
+          "passThreshold": 0.7,
+          "question": "Which of these are capital cities?",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "all-or-nothing",
+          "title": "Capitals",
+          "type": "multiple-choice"
+        },
+        {
+          "hintsRevealed": 1,
+          "selectedOptionIds": ["a", "c"],
+          "type": "multiple-choice"
+        },
+        {
+          "scoring": {
+            "hintPenalty": 0.305
+          }
+        }
+      ],
+      "expect": {
+        "details": [
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "a",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "b",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["selected"],
+            "itemId": "c",
+            "learnerResponse": ["selected"],
+            "outcome": "correct",
+            "weight": 1
+          },
+          {
+            "correct": true,
+            "correctResponse": ["not-selected"],
+            "itemId": "d",
+            "learnerResponse": ["not-selected"],
+            "outcome": "correct-omission",
+            "weight": 1
+          }
+        ],
+        "feedback": null,
+        "maxScore": 1,
+        "passed": false,
+        "score": 0.695,
+        "status": "scored"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/mc/hint-cost-raw-pass-line"
+    },
+    {
+      "args": [
+        {
+          "id": "mc-capitals",
+          "mode": "multi",
+          "options": [
+            {
+              "id": "a",
+              "isCorrect": true,
+              "text": "Lisbon"
+            },
+            {
+              "id": "b",
+              "isCorrect": false,
+              "text": "Porto"
+            },
+            {
+              "id": "c",
+              "isCorrect": true,
+              "text": "Madrid"
+            },
+            {
+              "id": "d",
+              "isCorrect": false,
+              "text": "Seville"
+            }
+          ],
+          "question": "Which of these are capital cities?",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "all-or-nothing",
+          "title": "Capitals",
+          "type": "multiple-choice"
+        },
+        [
+          {
+            "selectedOptionIds": ["a"],
+            "type": "multiple-choice"
+          },
+          {
+            "hintsRevealed": 1,
+            "selectedOptionIds": ["a", "c"],
+            "type": "multiple-choice"
+          }
+        ],
+        {
+          "scoring": {
+            "counts": "best",
+            "hintPenalty": 0.1,
+            "retries": 2,
+            "retryPenalty": 0.25
+          }
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "outcome": {
+          "details": [
+            {
+              "correct": true,
+              "correctResponse": ["selected"],
+              "itemId": "a",
+              "learnerResponse": ["selected"],
+              "outcome": "correct",
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": ["not-selected"],
+              "itemId": "b",
+              "learnerResponse": ["not-selected"],
+              "outcome": "correct-omission",
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": ["selected"],
+              "itemId": "c",
+              "learnerResponse": ["selected"],
+              "outcome": "correct",
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": ["not-selected"],
+              "itemId": "d",
+              "learnerResponse": ["not-selected"],
+              "outcome": "correct-omission",
+              "weight": 1
+            }
+          ],
+          "feedback": null,
+          "maxScore": 1,
+          "passed": false,
+          "score": 0.65,
+          "status": "scored"
+        },
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0,
+            "scored": 0
+          },
+          {
+            "hintsRevealed": 1,
+            "maxScore": 1,
+            "penalty": 0.35,
+            "score": 1,
+            "scored": 0.65
+          }
+        ]
+      },
+      "fn": "evaluateTries",
+      "id": "evaluateTries/mc/best-a-later-right-answer"
+    },
+    {
+      "args": [
+        {
+          "blanks": [
+            {
+              "acceptedAnswers": ["a"],
+              "id": "b1"
+            },
+            {
+              "acceptedAnswers": ["the"],
+              "id": "b2"
+            },
+            {
+              "acceptedAnswers": ["the"],
+              "id": "b3"
+            }
+          ],
+          "id": "fib-articles",
+          "passage": "We bought {{b1}} lamp for {{b2}} study and {{b3}} hall.",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "partial",
+          "title": "Articles",
+          "type": "fill-in-the-blanks"
+        },
+        [
+          {
+            "answers": {
+              "b1": "a",
+              "b2": "x",
+              "b3": "x"
+            },
+            "type": "fill-in-the-blanks"
+          },
+          {
+            "answers": {
+              "b1": "a",
+              "b2": "the",
+              "b3": "x"
+            },
+            "hintsRevealed": 1,
+            "type": "fill-in-the-blanks"
+          }
+        ],
+        {
+          "scoring": {
+            "counts": "last",
+            "hintPenalty": 0.1,
+            "retries": 1
+          }
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "outcome": {
+          "details": [
+            {
+              "correct": true,
+              "correctResponse": ["a"],
+              "itemId": "b1",
+              "learnerResponse": ["a"],
+              "outcome": "correct",
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": ["the"],
+              "itemId": "b2",
+              "learnerResponse": ["the"],
+              "outcome": "correct",
+              "weight": 1
+            },
+            {
+              "correct": false,
+              "correctResponse": ["the"],
+              "itemId": "b3",
+              "learnerResponse": ["x"],
+              "outcome": "incorrect",
+              "weight": 1
+            }
+          ],
+          "feedback": null,
+          "maxScore": 1,
+          "passed": false,
+          "score": 0.566666666667,
+          "status": "scored"
+        },
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0.3333333333333333,
+            "scored": 0.3333333333333333
+          },
+          {
+            "hintsRevealed": 1,
+            "maxScore": 1,
+            "penalty": 0.1,
+            "score": 0.6666666666666666,
+            "scored": 0.566666666667
+          }
+        ]
+      },
+      "fn": "evaluateTries",
+      "id": "evaluateTries/fib/last-on-partial-credit"
+    },
+    {
+      "args": [
+        {
+          "banks": [
+            {
+              "choices": [
+                {
+                  "id": "of",
+                  "text": "of"
+                },
+                {
+                  "id": "from",
+                  "text": "from"
+                },
+                {
+                  "id": "to",
+                  "text": "to"
+                },
+                {
+                  "id": "on",
+                  "text": "on"
+                }
+              ],
+              "id": "prep"
+            }
+          ],
+          "gaps": [
+            {
+              "bankId": "prep",
+              "correctChoiceId": "from",
+              "id": "a"
+            },
+            {
+              "bankId": "prep",
+              "correctChoiceId": "from",
+              "id": "b"
+            }
+          ],
+          "id": "gs-prepositions",
+          "passage": "Where are you {{a}}? I'm {{b}} Spain.",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "partial",
+          "title": "Prepositions",
+          "type": "gap-select"
+        },
+        [
+          {
+            "selections": {
+              "a": "of"
+            },
+            "type": "gap-select"
+          },
+          {
+            "selections": {
+              "a": "from",
+              "b": "from"
+            },
+            "type": "gap-select"
+          }
+        ],
+        {
+          "scoring": {
+            "retries": 1
+          }
+        }
+      ],
+      "expect": {
+        "counted": 0,
+        "outcome": {
+          "details": [
+            {
+              "correct": false,
+              "correctResponse": ["from"],
+              "itemId": "a",
+              "learnerResponse": ["of"],
+              "outcome": "incorrect",
+              "weight": 1
+            },
+            {
+              "correct": false,
+              "correctResponse": ["from"],
+              "itemId": "b",
+              "learnerResponse": [""],
+              "outcome": "incorrect-omission",
+              "weight": 1
+            }
+          ],
+          "feedback": null,
+          "maxScore": 1,
+          "passed": false,
+          "score": 0,
+          "status": "scored"
+        },
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0,
+            "scored": 0
+          },
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 1,
+            "scored": 1
+          }
+        ]
+      },
+      "fn": "evaluateTries",
+      "id": "evaluateTries/gs/first-counts-by-default"
+    },
+    {
+      "args": [
+        {
+          "id": "dc-lisbon",
+          "media": {
+            "alt": "Recording",
+            "type": "audio",
+            "url": "https://cdn.example/lisbon.mp3"
+          },
+          "schemaVersion": "1.0",
+          "title": "Listen and type the sentence",
+          "transcript": "It isn't raining in Lisbon today.",
+          "type": "dictation"
+        },
+        [
+          {
+            "text": "It is raining in Lisbon.",
+            "type": "dictation"
+          },
+          {
+            "hintsRevealed": 2,
+            "text": "It isn't raining in Lisbon today.",
+            "type": "dictation"
+          }
+        ],
+        {
+          "scoring": {
+            "counts": "best",
+            "hintPenalty": 0.05,
+            "retries": 1,
+            "retryPenalty": 0.1
+          }
+        }
+      ],
+      "expect": {
+        "counted": 1,
+        "outcome": {
+          "details": [
+            {
+              "correct": true,
+              "correctResponse": "it",
+              "itemId": "w1",
+              "learnerResponse": "it",
+              "outcome": "correct",
+              "score": 1,
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": "isn't",
+              "itemId": "w2",
+              "learnerResponse": "isn't",
+              "outcome": "correct",
+              "score": 1,
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": "raining",
+              "itemId": "w3",
+              "learnerResponse": "raining",
+              "outcome": "correct",
+              "score": 1,
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": "in",
+              "itemId": "w4",
+              "learnerResponse": "in",
+              "outcome": "correct",
+              "score": 1,
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": "lisbon",
+              "itemId": "w5",
+              "learnerResponse": "lisbon",
+              "outcome": "correct",
+              "score": 1,
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": "today",
+              "itemId": "w6",
+              "learnerResponse": "today",
+              "outcome": "correct",
+              "score": 1,
+              "weight": 1
+            }
+          ],
+          "feedback": null,
+          "maxScore": 1,
+          "passed": true,
+          "score": 0.8,
+          "status": "scored"
+        },
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0.71875,
+            "scored": 0.71875
+          },
+          {
+            "hintsRevealed": 2,
+            "maxScore": 1,
+            "penalty": 0.2,
+            "score": 1,
+            "scored": 0.8
+          }
+        ]
+      },
+      "fn": "evaluateTries",
+      "id": "evaluateTries/dc/best"
+    },
+    {
+      "args": [
+        {
+          "id": "wr-weekend",
+          "maxWords": 12,
+          "minWords": 5,
+          "prompt": "Describe your last weekend in 5 to 12 words.",
+          "schemaVersion": "1.0",
+          "title": "Your weekend",
+          "type": "written-response"
+        },
+        {
+          "text": "One two three four five six.",
+          "type": "written-response",
+          "wordCount": 6
+        },
+        {
+          "scoring": {
+            "hintPenalty": 0.5
+          }
+        }
+      ],
+      "expect": {
+        "maxScore": 1,
+        "partial": {
+          "withinWordBounds": true,
+          "wordCount": 6
+        },
+        "reason": "requires_async_grading",
+        "status": "deferred"
+      },
+      "fn": "evaluate",
+      "id": "evaluate/wr/scoring-policy-leaves-deferred",
+      "note": "A written response is graded later: a scoring policy has nothing to charge yet, and the outcome stays deferred rather than turning unscorable."
+    },
+    {
+      "args": [
+        {
+          "id": "wr-weekend",
+          "maxWords": 12,
+          "minWords": 5,
+          "prompt": "Describe your last weekend in 5 to 12 words.",
+          "schemaVersion": "1.0",
+          "title": "Your weekend",
+          "type": "written-response"
+        },
+        [
+          {
+            "text": "One two three four five six.",
+            "type": "written-response",
+            "wordCount": 6
+          }
+        ],
+        {
+          "scoring": {
+            "retries": 1
+          }
+        }
+      ],
+      "expect": {
+        "counted": null,
+        "outcome": {
+          "maxScore": 1,
+          "partial": {
+            "withinWordBounds": true,
+            "wordCount": 6
+          },
+          "reason": "requires_async_grading",
+          "status": "deferred"
+        },
+        "tries": []
+      },
+      "fn": "evaluateTries",
+      "id": "evaluateTries/wr/scoring-policy-leaves-deferred"
+    },
+    {
+      "args": [
+        {
+          "id": "mc-capitals",
+          "mode": "multi",
+          "options": [
+            {
+              "id": "a",
+              "isCorrect": true,
+              "text": "Lisbon"
+            },
+            {
+              "id": "b",
+              "isCorrect": false,
+              "text": "Porto"
+            },
+            {
+              "id": "c",
+              "isCorrect": true,
+              "text": "Madrid"
+            },
+            {
+              "id": "d",
+              "isCorrect": false,
+              "text": "Seville"
+            }
+          ],
+          "question": "Which of these are capital cities?",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "all-or-nothing",
+          "title": "Capitals",
+          "type": "multiple-choice"
+        },
+        [],
+        {
+          "scoring": {
+            "counts": "best",
+            "hintPenalty": 0.1,
+            "retries": 2,
+            "retryPenalty": 0.25
+          }
+        }
+      ],
+      "expect": {
+        "counted": null,
+        "outcome": {
+          "maxScore": 1,
+          "reason": "no_response_recorded",
+          "status": "deferred"
+        },
+        "tries": []
+      },
+      "fn": "evaluateTries",
+      "id": "evaluateTries/no-try-is-no-response-recorded"
+    },
+    {
+      "args": [
+        {
+          "id": "mc-capitals",
+          "mode": "multi",
+          "options": [
+            {
+              "id": "a",
+              "isCorrect": true,
+              "text": "Lisbon"
+            },
+            {
+              "id": "b",
+              "isCorrect": false,
+              "text": "Porto"
+            },
+            {
+              "id": "c",
+              "isCorrect": true,
+              "text": "Madrid"
+            },
+            {
+              "id": "d",
+              "isCorrect": false,
+              "text": "Seville"
+            }
+          ],
+          "question": "Which of these are capital cities?",
+          "schemaVersion": "1.0",
+          "scoringStrategy": "all-or-nothing",
+          "title": "Capitals",
+          "type": "multiple-choice"
+        },
+        [
+          {
+            "selectedOptionIds": ["a"],
+            "type": "multiple-choice"
+          },
+          {
+            "selectedOptionIds": ["a", "c"],
+            "type": "multiple-choice"
+          }
+        ]
+      ],
+      "expect": {
+        "counted": 0,
+        "outcome": {
+          "details": [
+            {
+              "correct": true,
+              "correctResponse": ["selected"],
+              "itemId": "a",
+              "learnerResponse": ["selected"],
+              "outcome": "correct",
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": ["not-selected"],
+              "itemId": "b",
+              "learnerResponse": ["not-selected"],
+              "outcome": "correct-omission",
+              "weight": 1
+            },
+            {
+              "correct": false,
+              "correctResponse": ["selected"],
+              "itemId": "c",
+              "learnerResponse": ["not-selected"],
+              "outcome": "incorrect-omission",
+              "weight": 1
+            },
+            {
+              "correct": true,
+              "correctResponse": ["not-selected"],
+              "itemId": "d",
+              "learnerResponse": ["not-selected"],
+              "outcome": "correct-omission",
+              "weight": 1
+            }
+          ],
+          "feedback": null,
+          "maxScore": 1,
+          "passed": false,
+          "score": 0,
+          "status": "scored"
+        },
+        "tries": [
+          {
+            "hintsRevealed": 0,
+            "maxScore": 1,
+            "penalty": 0,
+            "score": 0,
+            "scored": 0
+          }
+        ]
+      },
+      "fn": "evaluateTries",
+      "id": "evaluateTries/no-policy-is-evaluate-of-the-first"
     }
   ]
 }