chaffjs 0.18.0 → 0.20.0

package/dist/grade/text.js

@@ -0,0 +1,117 @@
+import { ATOM_KINDS } from "../compare/atom.js";
+/** Where the problem is, when it is on a line of a file. An item given to grade() has none. */
+const lineJa = (problem) => (problem.line > 0 ? `${String(problem.line)} 行目: ` : "");
+const lineEn = (problem) => (problem.line > 0 ? `line ${String(problem.line)}: ` : "");
+const PROBLEM_JA = {
+    "not-json": (problem) => `${lineJa(problem)}JSON として読めません（${problem.detail ?? ""}）`,
+    "not-object": (problem) => `${lineJa(problem)}1 行に 1 つのオブジェクト {…} を書いてください`,
+    "no-id": (problem) => `${lineJa(problem)}id（空でない文字列）がありません`,
+    "no-output": (problem) => `${lineJa(problem)}output（文字列）がありません`,
+    "not-string": (problem) => `${lineJa(problem)}${problem.detail ?? ""} は文字列で書いてください`,
+    "not-text-map": (problem) => `${lineJa(problem)}${problem.detail ?? ""} は { "名前": "本文" } の形で書いてください`,
+    "not-citations": (problem) => `${lineJa(problem)}citations は [{ "source", "address", "quote" }] の配列で書いてください`,
+    "citations-without-sources": (problem) => `${lineJa(problem)}citations があるのに sources がありません。引用を照らす原文を sources に入れてください`,
+    "not-contexts": (problem) => `${lineJa(problem)}contexts は文字列の配列（["検索で取った一節", …]）で書いてください`,
+    "duplicate-id": (problem) => `${lineJa(problem)}id "${problem.detail ?? ""}" は ${String(problem.first ?? 0)} 行目と同じです`,
+    "no-variant": (problem) => `${lineJa(problem)}${problem.detail ?? ""}（どの prompt や model の出力かを示す名前）がありません。並べて比べるファイルでは、どの行にも書いてください`,
+    "unknown-source": (problem) => `${lineJa(problem)}引用の source "${problem.detail ?? ""}" が sources にありません`,
+    "which-source": (problem) => `${lineJa(problem)}番地 ${problem.detail ?? ""} の引用に source がありません。原文が二つ以上あるときは名前を書いてください`,
+    "unknown-language": (problem) => `${lineJa(problem)}language "${problem.detail ?? ""}" は読めません`,
+    "unknown-genre": (problem) => `${lineJa(problem)}genre "${problem.detail ?? ""}" は chaff の知らないジャンルです（chaff genres で一覧）`,
+    empty: () => "採点する出力が 1 行もありません",
+};
+const PROBLEM_EN = {
+    "not-json": (problem) => `${lineEn(problem)}not valid JSON (${problem.detail ?? ""})`,
+    "not-object": (problem) => `${lineEn(problem)}write one object {…} per line`,
+    "no-id": (problem) => `${lineEn(problem)}no id (a non-empty string)`,
+    "no-output": (problem) => `${lineEn(problem)}no output (a string)`,
+    "not-string": (problem) => `${lineEn(problem)}${problem.detail ?? ""} must be a string`,
+    "not-text-map": (problem) => `${lineEn(problem)}${problem.detail ?? ""} must be { "name": "text" }`,
+    "not-citations": (problem) => `${lineEn(problem)}citations must be an array of { "source", "address", "quote" }`,
+    "citations-without-sources": (problem) => `${lineEn(problem)}citations without sources; put the text they quote in sources`,
+    "not-contexts": (problem) => `${lineEn(problem)}contexts must be an array of strings (["a retrieved passage", …])`,
+    "duplicate-id": (problem) => `${lineEn(problem)}id "${problem.detail ?? ""}" is already on line ${String(problem.first ?? 0)}`,
+    "no-variant": (problem) => `${lineEn(problem)}no ${problem.detail ?? ""} (the variant's name); when a file compares variants, every line needs one`,
+    "unknown-source": (problem) => `${lineEn(problem)}the citation's source "${problem.detail ?? ""}" is not in sources`,
+    "which-source": (problem) => `${lineEn(problem)}the citation of ${problem.detail ?? ""} names no source; with two or more sources, name one`,
+    "unknown-language": (problem) => `${lineEn(problem)}language "${problem.detail ?? ""}" cannot be read`,
+    "unknown-genre": (problem) => `${lineEn(problem)}genre "${problem.detail ?? ""}" is not one chaff knows (chaff genres lists them)`,
+    empty: () => "no output to grade: the file has no item",
+};
+const EXPECTED_JA = {
+    map: "キーと値の組（{ … }）",
+    count: " 0 以上の整数",
+    number: " 0 以上の数",
+    boolean: " true か false",
+    words: "空でない文字列の並び",
+    kinds: `事実の種類（${ATOM_KINDS.join("・")}）の並び`,
+    "known-key": "知っているキーだけ",
+};
+const EXPECTED_EN = {
+    map: "a map ({ … })",
+    count: "a whole number from 0",
+    number: "a number from 0",
+    boolean: "true or false",
+    words: "a list of non-empty strings",
+    kinds: `a list of fact kinds (${ATOM_KINDS.join(", ")})`,
+    "known-key": "a key chaff knows",
+};
+/** A bracketed aside, or nothing when there is nothing to say. */
+const aside = (open, inner, close) => (inner === "" ? "" : [open, inner, close].join(""));
+export const GRADE_TEXT = {
+    ja: {
+        usage: "使い方: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <前の results.jsonl>] [--variant-key <欄の名前>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <ジャンル>]",
+        noReference: "reference が無い（事実は reference と照らす）",
+        noCitations: "citations が無い（chaff は出力から引用を推測しない）",
+        noContexts: "contexts が無い（回答の事実は、検索で取った一節と照らす）",
+        contextsUnread: (kind, reason) => `contexts と照らすとき、${kind} を読み切れなかった（${reason}）。その種類の事実は、一節にあっても見つからないことがある`,
+        uncheckedSentences: (count) => `数・日付・名前・引用の無い文 ${String(count)} 件は、contexts に支えられているかを照らしていない（意味を読む必要がある）`,
+        unknownRule: "grade: に書かれているが、chaff の知らないルール",
+        rubricProblem: (problem) => `chaff.yaml の ${problem.path} は${EXPECTED_JA[problem.expected]}で書いてください（書かれていたのは ${problem.written}）`,
+        penalty: (total) => `減点の和: ${String(total)}`,
+        noStructure: (source, language) => `原文 ${source} の言語 ${language} のパッケージは文書の構造を読めない`,
+        unreadable: (path, why) => `${path} を読めませんでした: ${why}`,
+        problem: (problem) => PROBLEM_JA[problem.kind](problem),
+        cannotLoad: (why) => `言語パッケージを読み込めません: ${why}`,
+        wrote: (path, count) => `出力ごとの結果を書きました: ${path}（${String(count)} 行）`,
+        totals: (path, total, passed) => `${path}: ${String(total)} 件の出力、${String(passed)} 件が通り、${String(total - passed)} 件が落ちた`,
+        failedHeading: "落ちた出力",
+        ratesHeading: (unit) => `ルールごとの率（1,000 ${unit}あたり、指摘のあった出力の数）`,
+        unit: { char: "字", word: "語" },
+        outputsWith: (count) => `${String(count)} 件`,
+        facts: (dropped, added, droppedKinds, addedKinds) => `事実: 落ちた ${String(dropped)}${aside("（", droppedKinds, "）")}、足された ${String(added)}${aside("（", addedKinds, "）")}`,
+        citations: (checked, failed) => `引用: ${String(checked)} 件を照らし、${String(failed)} 件が外れた`,
+        contexts: (outputs, checked, unsupported, kinds) => `contexts: ${String(outputs)} 件の出力の事実 ${String(checked)} 件を照らし、${String(unsupported)} 件がどの一節にも無かった${aside("（", kinds, "）")}`,
+        notRunHeading: (count) => `動かなかったもの ${String(count)} 件`,
+        inOutputs: (count) => `${String(count)} 件の出力`,
+        stamp: "再現の印",
+    },
+    en: {
+        usage: "usage: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <earlier results.jsonl>] [--variant-key <field>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <genre>]",
+        noReference: "no reference given (facts are checked against a reference)",
+        noCitations: "no citations given (chaff does not guess quotations from the output)",
+        noContexts: "no contexts given (the output's facts are checked against retrieved passages)",
+        contextsUnread: (kind, reason) => `against contexts, ${kind} could not be fully read (${reason}); a fact of that kind may be in a passage and still not be found`,
+        uncheckedSentences: (count) => `${String(count)} ${count === 1 ? "sentence states" : "sentences state"} no number, date, name or quotation, so their support by contexts is not checked (it needs reading)`,
+        unknownRule: "named under grade: but not a rule chaff knows",
+        rubricProblem: (problem) => `chaff.yaml: ${problem.path} must be ${EXPECTED_EN[problem.expected]} (found ${problem.written})`,
+        penalty: (total) => `Penalty points: ${String(total)}`,
+        noStructure: (source, language) => `the ${language} package cannot read the structure of source ${source}`,
+        unreadable: (path, why) => `Could not read ${path}: ${why}`,
+        problem: (problem) => PROBLEM_EN[problem.kind](problem),
+        cannotLoad: (why) => `Cannot load a language package: ${why}`,
+        wrote: (path, count) => `Wrote one result per output: ${path} (${String(count)} lines)`,
+        totals: (path, total, passed) => `${path}: ${String(total)} outputs, ${String(passed)} passed, ${String(total - passed)} failed`,
+        failedHeading: "Failed outputs",
+        ratesHeading: (unit) => `Rule rates (per 1,000 ${unit}, outputs with a finding)`,
+        unit: { char: "characters", word: "words" },
+        outputsWith: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"}`,
+        facts: (dropped, added, droppedKinds, addedKinds) => `Facts: ${String(dropped)} dropped${aside(" (", droppedKinds, ")")}, ${String(added)} added${aside(" (", addedKinds, ")")}`,
+        citations: (checked, failed) => `Quotations: ${String(checked)} checked, ${String(failed)} failed`,
+        contexts: (outputs, checked, unsupported, kinds) => `Contexts: ${String(checked)} facts checked in ${String(outputs)} ${outputs === 1 ? "output" : "outputs"}, ${String(unsupported)} in no passage${aside(" (", kinds, ")")}`,
+        notRunHeading: (count) => `${String(count)} not run`,
+        inOutputs: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"}`,
+        stamp: "Stamp",
+    },
+};
+//# sourceMappingURL=text.js.map

package/dist/grade/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/grade/text.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAgChD,+FAA+F;AAC/F,MAAM,MAAM,GAAG,CAAC,OAAoB,EAAU,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAE1G,MAAM,MAAM,GAAG,CAAC,OAAoB,EAAU,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAE5G,MAAM,UAAU,GAAwE;IACtF,UAAU,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,OAAO,CAAC,MAAM,IAAI,EAAE,GAAG;IACnF,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,8BAA8B;IAC3E,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB;IAC3D,WAAW,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB;IAC/D,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,eAAe;IACnF,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,8BAA8B;IACpG,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,4DAA4D;IAC5G,2BAA2B,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,2DAA2D;IACvH,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,2CAA2C;IAC1F,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,OAAO,CAAC,MAAM,IAAI,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,UAAU;IACrH,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CACxB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,+DAA+D;IAC1G,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,eAAe,OAAO,CAAC,MAAM,IAAI,EAAE,oBAAoB;IACxG,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,OAAO,CAAC,MAAM,IAAI,EAAE,4CAA4C;IACrH,kBAAkB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,OAAO,CAAC,MAAM,IAAI,EAAE,UAAU;IAC9F,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,OAAO,CAAC,MAAM,IAAI,EAAE,yCAAyC;IACvH,KAAK,EAAE,GAAG,EAAE,CAAC,mBAAmB;CACjC,CAAC;AAEF,MAAM,UAAU,GAAsB;IACpC,UAAU,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB,OAAO,CAAC,MAAM,IAAI,EAAE,GAAG;IACrF,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,+BAA+B;IAC5E,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,4BAA4B;IACpE,WAAW,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,sBAAsB;IAClE,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,mBAAmB;IACvF,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,6BAA6B;IACnG,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,gEAAgE;IAChH,2BAA2B,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,+DAA+D;IAC3H,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,mEAAmE;IAClH,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,OAAO,CAAC,MAAM,IAAI,EAAE,wBAAwB,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE;IAC9H,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,OAAO,CAAC,MAAM,IAAI,EAAE,4EAA4E;IACnJ,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,0BAA0B,OAAO,CAAC,MAAM,IAAI,EAAE,qBAAqB;IACpH,cAAc,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB,OAAO,CAAC,MAAM,IAAI,EAAE,sDAAsD;IAC5I,kBAAkB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,OAAO,CAAC,MAAM,IAAI,EAAE,kBAAkB;IACtG,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,OAAO,CAAC,MAAM,IAAI,EAAE,oDAAoD;IAClI,KAAK,EAAE,GAAG,EAAE,CAAC,0CAA0C;CACxD,CAAC;AAEF,MAAM,WAAW,GAAuC;IACtD,GAAG,EAAE,eAAe;IACpB,KAAK,EAAE,UAAU;IACjB,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,eAAe;IACxB,KAAK,EAAE,YAAY;IACnB,KAAK,EAAE,SAAS,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;IAC1C,WAAW,EAAE,WAAW;CACzB,CAAC;AAEF,MAAM,WAAW,GAAuC;IACtD,GAAG,EAAE,eAAe;IACpB,KAAK,EAAE,uBAAuB;IAC9B,MAAM,EAAE,iBAAiB;IACzB,OAAO,EAAE,eAAe;IACxB,KAAK,EAAE,6BAA6B;IACpC,KAAK,EAAE,yBAAyB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;IACxD,WAAW,EAAE,mBAAmB;CACjC,CAAC;AAEF,kEAAkE;AAClE,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,KAAa,EAAU,EAAE,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAE1H,MAAM,CAAC,MAAM,UAAU,GAAqB;IAC1C,EAAE,EAAE;QACF,KAAK,EACH,oMAAoM;QACtM,WAAW,EAAE,mCAAmC;QAChD,WAAW,EAAE,oCAAoC;QACjD,UAAU,EAAE,mCAAmC;QAC/C,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,mBAAmB,IAAI,cAAc,MAAM,+BAA+B;QAC5G,kBAAkB,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,4CAA4C;QAC1G,WAAW,EAAE,gCAAgC;QAC7C,aAAa,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,gBAAgB,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,OAAO,CAAC,OAAO,GAAG;QACjI,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,MAAM,CAAC,KAAK,CAAC,EAAE;QAC5C,WAAW,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,MAAM,MAAM,QAAQ,QAAQ,oBAAoB;QACnF,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,eAAe,GAAG,EAAE;QACtD,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;QACvD,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,GAAG,EAAE;QAC9C,KAAK,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,kBAAkB,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK;QACpE,MAAM,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,SAAS,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,QAAQ;QACxH,aAAa,EAAE,OAAO;QACtB,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,IAAI,iBAAiB;QAC9D,IAAI,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE;QAC9B,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI;QAC5C,KAAK,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,EAAE,CAClD,WAAW,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,CAAC,SAAS,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE;QAClH,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,UAAU,MAAM,CAAC,MAAM,CAAC,QAAQ;QACtF,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,CACjD,aAAa,MAAM,CAAC,OAAO,CAAC,YAAY,MAAM,CAAC,OAAO,CAAC,UAAU,MAAM,CAAC,WAAW,CAAC,gBAAgB,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE;QAC9H,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,IAAI;QACvD,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO;QAC7C,KAAK,EAAE,MAAM;KACd;IACD,EAAE,EAAE;QACF,KAAK,EACH,6MAA6M;QAC/M,WAAW,EAAE,4DAA4D;QACzE,WAAW,EAAE,sEAAsE;QACnF,UAAU,EAAE,+EAA+E;QAC3F,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAC/B,qBAAqB,IAAI,6BAA6B,MAAM,mEAAmE;QACjI,kBAAkB,EAAE,CAAC,KAAK,EAAE,EAAE,CAC5B,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,qGAAqG;QAC9K,WAAW,EAAE,+CAA+C;QAC5D,aAAa,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,eAAe,OAAO,CAAC,IAAI,YAAY,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,OAAO,CAAC,OAAO,GAAG;QAC7H,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,MAAM,CAAC,KAAK,CAAC,EAAE;QACtD,WAAW,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,OAAO,QAAQ,gDAAgD,MAAM,EAAE;QAC1G,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,kBAAkB,IAAI,KAAK,GAAG,EAAE;QAC3D,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;QACvD,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,mCAAmC,GAAG,EAAE;QAC7D,KAAK,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,gCAAgC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,SAAS;QACvF,MAAM,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,YAAY,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,SAAS;QAChI,aAAa,EAAE,gBAAgB;QAC/B,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,yBAAyB,IAAI,2BAA2B;QAChF,IAAI,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE;QAC3C,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE;QAChF,KAAK,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,EAAE,CAClD,UAAU,MAAM,CAAC,OAAO,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE;QAC7H,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,SAAS;QAClG,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,CACjD,aAAa,MAAM,CAAC,OAAO,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,WAAW,CAAC,iBAAiB,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE;QAC5K,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU;QACpD,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE;QAC9E,KAAK,EAAE,OAAO;KACf;CACF,CAAC"}

package/dist/grade/variants-input.d.ts

@@ -0,0 +1,12 @@
+import type { GradeResult } from "./result.ts";
+/** Each variant's results, by variant label then id, in order of first appearance. */
+export type VariantGroups = ReadonlyMap<string, ReadonlyMap<string, GradeResult>>;
+/** Labelled results, or each variant's results under its label. */
+export type VariantInput = readonly GradeResult[] | Readonly<Record<string, readonly GradeResult[]>>;
+/** The results grouped by variant, or every reason some cannot be. No result at all is a problem, not an empty comparison. */
+export declare const variantGroupsOf: (input: unknown) => {
+    readonly groups: VariantGroups;
+} | {
+    readonly problems: readonly string[];
+};
+//# sourceMappingURL=variants-input.d.ts.map

package/dist/grade/variants-input.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"variants-input.d.ts","sourceRoot":"","sources":["../../src/grade/variants-input.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAM/C,sFAAsF;AACtF,MAAM,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;AAElF,mEAAmE;AACnE,MAAM,MAAM,YAAY,GAAG,SAAS,WAAW,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC,CAAC;AA6BrG,8HAA8H;AAC9H,eAAO,MAAM,eAAe,GAAI,OAAO,OAAO,KAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GAAG;IAAE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAA;CAa3H,CAAC"}

package/dist/grade/variants-input.js

@@ -0,0 +1,42 @@
+import { isGradeResult } from "./results-read.js";
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+const labelledOf = (input) => {
+    if (Array.isArray(input))
+        return input.map((result, index) => ({ variant: isRecord(result) ? result["variant"] : undefined, result, place: `results[${String(index)}]` }));
+    if (!isRecord(input))
+        return undefined;
+    return Object.entries(input).flatMap(([variant, results]) => Array.isArray(results)
+        ? results.map((result, index) => ({ variant, result, place: `${variant}[${String(index)}]` }))
+        : [{ variant, result: results, place: variant }]);
+};
+/** A whole result line, as `--out` writes it, with an id to match by: a partial object would break the table, not be compared. */
+const isResult = (value) => isGradeResult(value) && value.id !== "";
+/** Why one entry cannot be compared, or undefined when it can. */
+const problemOf = (entry, groups) => {
+    if (!isResult(entry.result))
+        return `${entry.place}: not a grade result with an id`;
+    if (typeof entry.variant !== "string" || entry.variant === "")
+        return `${entry.place}: id "${entry.result.id}" has no variant`;
+    return groups.get(entry.variant)?.has(entry.result.id) === true
+        ? `${entry.place}: id "${entry.result.id}" is already in variant "${entry.variant}"`
+        : undefined;
+};
+/** The results grouped by variant, or every reason some cannot be. No result at all is a problem, not an empty comparison. */
+export const variantGroupsOf = (input) => {
+    const labelled = labelledOf(input);
+    if (labelled === undefined)
+        return { problems: ["results must be an array of grade results, or { variant: results[] }"] };
+    const groups = new Map();
+    const problems = labelled.flatMap((entry) => {
+        const problem = problemOf(entry, groups);
+        if (problem !== undefined || !isResult(entry.result) || typeof entry.variant !== "string")
+            return problem === undefined ? [] : [problem];
+        const byId = groups.get(entry.variant) ?? new Map();
+        groups.set(entry.variant, byId.set(entry.result.id, { ...entry.result, variant: entry.variant }));
+        return [];
+    });
+    if (problems.length > 0)
+        return { problems };
+    return groups.size === 0 ? { problems: ["no results to compare"] } : { groups };
+};
+//# sourceMappingURL=variants-input.js.map

package/dist/grade/variants-input.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"variants-input.js","sourceRoot":"","sources":["../../src/grade/variants-input.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAalD,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,UAAU,GAAG,CAAC,KAAc,EAA0B,EAAE;IAC5D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACtB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,MAAe,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5J,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAC1D,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QACpB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAe,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;QACvG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CACnD,CAAC;AACJ,CAAC,CAAC;AAEF,kIAAkI;AAClI,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAwB,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC;AAEnG,kEAAkE;AAClE,MAAM,SAAS,GAAG,CAAC,KAAe,EAAE,MAA6D,EAAsB,EAAE;IACvH,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,GAAG,KAAK,CAAC,KAAK,iCAAiC,CAAC;IACpF,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,KAAK,EAAE;QAAE,OAAO,GAAG,KAAK,CAAC,KAAK,SAAS,KAAK,CAAC,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC/H,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI;QAC7D,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,SAAS,KAAK,CAAC,MAAM,CAAC,EAAE,4BAA4B,KAAK,CAAC,OAAO,GAAG;QACpF,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC,CAAC;AAEF,8HAA8H;AAC9H,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAAc,EAAiF,EAAE;IAC/H,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,sEAAsE,CAAC,EAAE,CAAC;IAC1H,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoC,CAAC;IAC3D,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1C,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACzC,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACzI,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,EAAuB,CAAC;QACzE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClG,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC7C,OAAO,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;AAClF,CAAC,CAAC"}

package/dist/grade/variants-text.d.ts

@@ -0,0 +1,31 @@
+import type { Texts } from "../ui.ts";
+export type VariantText = {
+    readonly heading: (compared: number, variants: number) => string;
+    readonly passed: string;
+    readonly passedCell: (passed: number, outputs: number, rate: string) => string;
+    readonly factsDropped: string;
+    readonly factsAdded: string;
+    readonly citationsFailed: string;
+    readonly citationsCell: (failed: number, checked: number) => string;
+    readonly unsupportedFacts: string;
+    readonly penalty: string;
+    readonly ratesHeading: (unit: string) => string;
+    readonly unit: Readonly<Record<"char" | "word", string>>;
+    readonly disagreeHeading: (count: number) => string;
+    readonly disagreement: (id: string, passedIn: string, failedIn: string) => string;
+    readonly failedIn: (variant: string, reasons: string) => string;
+    readonly noDisagreement: string;
+    readonly missing: (id: string, variants: string) => string;
+    readonly readOtherwise: (ids: string) => string;
+    readonly notComparedHeading: string;
+    /** Markdown only: the run's heading and its totals. */
+    readonly runHeading: (path: string) => string;
+    readonly totals: (total: number, passed: number) => string;
+    readonly failedHeading: string;
+    readonly regressionsHeading: (count: number) => string;
+    readonly noRegression: string;
+    readonly stamp: string;
+    readonly separator: string;
+};
+export declare const VARIANT_TEXT: Texts<VariantText>;
+//# sourceMappingURL=variants-text.d.ts.map

package/dist/grade/variants-text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"variants-text.d.ts","sourceRoot":"","sources":["../../src/grade/variants-text.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAEtC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IAC/E,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;IACpE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IAChD,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,QAAQ,CAAC,eAAe,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IACpD,QAAQ,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC;IAClF,QAAQ,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;IAChE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC;IAC3D,QAAQ,CAAC,aAAa,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IAChD,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,uDAAuD;IACvD,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IAC9C,QAAQ,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IAC3D,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IACvD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,eAAO,MAAM,YAAY,EAAE,KAAK,CAAC,WAAW,CAwD3C,CAAC"}

package/dist/grade/variants-text.js

@@ -0,0 +1,57 @@
+export const VARIANT_TEXT = {
+    ja: {
+        heading: (compared, variants) => `${String(variants)} つの variant を並べた: どの variant にもある id の出力 ${String(compared)} 件`,
+        passed: "通った",
+        passedCell: (passed, outputs, rate) => `${String(passed)}/${String(outputs)}（${rate}）`,
+        factsDropped: "落ちた事実",
+        factsAdded: "足された事実",
+        citationsFailed: "外れた引用",
+        citationsCell: (failed, checked) => `${String(failed)}/${String(checked)}`,
+        unsupportedFacts: "一節に無い事実",
+        penalty: "減点の和",
+        ratesHeading: (unit) => `ルールごとの率（1,000 ${unit}あたり）`,
+        unit: { char: "字", word: "語" },
+        disagreeHeading: (count) => `合否が分かれた出力 ${String(count)} 件`,
+        disagreement: (id, passedIn, failedIn) => `${id}: 通った ${passedIn}、落ちた ${failedIn}`,
+        failedIn: (variant, reasons) => `${variant}（${reasons}）`,
+        noDisagreement: "合否が分かれた出力はありません",
+        missing: (id, variants) => `${id}: ${variants} に無い`,
+        readOtherwise: (ids) => `言語かジャンルが違う: ${ids}`,
+        notComparedHeading: "比べなかった id",
+        runHeading: (path) => `chaff grade: ${path}`,
+        totals: (total, passed) => `${String(total)} 件の出力、${String(passed)} 件が通り、${String(total - passed)} 件が落ちた`,
+        failedHeading: "落ちた出力",
+        regressionsHeading: (count) => `前の回と比べた回帰 ${String(count)} 件`,
+        noRegression: "前の回と比べた回帰はありません",
+        stamp: "再現の印",
+        separator: "、",
+    },
+    en: {
+        heading: (compared, variants) => `${String(variants)} variants side by side: ${String(compared)} ${compared === 1 ? "output" : "outputs"} with an id every variant has`,
+        passed: "Passed",
+        passedCell: (passed, outputs, rate) => `${String(passed)}/${String(outputs)} (${rate})`,
+        factsDropped: "Facts dropped",
+        factsAdded: "Facts added",
+        citationsFailed: "Quotations failed",
+        citationsCell: (failed, checked) => `${String(failed)}/${String(checked)}`,
+        unsupportedFacts: "Unsupported facts",
+        penalty: "Penalty points",
+        ratesHeading: (unit) => `Rule rates (per 1,000 ${unit})`,
+        unit: { char: "characters", word: "words" },
+        disagreeHeading: (count) => `${String(count)} ${count === 1 ? "output" : "outputs"} where pass or fail differs`,
+        disagreement: (id, passedIn, failedIn) => `${id}: passed in ${passedIn}; failed in ${failedIn}`,
+        failedIn: (variant, reasons) => `${variant} (${reasons})`,
+        noDisagreement: "Every variant passed or failed the same outputs",
+        missing: (id, variants) => `${id}: missing from ${variants}`,
+        readOtherwise: (ids) => `another language or genre: ${ids}`,
+        notComparedHeading: "Ids not compared",
+        runHeading: (path) => `chaff grade: ${path}`,
+        totals: (total, passed) => `${String(total)} outputs, ${String(passed)} passed, ${String(total - passed)} failed`,
+        failedHeading: "Failed outputs",
+        regressionsHeading: (count) => `${String(count)} ${count === 1 ? "regression" : "regressions"} against the baseline`,
+        noRegression: "No regression against the baseline",
+        stamp: "Stamp",
+        separator: "; ",
+    },
+};
+//# sourceMappingURL=variants-text.js.map

package/dist/grade/variants-text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"variants-text.js","sourceRoot":"","sources":["../../src/grade/variants-text.ts"],"names":[],"mappings":"AA+BA,MAAM,CAAC,MAAM,YAAY,GAAuB;IAC9C,EAAE,EAAE;QACF,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,4CAA4C,MAAM,CAAC,QAAQ,CAAC,IAAI;QACpH,MAAM,EAAE,KAAK;QACb,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG;QACtF,YAAY,EAAE,OAAO;QACrB,UAAU,EAAE,QAAQ;QACpB,eAAe,EAAE,OAAO;QACxB,aAAa,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE;QAC1E,gBAAgB,EAAE,SAAS;QAC3B,OAAO,EAAE,MAAM;QACf,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,IAAI,MAAM;QACnD,IAAI,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE;QAC9B,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,MAAM,CAAC,KAAK,CAAC,IAAI;QAC1D,YAAY,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,SAAS,QAAQ,QAAQ,QAAQ,EAAE;QAClF,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,IAAI,OAAO,GAAG;QACxD,cAAc,EAAE,iBAAiB;QACjC,OAAO,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,QAAQ,MAAM;QACnD,aAAa,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,GAAG,EAAE;QAC5C,kBAAkB,EAAE,WAAW;QAC/B,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,IAAI,EAAE;QAC5C,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,SAAS,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,QAAQ;QACzG,aAAa,EAAE,OAAO;QACtB,kBAAkB,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,MAAM,CAAC,KAAK,CAAC,IAAI;QAC7D,YAAY,EAAE,iBAAiB;QAC/B,KAAK,EAAE,MAAM;QACb,SAAS,EAAE,GAAG;KACf;IACD,EAAE,EAAE;QACF,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAC9B,GAAG,MAAM,CAAC,QAAQ,CAAC,2BAA2B,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,+BAA+B;QACxI,MAAM,EAAE,QAAQ;QAChB,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,GAAG;QACvF,YAAY,EAAE,eAAe;QAC7B,UAAU,EAAE,aAAa;QACzB,eAAe,EAAE,mBAAmB;QACpC,aAAa,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE;QAC1E,gBAAgB,EAAE,mBAAmB;QACrC,OAAO,EAAE,gBAAgB;QACzB,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,yBAAyB,IAAI,GAAG;QACxD,IAAI,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE;QAC3C,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,6BAA6B;QAC/G,YAAY,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,eAAe,QAAQ,eAAe,QAAQ,EAAE;QAC/F,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,KAAK,OAAO,GAAG;QACzD,cAAc,EAAE,iDAAiD;QACjE,OAAO,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,EAAE,kBAAkB,QAAQ,EAAE;QAC5D,aAAa,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,8BAA8B,GAAG,EAAE;QAC3D,kBAAkB,EAAE,kBAAkB;QACtC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,IAAI,EAAE;QAC5C,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,YAAY,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,SAAS;QACjH,aAAa,EAAE,gBAAgB;QAC/B,kBAAkB,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,uBAAuB;QACpH,YAAY,EAAE,oCAAoC;QAClD,KAAK,EAAE,OAAO;QACd,SAAS,EAAE,IAAI;KAChB;CACF,CAAC"}

package/dist/grade/variants.d.ts

@@ -0,0 +1,63 @@
+import type { LengthUnit } from "../plugin.ts";
+import type { GradeResult } from "./result.ts";
+import { type VariantGroups } from "./variants-input.ts";
+/** One variant over the compared ids. */
+export type VariantColumn = {
+    readonly variant: string;
+    readonly outputs: number;
+    readonly passed: number;
+    /** The share of outputs that passed, in percent to one decimal. Undefined when no id was compared. */
+    readonly passRate?: number | undefined;
+    /** Facts dropped and added against the reference, not counting the kinds the rubric allows. */
+    readonly facts: {
+        readonly dropped: number;
+        readonly added: number;
+    };
+    readonly citations: {
+        readonly checked: number;
+        readonly failed: number;
+    };
+    /** Facts checked against the retrieved passages, and those in no passage. Only when some output had contexts. */
+    readonly contexts?: {
+        readonly checked: number;
+        readonly unsupported: number;
+    } | undefined;
+    /** Only with a `grade:` rubric. */
+    readonly penalty?: number | undefined;
+};
+/** One rule's rate per 1,000 units in each variant, by variant label; 0 where the variant had no finding of it. */
+export type VariantRates = {
+    readonly rule: string;
+    readonly unit: LengthUnit;
+    readonly rates: Readonly<Record<string, number>>;
+};
+/** An id some variants passed and others failed, with why each failed. */
+export type Disagreement = {
+    readonly id: string;
+    readonly passedIn: readonly string[];
+    readonly failedIn: readonly {
+        readonly variant: string;
+        readonly failedBecause: readonly string[];
+    }[];
+};
+export type VariantComparison = {
+    /** In the order they first appear. */
+    readonly variants: readonly string[];
+    /** The ids every variant answered, read the same way. */
+    readonly compared: readonly string[];
+    /** Ids some variant did not answer, and which. */
+    readonly missing: readonly {
+        readonly id: string;
+        readonly missingFrom: readonly string[];
+    }[];
+    /** Ids every variant answered, but in another language or genre: not the same task graded the same way. */
+    readonly readOtherwise: readonly string[];
+    readonly columns: readonly VariantColumn[];
+    readonly rules: readonly VariantRates[];
+    readonly disagreements: readonly Disagreement[];
+};
+/** The variants side by side over the ids they all answered. `grouped`: each variant's results by id, the variants in order. */
+export declare const compareVariantGroups: (grouped: VariantGroups) => VariantComparison;
+/** A run's results compared variant by variant, or undefined when none has a variant label (the command line's input already checked). */
+export declare const variantsOfRun: (results: readonly GradeResult[]) => VariantComparison | undefined;
+//# sourceMappingURL=variants.d.ts.map

package/dist/grade/variants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"variants.d.ts","sourceRoot":"","sources":["../../src/grade/variants.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAAmB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAK1E,yCAAyC;AACzC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,sGAAsG;IACtG,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,+FAA+F;IAC/F,QAAQ,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrE,QAAQ,CAAC,SAAS,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1E,iHAAiH;IACjH,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC;IAC3F,mCAAmC;IACnC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC,CAAC;AAEF,mHAAmH;AACnH,MAAM,MAAM,YAAY,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CAAE,CAAC;AAElI,0EAA0E;AAC1E,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;CACvG,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,sCAAsC;IACtC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,kDAAkD;IAClD,QAAQ,CAAC,OAAO,EAAE,SAAS;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC9F,2GAA2G;IAC3G,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAC3C,QAAQ,CAAC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,YAAY,EAAE,CAAC;CACjD,CAAC;AA0CF,gIAAgI;AAChI,eAAO,MAAM,oBAAoB,GAAI,SAAS,aAAa,KAAG,iBAoB7D,CAAC;AAEF,0IAA0I;AAC1I,eAAO,MAAM,aAAa,GAAI,SAAS,SAAS,WAAW,EAAE,KAAG,iBAAiB,GAAG,SAInF,CAAC"}

package/dist/grade/variants.js

@@ -0,0 +1,66 @@
+import { rateRows, sameReading } from "./pairing.js";
+import { summaryOf } from "./summary.js";
+import { variantGroupsOf } from "./variants-input.js";
+const PERCENT = 100;
+const PERCENT_SCALE = 10;
+const percentOf = (passed, outputs) => outputs === 0 ? undefined : Math.round((passed * PERCENT * PERCENT_SCALE) / outputs) / PERCENT_SCALE;
+const sum = (counts) => Object.values(counts).reduce((total, count) => total + count, 0);
+const columnOf = (variant, results) => {
+    const summary = summaryOf(results);
+    return {
+        variant,
+        outputs: summary.total,
+        passed: summary.passed,
+        passRate: percentOf(summary.passed, summary.total),
+        facts: { dropped: sum(summary.facts.dropped), added: sum(summary.facts.added) },
+        citations: summary.citations,
+        ...(summary.contexts === undefined ? {} : { contexts: { checked: summary.contexts.checked, unsupported: sum(summary.contexts.unsupported) } }),
+        ...(summary.penalty === undefined ? {} : { penalty: summary.penalty }),
+    };
+};
+const disagreementOf = (id, row, variants) => {
+    const passedIn = variants.filter((_, index) => row[index]?.pass === true);
+    if (passedIn.length === 0 || passedIn.length === variants.length)
+        return [];
+    const failedIn = variants.flatMap((variant, index) => {
+        const result = row[index];
+        return result === undefined || result.pass ? [] : [{ variant, failedBecause: result.failedBecause }];
+    });
+    return [{ id, passedIn, failedIn }];
+};
+/** Each id in order of first appearance, with its result in each variant, or undefined where the variant has none. */
+const rowsOf = (grouped, variants) => {
+    const ids = [...new Set([...grouped.values()].flatMap((byId) => [...byId.keys()]))];
+    return ids.map((id) => ({ id, row: variants.map((variant) => grouped.get(variant)?.get(id)) }));
+};
+const isComplete = (row) => row.every((result) => result !== undefined);
+/** The variants side by side over the ids they all answered. `grouped`: each variant's results by id, the variants in order. */
+export const compareVariantGroups = (grouped) => {
+    const variants = [...grouped.keys()];
+    const rows = rowsOf(grouped, variants);
+    const complete = rows.flatMap(({ id, row }) => (isComplete(row) ? [{ id, row }] : []));
+    const compared = complete.filter(({ row }) => sameReading(row));
+    const byVariant = variants.map((_, index) => compared.flatMap(({ row }) => row.slice(index, index + 1)));
+    return {
+        variants,
+        compared: compared.map(({ id }) => id),
+        missing: rows.flatMap(({ id, row }) => {
+            const missingFrom = variants.filter((_, index) => row[index] === undefined);
+            return missingFrom.length === 0 ? [] : [{ id, missingFrom }];
+        }),
+        readOtherwise: complete.filter(({ row }) => !sameReading(row)).map(({ id }) => id),
+        columns: variants.map((variant, index) => columnOf(variant, byVariant[index] ?? [])),
+        rules: rateRows(byVariant)
+            .filter((row) => row.rates.some((rate) => rate > 0))
+            .map(({ rule, unit, rates }) => ({ rule, unit, rates: Object.fromEntries(variants.map((variant, index) => [variant, rates[index] ?? 0])) })),
+        disagreements: compared.flatMap(({ id, row }) => disagreementOf(id, row, variants)),
+    };
+};
+/** A run's results compared variant by variant, or undefined when none has a variant label (the command line's input already checked). */
+export const variantsOfRun = (results) => {
+    if (!results.some((result) => result.variant !== undefined))
+        return undefined;
+    const read = variantGroupsOf(results);
+    return "groups" in read ? compareVariantGroups(read.groups) : undefined;
+};
+//# sourceMappingURL=variants.js.map

package/dist/grade/variants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"variants.js","sourceRoot":"","sources":["../../src/grade/variants.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AA6C1E,MAAM,OAAO,GAAG,GAAG,CAAC;AACpB,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,OAAe,EAAsB,EAAE,CACxE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,OAAO,GAAG,aAAa,CAAC,GAAG,OAAO,CAAC,GAAG,aAAa,CAAC;AAEvG,MAAM,GAAG,GAAG,CAAC,MAAwC,EAAU,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;AAEnI,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,OAA+B,EAAiB,EAAE;IACnF,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACnC,OAAO;QACL,OAAO;QACP,OAAO,EAAE,OAAO,CAAC,KAAK;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;QAClD,KAAK,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;QAC/E,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;QAC9I,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;KACvE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,EAAU,EAAE,GAA2B,EAAE,QAA2B,EAAkB,EAAE;IAC9G,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC5E,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QACnD,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1B,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACvG,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;AACtC,CAAC,CAAC;AAEF,sHAAsH;AACtH,MAAM,MAAM,GAAG,CAAC,OAAsB,EAAE,QAA2B,EAAiF,EAAE;IACpJ,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACpF,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAClG,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,GAAyC,EAAiC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AAE7I,gIAAgI;AAChI,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,OAAsB,EAAqB,EAAE;IAChF,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,OAAO;QACL,QAAQ;QACR,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;YACpC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,CAAC;YAC5E,OAAO,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAC/D,CAAC,CAAC;QACF,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;QAClF,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACpF,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC;aACvB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;aACnD,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9I,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,cAAc,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;KACpF,CAAC;AACJ,CAAC,CAAC;AAEF,0IAA0I;AAC1I,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAA+B,EAAiC,EAAE;IAC9F,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9E,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACtC,OAAO,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,CAAC,CAAC"}

package/dist/grade/verdict.d.ts

@@ -0,0 +1,18 @@
+import type { GradeFact, GradeResult } from "./result.ts";
+/** What pass or fail is decided from. */
+export type Graded = Pick<GradeResult, "findings" | "facts" | "citations" | "contexts">;
+export type Verdict = {
+    readonly pass: boolean;
+    readonly failedBecause: readonly string[];
+};
+/** A condition over a count, written as the reason when the count goes past its limit: `facts.dropped 3 > 0`. */
+export declare const overLimit: (name: string, count: number, limit: number) => string[];
+/** Facts that count against the output: those whose kind was not allowed to change. */
+export declare const counted: (facts: readonly GradeFact[]) => number;
+export declare const verdictOf: (failedBecause: readonly string[]) => Verdict;
+/**
+ * The pass or fail with no `grade:` in chaff.yaml (spec §29.3): an error finding, a fact dropped or added against the
+ * reference, a quotation not found, or a fact no retrieved passage states fails the output. Warnings and info are rates to compare, not a verdict on one output.
+ */
+export declare const defaultVerdict: (graded: Graded) => Verdict;
+//# sourceMappingURL=verdict.d.ts.map

package/dist/grade/verdict.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/grade/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1D,yCAAyC;AACzC,MAAM,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,UAAU,GAAG,OAAO,GAAG,WAAW,GAAG,UAAU,CAAC,CAAC;AAExF,MAAM,MAAM,OAAO,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,CAAC;AAE5F,iHAAiH;AACjH,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,OAAO,MAAM,EAAE,OAAO,MAAM,KAAG,MAAM,EAA4E,CAAC;AAE1J,uFAAuF;AACvF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,SAAS,EAAE,KAAG,MAAsD,CAAC;AAE7G,eAAO,MAAM,SAAS,GAAI,eAAe,SAAS,MAAM,EAAE,KAAG,OAAgE,CAAC;AAE9H;;;GAGG;AACH,eAAO,MAAM,cAAc,GAAI,QAAQ,MAAM,KAAG,OAO5C,CAAC"}

package/dist/grade/verdict.js

@@ -0,0 +1,17 @@
+/** A condition over a count, written as the reason when the count goes past its limit: `facts.dropped 3 > 0`. */
+export const overLimit = (name, count, limit) => (count > limit ? [`${name} ${String(count)} > ${String(limit)}`] : []);
+/** Facts that count against the output: those whose kind was not allowed to change. */
+export const counted = (facts) => facts.filter((fact) => !fact.allowed).length;
+export const verdictOf = (failedBecause) => ({ pass: failedBecause.length === 0, failedBecause });
+/**
+ * The pass or fail with no `grade:` in chaff.yaml (spec §29.3): an error finding, a fact dropped or added against the
+ * reference, a quotation not found, or a fact no retrieved passage states fails the output. Warnings and info are rates to compare, not a verdict on one output.
+ */
+export const defaultVerdict = (graded) => verdictOf([
+    ...overLimit("findings.error", graded.findings.filter((finding) => finding.level === "error").length, 0),
+    ...overLimit("facts.dropped", counted(graded.facts?.dropped ?? []), 0),
+    ...overLimit("facts.added", counted(graded.facts?.added ?? []), 0),
+    ...overLimit("citations.failed", graded.citations?.failed.length ?? 0, 0),
+    ...overLimit("contexts.unsupported", counted(graded.contexts?.unsupported ?? []), 0),
+]);
+//# sourceMappingURL=verdict.js.map

package/dist/grade/verdict.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/grade/verdict.ts"],"names":[],"mappings":"AAOA,iHAAiH;AACjH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,KAAa,EAAY,EAAE,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAE1J,uFAAuF;AACvF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,KAA2B,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;AAE7G,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,aAAgC,EAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC;AAE9H;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAc,EAAW,EAAE,CACxD,SAAS,CAAC;IACR,GAAG,SAAS,CAAC,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IACxG,GAAG,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IACtE,GAAG,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IAClE,GAAG,SAAS,CAAC,kBAAkB,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;IACzE,GAAG,SAAS,CAAC,sBAAsB,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;CACrF,CAAC,CAAC"}

package/dist/grade-api.d.ts

@@ -0,0 +1,53 @@
+import { type Config } from "./config/load.ts";
+import type { GradeResult } from "./grade/result.ts";
+import { type VariantComparison } from "./grade/variants.ts";
+import { type VariantInput } from "./grade/variants-input.ts";
+export type { FailedCitation, GradeCitations, GradeContexts, GradeFact, GradeFacts, GradeFinding, GradeResult, GradeScore, NotRunEntry, ScoreItem, Stamp, SupportedFact, } from "./grade/result.ts";
+export type { OutputSize } from "./grade/rates.ts";
+export { toScorer, type ChaffScore } from "./grade/scorer.ts";
+export type { Disagreement, VariantColumn, VariantComparison, VariantRates } from "./grade/variants.ts";
+export type { VariantInput } from "./grade/variants-input.ts";
+export type GradeOptions = {
+    /** The result's id. "output" when left out. */
+    readonly id?: string;
+    /** The document the output was made from: facts are checked against it, as `chaff compare` does. */
+    readonly reference?: string;
+    /** Each source's name and text. */
+    readonly sources?: Readonly<Record<string, string>>;
+    /** What the output quoted. `source` may be left out when there is one source. */
+    readonly citations?: readonly {
+        readonly source?: string;
+        readonly address: string;
+        readonly quote: string;
+    }[];
+    /** The retrieved passages the output was meant to rest on: each fact of the output is looked for in them. */
+    readonly contexts?: readonly string[];
+    readonly language?: string;
+    readonly genre?: string;
+    /** The prompt, model or setting that produced the output, kept on the result for compareVariants(). */
+    readonly variant?: string;
+    /** Run the experimental rules too, as --experimental does. chaff.yaml's `experimental` when left out. */
+    readonly experimental?: boolean;
+    /** chaff.yaml's path, or settings already read. Without it chaff's defaults apply and no file is read. */
+    readonly config?: string | Config;
+};
+/** An input or a chaff.yaml grade() cannot use: the same cases end `chaff grade` with exit 2. */
+export declare class GradeInputError extends Error {
+    readonly problems: readonly string[];
+    constructor(problems: readonly string[]);
+}
+/**
+ * Grades one output (spec §29.6): its findings and rates, its facts against `reference`, its `citations` against
+ * `sources`, and pass or fail by chaff.yaml's `grade:` or the default. Reads a file only when `config` is a path, and
+ * never writes one. Throws GradeInputError for an input or a chaff.yaml `chaff grade` would refuse with exit 2.
+ */
+export declare const grade: (output: string, options?: GradeOptions) => Promise<GradeResult>;
+/**
+ * Several variants' results side by side on the same inputs (spec §29.5), as `chaff grade` prints them for a file whose
+ * lines carry a variant: per variant the pass rate, each rule's rate, facts dropped and added, and failed quotations,
+ * over the ids every variant answered; and the ids where pass or fail differs. Takes results labelled by `variant`
+ * (grade() with `variant`, or `chaff grade --out`), or `{ [variant]: results }`. Throws GradeInputError for a result
+ * without an id or a variant, or an id given twice in one variant.
+ */
+export declare const compareVariants: (results: VariantInput) => VariantComparison;
+//# sourceMappingURL=grade-api.d.ts.map

package/dist/grade-api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grade-api.d.ts","sourceRoot":"","sources":["../src/grade-api.ts"],"names":[],"mappings":"AAOA,OAAO,EAAqB,KAAK,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAMlE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,EAAwB,KAAK,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACnF,OAAO,EAAmB,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAI/E,YAAY,EACV,cAAc,EACd,cAAc,EACd,aAAa,EACb,SAAS,EACT,UAAU,EACV,YAAY,EACZ,WAAW,EACX,UAAU,EACV,WAAW,EACX,SAAS,EACT,KAAK,EACL,aAAa,GACd,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC9D,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxG,YAAY,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,MAAM,MAAM,YAAY,GAAG;IACzB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;IACrB,oGAAoG;IACpG,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,mCAAmC;IACnC,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,iFAAiF;IACjF,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC/G,6GAA6G;IAC7G,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,uGAAuG;IACvG,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,yGAAyG;IACzG,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,0GAA0G;IAC1G,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC,CAAC;AAEF,iGAAiG;AACjG,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;gBAEzB,QAAQ,EAAE,SAAS,MAAM,EAAE;CAKxC;AA6ED;;;;GAIG;AACH,eAAO,MAAM,KAAK,GAAU,QAAQ,MAAM,EAAE,UAAS,YAAiB,KAAG,OAAO,CAAC,WAAW,CAiB3F,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAAI,SAAS,YAAY,KAAG,iBAIvD,CAAC"}