champollion 0.3.3

package/shared/catalogue/metric-coverage.json

@@ -0,0 +1,170 @@
+{
+  "_doc": "Which evaluation METRIC MODELS cover which languages. Transcribed cite-only from each model's own published list, with the URL and the date read — the same discipline as method-coverage.json. This exists because the harness selects a specialised COMET model per language, and the previous source for that was two hardcoded ISO lists inside a script, one of which cited the wrong paper on all 109 cards it touched.",
+  "_boundary": "This is RESOURCE EXISTENCE — that a metric model claims to cover a language. It is never a score, and never a claim about how well anything evaluates. A measured metric value is a run result keyed by (method, dataset, metric) and belongs on the leaderboard.",
+  "version": 1,
+  "asOf": "2026-08-05",
+  "models": [
+    {
+      "key": "africomet",
+      "label": "AfriCOMET-MTL",
+      "model": "masakhane/africomet-mtl",
+      "purpose": "reference-based MT evaluation for African languages",
+      "source_url": "https://huggingface.co/masakhane/africomet-mtl",
+      "asOf": "2026-08-05",
+      "verified": "publisher-list",
+      "citation": "Wang et al. AfriMTE and AfriCOMET: Empowering COMET to Embrace Under-resourced African Languages. arXiv:2311.09828.",
+      "citationNote": "The previous corpus cited this as 'wan-2022' on all 109 cards carrying the field. The model card names arXiv:2311.09828; the work was subsequently published at NAACL 2024, so both a 2023 (preprint) and 2024 (proceedings) year are defensible and the arXiv identifier is the unambiguous reference.",
+      "coverageStatement": "This model builds on top of AfroXLMR which cover the following languages: [...]. Thus, results for language pairs containing uncovered languages are unreliable!",
+      "publishedLanguageNames": [
+        "Afrikaans",
+        "Arabic",
+        "Amharic",
+        "English",
+        "French",
+        "Hausa",
+        "Igbo",
+        "Malagasy",
+        "Chichewa",
+        "Oromo",
+        "Nigerian-Pidgin",
+        "Kinyarwanda",
+        "Kirundi",
+        "Shona",
+        "Somali",
+        "Sesotho",
+        "Swahili",
+        "isiXhosa",
+        "Yoruba",
+        "isiZulu"
+      ],
+      "isoMappingNote": "The publisher lists NAMES, not codes. The ISO 639-3 mapping below is OURS and is marked as such — it is a standard, checkable correspondence, but it is our step and not the publisher's. Four of the twenty resolve to MACROLANGUAGES (ara, mlg, orm, swa); whether coverage of a macrolanguage extends to its individual members is the same open question the atlas records for method coverage, and it is not answered here.",
+      "iso6393": [
+        "afr",
+        "ara",
+        "amh",
+        "eng",
+        "fra",
+        "hau",
+        "ibo",
+        "mlg",
+        "nya",
+        "orm",
+        "pcm",
+        "kin",
+        "run",
+        "sna",
+        "som",
+        "sot",
+        "swa",
+        "xho",
+        "yor",
+        "zul"
+      ],
+      "count": 20,
+      "pivots": [
+        "eng",
+        "fra"
+      ],
+      "pivotNote": "eng and fra appear in the publisher's language list as the evaluation PIVOTS its African-language pairs run through, not as African targets. Membership stays faithful to the publisher; the pivot marking is what keeps 'has AfriCOMET' meaning what the model is FOR — recommending africomet-mtl for French because French is a pivot would route a high-resource language away from the default COMET its own validation covers.",
+      "qeModel": "masakhane/africomet-qe-stl",
+      "qeNote": "Reference-free QE sibling (AfriCOMET-QE-STL), same publisher, same language list, same citation — relocated from the per-card `qe` variant every legacy card stamped with africomet-wan-2022."
+    },
+    {
+      "key": "xlmr",
+      "label": "XLM-R top-tier training languages",
+      "model": "xlm-roberta-large",
+      "purpose": "COMET reliability signal: languages with high XLM-R representation produce more reliable COMET scores",
+      "citation": "Conneau et al. 2020, Unsupervised Cross-lingual Representation Learning at Scale (XLM-R) — top-100 languages by CommonCrawl training volume",
+      "tierMeaning": "membership in this list IS the 'high' tier; the paper publishes the list, not per-language scores",
+      "relocationNote": "2026-08-11: transcribed cite-only onto cards long ago (source stamp 'xlmr-conneau-2020' on every entry) and relocated here, the register such transcriptions belong in — the same move as DeepL formality.",
+      "iso6393": [
+        "afr",
+        "als",
+        "arb",
+        "ast",
+        "aze",
+        "ben",
+        "bos",
+        "bul",
+        "cat",
+        "ces",
+        "cmn",
+        "cym",
+        "dan",
+        "deu",
+        "ell",
+        "eng",
+        "est",
+        "eus",
+        "fin",
+        "fra",
+        "gle",
+        "glg",
+        "guj",
+        "heb",
+        "hin",
+        "hrv",
+        "hun",
+        "hye",
+        "ind",
+        "isl",
+        "ita",
+        "jpn",
+        "kan",
+        "kat",
+        "kaz",
+        "khm",
+        "kir",
+        "kor",
+        "lao",
+        "lat",
+        "lav",
+        "lit",
+        "mal",
+        "mar",
+        "mkd",
+        "mlt",
+        "mon",
+        "mya",
+        "nep",
+        "nld",
+        "nno",
+        "nob",
+        "oci",
+        "pan",
+        "pes",
+        "pol",
+        "por",
+        "ron",
+        "rus",
+        "sin",
+        "slk",
+        "slv",
+        "spa",
+        "srp",
+        "swe",
+        "swh",
+        "tam",
+        "tel",
+        "tgl",
+        "tha",
+        "tur",
+        "ukr",
+        "urd",
+        "uzb",
+        "vie",
+        "zsm"
+      ],
+      "count": 76
+    }
+  ],
+  "unresolved": {
+    "xlmr": {
+      "whatItWasFor": "arena/mt_eval_harness/language_cards.py::is_xlmr_high_resource, read by metrics_comet.py and metrics_metricx.py to flag a score as potentially noisy because the underlying embeddings are weak for that language.",
+      "published": "XLM-RoBERTa's model card states it was pre-trained on 'filtered CommonCrawl data containing 100 languages' (Conneau et al., arXiv:1911.02116). Checked 2026-08-05.",
+      "whyUnresolved": "The model card gives the COUNT and not the LIST. The 100 language codes are in the paper's appendix, which is not a source that can be fetched and pinned the way a model card can. Transcribing them by hand is exactly how the previous register acquired a 'Top-100' note over a list of 77.",
+      "failSafe": "Until the list is pinned, is_xlmr_high_resource returns False for every language, so every score carries the low-resource caveat. That OVER-warns rather than under-warns, which is the only safe direction for a reliability flag: a caveat on a reliable score costs a reader nothing, and a missing caveat on an unreliable one costs them their conclusion.",
+      "theFix": "A fetcher against a citable enumeration of CC-100 / XLM-R's language set, pinned like any other source."
+    }
+  }
+}