whatlanguage 1.0.6 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ccd0489249d639e17bda9ae90c496d33e2e6048a
-  data.tar.gz: 83a969f3f186de199996e19bd6c6f391ba7cae24
+SHA256:
+  metadata.gz: 53fea55810c7a8eb102970d6d8c3fe039b031680ff0f23eb6a1c7fec6ad1a602
+  data.tar.gz: f960efb55e0d04b5e457ef53f6c382434a334eea9d44fab2065e7ce9d4376747
 SHA512:
-  metadata.gz: a1fbd9f4745e74637c8eddf8a0c4ac74fed02493d00b011c8e3de80887000253b0d6e39260d54afed9cc1918270c3dad227c963c07591840ad4692ecd775682a
-  data.tar.gz: 59a45845d1f19f073d0f1af57526fcedea5813126f93a7795fe4218842bcf6212d7169f6a34d04d060d587f24c172558c7548ae7d063345f7e5a4800c105f078
+  metadata.gz: 99e1be592f692ba51e3a973a489703f955238da71985feb91b424e9203b8ae29d1ff635d5479ee6cea0226a06cd74c10c4f859a722c76914f277df3d41c3c779
+  data.tar.gz: e13bf56aa67c969d86213e9f1bd34bb2f0aa4ab0ab5db4c1b735ef84c9c3d2ad8e7891f0104a0881833fb5f54c42235132cf48f7a75af38f1cf5f71280529a31

data/CHANGELOG.md

@@ -0,0 +1,47 @@
+# Changelog
+
+## 2.0.1 / 2026-07-11
+
+- Added a `whatlanguage` command-line executable that reads from files given as
+  arguments (or stdin) and prints the ISO 639 code of the detected language, or
+  `und` if undetermined. (#50, suggested by Keith Bennett)
+
+## 2.0.0 / 2026-06-10
+
+- Rewrote the detection engine from scratch. Bloom-filter dictionary lookups are
+  replaced by a script router plus character-trigram out-of-place scoring (a pure-Ruby
+  port of whatlang/Franc, public-domain UDHR-derived profiles).
+- Expanded from 20 to 160+ languages across 20+ scripts.
+- Removed the ~5 MB of binary `.lang` filter files and the wordlist corpus; the model
+  is now a single ~220 KB JSON file. No dependencies beyond the standard library.
+- Requires Ruby 3.0 or newer.
+- Added `#detect`, `#ranked`, and `#score_hash` for callers that need result metadata
+  or runner-up scores. `#scores` and `#process_text` remain as compatibility aliases.
+- Added class-level convenience methods: `WhatLanguage.detect`, `.language`,
+  `.language_iso`, `.ranked`, `.score_hash`, and `.languages`.
+- Added `only:` for configured detectors, replacing positional language
+  selection as the documented API.
+- Removed the `String#language` and `String#language_iso` monkeypatch.
+- Short shared-script fragments now return `nil` by default; pass `min_chars: 0`
+  to preserve the previous best-effort behavior.
+- Removed internals: `BloominSimple`, `BitField`, the filter-build scripts.
+
+## 1.0.6 / 2016-01-28
+
+- Minor test fixes and tweaks
+- New release taking into account a handful of pull requests
+
+## 1.0.5 / 2013-10-05
+
+- Many more languages supported
+
+## 1.0.4 / 2013-03-07
+
+## 1.0.1 / 2008-08-22
+
+- Public release
+- Removed wordlists from distribution to reduce size
+
+## 1.0.0 / 2007-07-02
+
+- First version with pre-built English, French, and Spanish filters

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
 
-# Specify your gem's dependencies in whatlanguage2.gemspec
+source 'https://rubygems.org'
 gemspec

data/LICENSE.txt

@@ -1,10 +1,43 @@
-Copyright (c) 2008-2013 Peter Cooper
-
 MIT License
 
+Ruby code is Copyright (c) 2007-2026 Peter Cooper
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+---
+
+lib/whatlanguage/trigrams.json is licensed as per the https://github.com/greyblake/whatlang-rs project.
+
+Its license is as follows:
+
+(The MIT License)
+
+Copyright (c) 2017 Sergey Potapov <blake131313@gmail.com>
+Copyright (c) 2014 Titus Wormer <tituswormer@gmail.com>
+Copyright (c) 2008 Kent S Johnson
+Copyright (c) 2006 Jacob R Rideout <kde@jacobrideout.net>
+Copyright (c) 2004 Maciej Ceglowski
+
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
+'Software'), to deal in the Software without restriction, including
 without limitation the rights to use, copy, modify, merge, publish,
 distribute, sublicense, and/or sell copies of the Software, and to
 permit persons to whom the Software is furnished to do so, subject to
@@ -13,10 +46,10 @@ the following conditions:
 The above copyright notice and this permission notice shall be
 included in all copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,114 +1,85 @@
 # whatlanguage
 
-by Peter Cooper
+Pure Ruby natural language detection for 160+ languages.
 
-Text language detection. Quick, fast, memory efficient, and all in pure Ruby. Uses Bloom filters for aforementioned speed and memory benefits. It works well on texts of over 10 words in length (e.g. blog posts or comments) and *very poorly* on short or Twitter-esque text, so be aware.
-
-Works with Dutch, English, Farsi, French, German, Italian, Pinyin, Swedish, Portuguese, Russian, Arabic, Finnish, Greek, Hebrew, Hungarian, Korean, Norwegian, Polish and Spanish out of the box.
+```ruby
+require 'whatlanguage'
 
-## Important note
+WhatLanguage.language("Que linguagem é essa? É uma pergunta sobre a língua portuguesa.")
+# => :portuguese
+```
 
-This library was first built in 2007 and has received only a few minor updates over the years. There are now more efficient and effective algorithms for doing language detection which I am investigating for a future WhatLanguage.
+- `gem install whatlanguage`
+- No runtime dependencies.
+- Supports 20+ writing systems.
+- Ships a compact ~220 KB trigram model.
+- Requires Ruby 3.0+ (JRuby and TruffleRuby also good)
+- **Best on sentence-length text or longer.** Short texts can have ambiguous results.
 
-This library has been updated to be distributed and to work on modern Ruby implementations but other than that, has had no significant improvements.
+> [!IMPORTANT]
+> v2.0 has many breaking changes as the entire library has been reimplemented, though the core `WhatLanguage.language` API remains similar. Versions 1.0.6 and earlier (so the 2007-2025 run of the library) used a Bloom-filter technique and had 5MB of binary files to handle ~20 languages. Version 2.0 is more accurate, faster, and supports more languages from a single 220KB JSON file :-)
 
-## Synopsis
+## How it works
 
-Full Example
+Detection is in two stages. First, the dominant Unicode script is detected; scripts used by a single language (Greek, Korean, Thai, Japanese using Hiragana/Katakana) resolve immediately. For scripts shared by several languages (e.g. Latin, Cyrillic, Arabic, Hebrew) trigrams are ranked by frequency and compared against candidate language profiles.
 
-```ruby
-require 'whatlanguage/string'
-
-texts = []
-texts << %q{Deux autres personnes ont été arrêtées durant la nuit}
-texts << %q{The links between the attempted car bombings in Glasgow and London are becoming clearer}
-texts << %q{En estado de máxima alertaen su nivel de crítico}
-texts << %q{Returns the object in enum with the maximum value.}
-texts << %q{Propose des données au sujet de la langue espagnole.}
-texts << %q{La palabra "mezquita" se usa en español para referirse a todo tipo de edificios dedicados.}
-texts << %q{اللغة التي هي هذه؟}
-texts << %q{Mitä kieltä tämä on?}
-texts << %q{Ποια γλώσσα είναι αυτή;}
-texts << %q{באיזו שפה זה?}
-texts << %q{Milyen nyelv ez?}
-texts << %q{이 어떤 언어인가?}
-texts << %q{Hvilket språk er dette?}
-texts << %q{W jakim języku to jest?}
-
-texts.each { |text| puts "#{text[0..18]}... is in #{text.language.to_s.capitalize}" }
-```
+The trigram profiles are vendored from [whatlang](https://github.com/greyblake/whatlang-rs), a port of [Franc](https://github.com/wooorm/franc), whose models are built from the public-domain UDHR corpus (see Credits). The model is a ~220 KB JSON file.
 
-Initialize WhatLanguage with all filters
 
-```ruby
-wl = WhatLanguage.new(:all)
-```
+## Usage
 
-Return language with best score
+Return a full detection result:
 
 ```ruby
-wl.language(text)
+wl = WhatLanguage.new
+text = "Die Stadt plant neue Investitionen in den öffentlichen Verkehr"
+result = wl.detect(text)
+result.language   # => :german
+result.iso        # => :de
+result.score      # => 79018
+result.ranked     # => [[:german, 79018], [:dutch, 77631], ... ]
 ```
 
-Return hash with scores for all relevant languages
+Return ranked scores, or the raw score hash:
 
 ```ruby
-wl.process_text(text)
+wl.ranked(text)       # => [[:german, 79018], [:dutch, 77631], ... ]
+wl.score_hash(text)   # => { german: 79018, dutch: 77631, ... }
 ```
 
-Convenience methods on String
+Restrict candidate languages:
 
 ```ruby
-"This is a test".language   # => :english
-"This is a test".language_iso   # => :en
+wl = WhatLanguage.new(only: [:english, :german, :french])
 ```
 
-Initialize WhatLanguage with certain languages
+Short Latin-script fragments are ignored by default because there is not enough signal to rank shared-script languages reliably. The threshold applies to the statistical trigram stage; scripts that identify a single supported language, such as Greek, Korean, or Thai, can still resolve from shorter text. The threshold can be adjusted:
 
 ```ruby
-wl = WhatLanguage.new(:english, :german, :french)
+wl = WhatLanguage.new(min_chars: 0)
 ```
 
-## Requirements
+## Command line
 
-None, minor libraries (BloominSimple and BitField) included with this release.
+Installing the gem also installs a `whatlanguage` executable that reads from files given as arguments (or stdin) and prints the ISO 639 code of the detected language, or `und` if undetermined:
 
-## Installation
-
-    gem install whatlanguage
-
-To test, go into irb, then:
-
-```ruby
-require 'whatlanguage'
-"Je suis un homme".language
+```bash
+$ whatlanguage README.md
+en
+$ echo "Wie geht es dir heute?" | whatlanguage
+de
 ```
 
-## Credits
-
-Contributions from Konrad Reiche, Salimane Adjao Moustapha, and others appreciated.
-
-## License
-
-MIT License
+## Known limitations
 
-Copyright (c) 2007-2016 Peter Cooper
+- Short fragments are unreliable. For languages resolved by statistical comparison, fewer than 10 significant characters returns `nil` by default. **This is the biggest thing I want to fix.** I think there should be a mechanism for "weighting" languages so that less likely languages don't frequently appear for small English samples, say.
+- Scores are relative ranking values, not probabilities. Use `#ranked` or `#detect.ranked` when close runners-up matter.
+- Closely related written languages can be hard to separate, especially Norwegian Bokmål/Danish, Hebrew/Yiddish, and similar language pairs.
+- Kanji-only Japanese text can classify as Chinese because Han characters alone do not identify the language.
+- Romanized text is classified by Latin-script trigram profiles; it is not treated as native-script text.
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+## Credits
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
+Contributions from Konrad Reiche, Salimane Adjao Moustapha, Andrew Cone, Lasse Skindstad Ebert, Henrik Nyh, Daniel Sandbecker, Michael Hartl, Pedro Lambert, Tobias Preuss, Pepijn Looije, Keith Bennett, and others appreciated.
 
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+The trigram language profiles in `lib/whatlanguage/trigrams.json` are taken from [whatlang](https://github.com/greyblake/whatlang-rs) (MIT, © Sergey Potapov), itself a derivative of [Franc](https://github.com/wooorm/franc) (MIT, © Titus Wormer). Those profiles are derived from the public-domain Universal Declaration of Human Rights translations.

data/Rakefile

@@ -1,6 +1,12 @@
-require "bundler/gem_tasks"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
 require 'rake/testtask'
 
-Rake::TestTask.new
+Rake::TestTask.new do |t|
+  t.libs << 'lib' << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.warning = false
+end
 
-task :default => :test
+task default: :test

data/exe/whatlanguage

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'whatlanguage'
+
+# Reads from files given as arguments, or stdin otherwise. Prints the ISO 639
+# code of the detected language, or "und" (undetermined) if detection fails.
+puts WhatLanguage.language_iso(ARGF.read) || 'und'

data/lib/whatlanguage/languages.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+# AUTO-GENERATED from the ISO 639-3 registry + whatlang dataset codes.
+# Maps ISO 639-3 code => [language name symbol, ISO 639-1 (or 639-3 fallback) symbol].
+# Original gem languages keep their historical symbols and 2-letter codes.
+class WhatLanguage
+  CODE_INFO = {
+    "ace"   => [:achinese,               :ace],
+    "ada"   => [:adangme,                :ada],
+    "afr"   => [:afrikaans,              :af],
+    "aka"   => [:akan,                   :ak],
+    "amh"   => [:amharic,                :am],
+    "ara"   => [:arabic,                 :ar],
+    "hye"   => [:armenian,               :hy],
+    "quy"   => [:ayacucho_quechua,       :quy],
+    "aze"   => [:azerbaijani,            :az],
+    "ban"   => [:balinese,               :ban],
+    "rmn"   => [:balkan_romani,          :rmn],
+    "bam"   => [:bambara,                :bm],
+    "bci"   => [:baoule,                 :bci],
+    "bel"   => [:belarusian,             :be],
+    "bem"   => [:bemba,                  :bem],
+    "ben"   => [:bengali,                :bn],
+    "bho"   => [:bhojpuri,               :bho],
+    "bin"   => [:bini,                   :bin],
+    "bos"   => [:bosnian,                :bs],
+    "bug"   => [:buginese,               :bug],
+    "bul"   => [:bulgarian,              :bg],
+    "mya"   => [:burmese,                :my],
+    "cat"   => [:catalan,                :ca],
+    "ceb"   => [:cebuano,                :ceb],
+    "tzm"   => [:central_atlas_tamazight, :tzm],
+    "ayr"   => [:central_aymara,         :ayr],
+    "bcl"   => [:central_bikol,          :bcl],
+    "knc"   => [:central_kanuri,         :knc],
+    "nhn"   => [:central_nahuatl,        :nhn],
+    "nya"   => [:chichewa,               :ny],
+    "qug"   => [:chimborazo_highland_quichua, :qug],
+    "cmn"   => [:chinese,                :zh],
+    "cjk"   => [:chokwe,                 :cjk],
+    "hrv"   => [:croatian,               :hr],
+    "quz"   => [:cusco_quechua,          :quz],
+    "ces"   => [:czech,                  :cs],
+    "dan"   => [:danish,                 :da],
+    "nld"   => [:dutch,                  :nl],
+    "dyu"   => [:dyula,                  :dyu],
+    "emk"   => [:eastern_maninkakan,     :emk],
+    "eng"   => [:english,                :en],
+    "epo"   => [:esperanto,              :eo],
+    "est"   => [:estonian,               :et],
+    "ewe"   => [:ewe,                    :ee],
+    "pes"   => [:farsi,                  :fa],
+    "fin"   => [:finnish,                :fi],
+    "fon"   => [:fon,                    :fon],
+    "fra"   => [:french,                 :fr],
+    "gaa"   => [:ga,                     :gaa],
+    "glg"   => [:galician,               :gl],
+    "lug"   => [:ganda,                  :lg],
+    "kat"   => [:georgian,               :ka],
+    "deu"   => [:german,                 :de],
+    "ell"   => [:greek,                  :el],
+    "guj"   => [:gujarati,               :gu],
+    "hat"   => [:haitian,                :ht],
+    "khk"   => [:halh_mongolian,         :khk],
+    "hau"   => [:hausa,                  :ha],
+    "heb"   => [:hebrew,                 :he],
+    "hil"   => [:hiligaynon,             :hil],
+    "hin"   => [:hindi,                  :hi],
+    "hun"   => [:hungarian,              :hu],
+    "ibb"   => [:ibibio,                 :ibb],
+    "ibo"   => [:igbo,                   :ig],
+    "ilo"   => [:iloko,                  :ilo],
+    "ind"   => [:indonesian,             :id],
+    "ita"   => [:italian,                :it],
+    "jpn"   => [:japanese,               :ja],
+    "jav"   => [:javanese,               :jv],
+    "kbp"   => [:kabiye,                 :kbp],
+    "kan"   => [:kannada,                :kn],
+    "kaz"   => [:kazakh,                 :kk],
+    "khm"   => [:khmer,                  :km],
+    "kmb"   => [:kimbundu,               :kmb],
+    "kin"   => [:kinyarwanda,            :rw],
+    "kir"   => [:kirghiz,                :ky],
+    "koi"   => [:komi_permyak,           :koi],
+    "kng"   => [:koongo,                 :kng],
+    "kor"   => [:korean,                 :ko],
+    "kur"   => [:kurdish,                :ku],
+    "lat"   => [:latin,                  :la],
+    "lav"   => [:latvian,                :lv],
+    "lin"   => [:lingala,                :ln],
+    "lit"   => [:lithuanian,             :lt],
+    "src"   => [:logudorese_sardinian,   :src],
+    "nds"   => [:low_german,             :nds],
+    "lua"   => [:luba_lulua,             :lua],
+    "lun"   => [:lunda,                  :lun],
+    "mkd"   => [:macedonian,             :mk],
+    "mad"   => [:madurese,               :mad],
+    "mag"   => [:magahi,                 :mag],
+    "mai"   => [:maithili,               :mai],
+    "vmw"   => [:makhuwa,                :vmw],
+    "kde"   => [:makonde,                :kde],
+    "mlg"   => [:malagasy,               :mg],
+    "mal"   => [:malayalam,              :ml],
+    "mar"   => [:marathi,                :mr],
+    "men"   => [:mende,                  :men],
+    "min"   => [:minangkabau,            :min],
+    "mos"   => [:mossi,                  :mos],
+    "ndo"   => [:ndonga,                 :ng],
+    "nep"   => [:nepali,                 :ne],
+    "dip"   => [:northeastern_dinka,     :dip],
+    "uzn"   => [:northern_uzbek,         :uzn],
+    "nob"   => [:norwegian,              :no],
+    "nno"   => [:norwegian_nynorsk,      :nn],
+    "nyn"   => [:nyankole,               :nyn],
+    "ori"   => [:oriya,                  :or],
+    "orm"   => [:oromo,                  :om],
+    "pam"   => [:pampanga,               :pam],
+    "pan"   => [:panjabi,                :pa],
+    "nso"   => [:pedi,                   :nso],
+    "pol"   => [:polish,                 :pl],
+    "por"   => [:portuguese,             :pt],
+    "ron"   => [:romanian,               :ro],
+    "run"   => [:rundi,                  :rn],
+    "rus"   => [:russian,                :ru],
+    "sag"   => [:sango,                  :sg],
+    "skr"   => [:saraiki,                :skr],
+    "srp"   => [:serbian,                :sr],
+    "sna"   => [:shona,                  :sn],
+    "sin"   => [:sinhala,                :si],
+    "snn"   => [:siona,                  :snn],
+    "slk"   => [:slovak,                 :sk],
+    "slv"   => [:slovenian,              :sl],
+    "som"   => [:somali,                 :so],
+    "snk"   => [:soninke,                :snk],
+    "hms"   => [:southern_qiandong_miao, :hms],
+    "sot"   => [:southern_sotho,         :st],
+    "spa"   => [:spanish,                :es],
+    "suk"   => [:sukuma,                 :suk],
+    "sun"   => [:sundanese,              :su],
+    "swh"   => [:swahili,                :swh],
+    "ssw"   => [:swati,                  :ss],
+    "swe"   => [:swedish,                :sv],
+    "tgl"   => [:tagalog,                :tl],
+    "tgk"   => [:tajik,                  :tg],
+    "tam"   => [:tamil,                  :ta],
+    "tat"   => [:tatar,                  :tt],
+    "tel"   => [:telugu,                 :te],
+    "tha"   => [:thai,                   :th],
+    "tir"   => [:tigrinya,               :ti],
+    "tem"   => [:timne,                  :tem],
+    "tiv"   => [:tiv,                    :tiv],
+    "tpi"   => [:tok_pisin,              :tpi],
+    "toi"   => [:tonga,                  :toi],
+    "als"   => [:tosk_albanian,          :als],
+    "tso"   => [:tsonga,                 :ts],
+    "tsn"   => [:tswana,                 :tn],
+    "tur"   => [:turkish,                :tr],
+    "tuk"   => [:turkmen,                :tk],
+    "uig"   => [:uighur,                 :ug],
+    "ukr"   => [:ukrainian,              :uk],
+    "umb"   => [:umbundu,                :umb],
+    "urd"   => [:urdu,                   :ur],
+    "uzb"   => [:uzbek,                  :uz],
+    "vie"   => [:vietnamese,             :vi],
+    "rmy"   => [:vlax_romani,            :rmy],
+    "war"   => [:waray,                  :war],
+    "cym"   => [:welsh,                  :cy],
+    "wol"   => [:wolof,                  :wo],
+    "xho"   => [:xhosa,                  :xh],
+    "yao"   => [:yao,                    :yao],
+    "yid"   => [:yiddish,                :yi],
+    "yor"   => [:yoruba,                 :yo],
+    "zul"   => [:zulu,                   :zu],
+  }.freeze
+
+  # Convenience aliases for legacy selection symbols that no longer map 1:1.
+  NAME_ALIASES = { pinyin: :chinese, mandarin: :chinese }.freeze
+
+  private_constant :CODE_INFO, :NAME_ALIASES
+end