generaltranslation 1.0.3 → 1.0.5

package/README.md

@@ -12,9 +12,9 @@ npm i generaltranslation
 
 ## Toolkit
 
-### getLanguageName
+### getLanguageName(codes)
 
-Returns a language name from a two or three-letter ISO 639 language code.
+Returns a language name from a two or three-letter ISO-639 language code, or an array of codes.
 
 ```
 import { getLanguageName } from 'generaltranslation'
@@ -22,8 +22,48 @@ import { getLanguageName } from 'generaltranslation'
 const language = getLanguageName('en');
 console.log(language) // 'English'
 
-const otherLanguage = getLanguageName('spa') // 'spa' is equivalent to 'es'
-console.log(otherLanguage) // 'Spanish'
+const languages = getLanguageName(['fr', 'es'])
+console.log(languages) // ['French', 'Spanish']
+```
+
+### getLanguageCode(languages)
+
+Returns an ISO-639 code from a language name or an array of language names.
+
+```
+import { getLanguageCode } from 'generaltranslation'
+
+const code = getLanguageCode('English');
+console.log(language) // 'en'
+
+const codes = getLanguageCodes(['French', 'Spanish'])
+console.log(codes) // ['fr', 'es']
+```
+
+### getModelLanguages(model)
+
+<i>in beta!</i>
+
+Get all languages known to be compatible with a given LLM. Returns an array of languages codes, [] if the model is unknown.
+
+```
+import { getModelLanguages } from 'generaltranslation'
+
+const languages = getModelLanguages('mixtral-8x7b')
+console.log(languages) // ['en', 'fr', 'de', 'es', 'it']
+```
+
+### isSupportedLanguage(model, language)
+
+<i>in beta!</i>
+
+Returns true if a model is known to be compatible with a given language, represented by an ISO-639 language code. Returns false otherwise.
+
+```
+import { isSupportedLanguage } from 'generaltranslation'
+
+const supported = isSupportedLanguage('gpt-4', 'fr')
+console.log(supported) // true
 ```
 
 ## API

package/{languages/639-1.js → codes/639-1/CodeToLanguage.json}

@@ -1,4 +1,4 @@
-const CodeToLanguage = {
+{
     "ab": "Abkhazian",
     "aa": "Afar",
     "af": "Afrikaans",
@@ -182,6 +182,4 @@ const CodeToLanguage = {
     "yo": "Yoruba",
     "za": "Zhuang",
     "zu": "Zulu"
-}
-
-module.exports = { CodeToLanguage }
+}

package/codes/639-1/LanguageToCode.json

@@ -0,0 +1,227 @@
+{
+    "abkhazian": "ab",
+    "afar": "aa",
+    "afrikaans": "af",
+    "akan": "ak",
+    "albanian": "sq",
+    "amharic": "am",
+    "arabic": "ar",
+    "aragonese": "an",
+    "armenian": "hy",
+    "assamese": "as",
+    "avaric": "av",
+    "avestan": "ae",
+    "aymara": "ay",
+    "azerbaijani": "az",
+    "bambara": "bm",
+    "bashkir": "ba",
+    "basque": "eu",
+    "belarusian": "be",
+    "bengali": "bn",
+    "bislama": "bi",
+    "bosnian": "bs",
+    "breton": "br",
+    "bulgarian": "bg",
+    "burmese": "my",
+    "catalan": "ca",
+    "valencian": "ca",
+    "chamorro": "ch",
+    "chechen": "ce",
+    "chichewa": "ny",
+    "chewa": "ny",
+    "nyanja": "ny",
+    "chinese": "zh",
+    "mandarin": "zh",
+    "church slavonic": "cu",
+    "old slavonic": "cu",
+    "old church slavonic": "cu",
+    "slavonic": "cu",
+    "chuvash": "cv",
+    "cornish": "kw",
+    "corsican": "co",
+    "cree": "cr",
+    "croatian": "hr",
+    "czech": "cs",
+    "danish": "da",
+    "divehi": "dv",
+    "dhivehi": "dv",
+    "maldivian": "dv",
+    "dutch": "nl",
+    "flemish": "nl",
+    "dzongkha": "dz",
+    "english": "en",
+    "esperanto": "eo",
+    "estonian": "et",
+    "ewe": "ee",
+    "faroese": "fo",
+    "fijian": "fj",
+    "finnish": "fi",
+    "french": "fr",
+    "western frisian": "fy",
+    "frisian": "fy",
+    "fulah": "ff",
+    "gaelic": "gd",
+    "scottish gaelic": "gd",
+    "galician": "gl",
+    "ganda": "lg",
+    "georgian": "ka",
+    "german": "de",
+    "greek": "el",
+    "modern greek": "el",
+    "kalaallisut": "kl",
+    "greenlandic": "kl",
+    "guarani": "gn",
+    "gujarati": "gu",
+    "haitian": "ht",
+    "haitian creole": "ht",
+    "hausa": "ha",
+    "hebrew": "he",
+    "herero": "hz",
+    "hindi": "hi",
+    "hiri motu": "ho",
+    "hungarian": "hu",
+    "icelandic": "is",
+    "ido": "io",
+    "igbo": "ig",
+    "indonesian": "id",
+    "interlingua": "ia",
+    "interlingue": "ie",
+    "occidental": "ie",
+    "inuktitut": "iu",
+    "inupiaq": "ik",
+    "irish": "ga",
+    "italian": "it",
+    "japanese": "ja",
+    "javanese": "jv",
+    "kannada": "kn",
+    "kanuri": "kr",
+    "kashmiri": "ks",
+    "kazakh": "kk",
+    "central khmer": "km",
+    "kikuyu": "ki",
+    "gikuyu": "ki",
+    "kinyarwanda": "rw",
+    "kirghiz": "ky",
+    "kyrgyz": "ky",
+    "komi": "kv",
+    "kongo": "kg",
+    "korean": "ko",
+    "kuanyama": "kj",
+    "kwanyama": "kj",
+    "kurdish": "ku",
+    "lao": "lo",
+    "latin": "la",
+    "latvian": "lv",
+    "limburgan": "li",
+    "limburger": "li",
+    "limburgish": "li",
+    "lingala": "ln",
+    "lithuanian": "lt",
+    "luba-katanga": "lu",
+    "luxembourgish": "lb",
+    "letzeburgesch": "lb",
+    "macedonian": "mk",
+    "malagasy": "mg",
+    "malay": "ms",
+    "malayalam": "ml",
+    "maltese": "mt",
+    "manx": "gv",
+    "maori": "mi",
+    "marathi": "mr",
+    "marshallese": "mh",
+    "mongolian": "mn",
+    "nauru": "na",
+    "navajo": "nv",
+    "navaho": "nv",
+    "north ndebele": "nd",
+    "south ndebele": "nr",
+    "northern ndebele": "nd",
+    "southern ndebele": "nr",
+    "ndonga": "ng",
+    "nepali": "ne",
+    "norwegian": "no",
+    "norwegian bokmål": "nb",
+    "norwegian nynorsk": "nn",
+    "sichuan yi": "ii",
+    "nuosu": "ii",
+    "occitan": "oc",
+    "ojibwa": "oj",
+    "ojibwe": "oj",
+    "oriya": "or",
+    "odia": "or",
+    "oromo": "om",
+    "ossetian": "os",
+    "ossetic": "os",
+    "pali": "pi",
+    "pashto": "ps",
+    "pushto": "ps",
+    "persian": "fa",
+    "polish": "pl",
+    "portuguese": "pt",
+    "punjabi": "pa",
+    "panjabi": "pa",
+    "quechua": "qu",
+    "romanian": "ro",
+    "moldavian": "ro",
+    "moldovan": "ro",
+    "romansh": "rm",
+    "rundi": "rn",
+    "kirundi": "rn",
+    "russian": "ru",
+    "northern sami": "se",
+    "sami": "se",
+    "samoan": "sm",
+    "sango": "sg",
+    "sanskrit": "sa",
+    "sardinian": "sc",
+    "serbian": "sr",
+    "shona": "sn",
+    "sindhi": "sd",
+    "sinhala": "si",
+    "sinhalese": "si",
+    "slovak": "sk",
+    "slovenian": "sl",
+    "slovene": "sl",
+    "somali": "so",
+    "southern sotho": "st",
+    "sotho": "st",
+    "spanish": "es",
+    "castilian": "es",
+    "sundanese": "su",
+    "swahili": "sw",
+    "swati": "ss",
+    "swedish": "sv",
+    "tagalog": "tl",
+    "tahitian": "ty",
+    "tajik": "tg",
+    "tamil": "ta",
+    "tatar": "tt",
+    "telugu": "te",
+    "thai": "th",
+    "tibetan": "bo",
+    "tigrinya": "ti",
+    "tonga": "to",
+    "tongan": "to",
+    "tsonga": "ts",
+    "tswana": "tn",
+    "turkish": "tr",
+    "turkmen": "tk",
+    "twi": "tw",
+    "uighur": "ug",
+    "uyghur": "ug",
+    "ukrainian": "uk",
+    "urdu": "ur",
+    "uzbek": "uz",
+    "venda": "ve",
+    "vietnamese": "vi",
+    "volapük": "vo",
+    "walloon": "wa",
+    "welsh": "cy",
+    "wolof": "wo",
+    "xhosa": "xh",
+    "yiddish": "yi",
+    "yoruba": "yo",
+    "zhuang": "za",
+    "chuang": "za",
+    "zulu": "zu"
+}

package/codes/639-3/CodeToLanguageTriletter.json

@@ -0,0 +1,3 @@
+{
+    "grc": "Ancient Greek"
+}

package/codes/639-3/LanguageToCodeTriletter.json

@@ -0,0 +1,3 @@
+{
+    "ancient greek": "grc"
+}

package/codes/codes.js

@@ -0,0 +1,30 @@
+// ----- IMPORTS ----- //
+
+const CodeToLanguage = require('./639-1/CodeToLanguage.json');
+const LanguageToCode = require('./639-1/LanguageToCode.json');
+
+// only for languages which have no two-letter code
+const CodeToLanguageTriletter = require('./639-3/CodeToLanguageTriletter.json');
+const LanguageToCodeTriletter = require('./639-3/LanguageToCodeTriletter.json');
+
+// ----- LANGUAGE CODES ----- //
+
+// Returns the name of a language from an ISO 639 code or an array of codes
+const _mapCodeToLanguage = code => CodeToLanguage[code] || CodeToLanguageTriletter[code] || '';
+const getLanguageName = codes => {
+    return Array.isArray(codes) ? codes.map(_mapCodeToLanguage) : _mapCodeToLanguage(codes);
+}
+
+// Returns an ISO 639 code from a language name or an array of language names
+// Preferentially returns two-letter codes
+const _mapLanguageToCode = language => {
+    const lowerCaseLanguage = language.toLowerCase();
+    return LanguageToCode[lowerCaseLanguage] || LanguageToCodeTriletter[lowerCaseLanguage] || '';
+}
+const getLanguageCode = languages => {
+    return Array.isArray(languages) ? languages.map(_mapLanguageToCode) : _mapLanguageToCode(languages);
+}
+
+module.exports = {
+    getLanguageName, getLanguageCode
+}

package/index.js

@@ -1,11 +1,15 @@
-const { CodeToLanguage } = require('./languages/639-1.js');
-const { CodeToLanguageTriletter } = require('./languages/639-3.js');
+// `generaltranslation` language toolkit
+// © 2024, General Translation, Inc.
 
-// Gets the name of a language from an ISO 639 code
-function getLanguageName(code) {
-    return ((code?.length === 2) ? (CodeToLanguage[code]) : (CodeToLanguageTriletter[code])) || '';
-}
+// ----- IMPORTS ----- //
 
-module.exports = { 
-    getLanguageName
+const { getLanguageName, getLanguageCode } = require('./codes/codes.js');
+
+const { getModelInfo, getModelLanguages, isLanguageSupported } = require('./models/models.js');
+
+// ----- EXPORTS ----- //
+
+module.exports = {
+    getLanguageName, getLanguageCode,
+    getModelInfo, getModelLanguages, isLanguageSupported
 }

package/models/data/AliasToModel.json

@@ -0,0 +1,28 @@
+{
+    "gpt-4": "gpt-4",
+    "gpt-4-0613": "gpt-4",
+    "gpt-4-1106-preview": "gpt-4",
+    "gpt-4-0125-preview": "gpt-4",
+    "gpt-4-vision-preview": "gpt-4",
+    "gpt-4-turbo-preview": "gpt-4",
+    "gpt-3.5-turbo": "gpt-3.5-turbo",
+    "gpt-3.5-turbo-16k": "gpt-3.5-turbo",
+    "gpt-3.5-turbo-16k-0613": "gpt-3.5-turbo",
+    "gpt-3.5-turbo-0301": "gpt-3.5-turbo",
+    "gpt-3.5-turbo-0613": "gpt-3.5-turbo",
+    "gpt-3.5-turbo-1106": "gpt-3.5-turbo",
+    "gpt-3.5-turbo-0125": "gpt-3.5-turbo",
+    "gpt-3.5-turbo-instruct": "gpt-3.5-turbo",
+    "gpt-3.5-turbo-instruct-0914": "gpt-3.5-turbo",
+    "open-mistral-7b": "mistral-7b",
+    "mixtral": "mixtral-8x7b",
+    "open-mixtral-8x7b": "mixtral-8x7b",
+    "mistral-small-2402": "mistral-small",
+    "mistral-small-latest": "mistral-small",
+    "mistral-medium-2312": "mistral-medium",
+    "mistral-medium-latest": "mistral-medium",
+    "mistral-large-2402": "mistral-large",
+    "mistral-large-latest": "mistral-large",
+    "claude-3-sonnet-20240229": "claude-3-sonnet",
+    "claude-3-opus-20240229": "claude-3-opus"
+}

package/models/data/Models.json

@@ -0,0 +1,44 @@
+{
+    "gpt-4": {
+        "languages": [
+            "en", "fr", "de", "es", "it",
+            "zh", "ja", "ko",
+            "grc", "la"
+        ]
+    },
+    "gpt-3.5-turbo": {
+        "languages": [
+            "en", "fr", "de", "es", "it"
+        ]
+    },
+    "mistral-7b": {
+        "languages": ["en"]
+    },
+    "mixtral-8x7b": {
+        "languages": ["en", "fr", "de", "es", "it"]
+    },
+    "mistral-small": {
+        "languages": ["en", "fr", "de", "es", "it"]
+    },
+    "mistral-medium": {
+        "languages": ["en", "fr", "de", "es", "it"]
+    },
+    "mistral-large": {
+        "languages": ["en", "fr", "de", "es", "it"]
+    },
+    "claude-3-sonnet": {
+        "languages": [
+            "en", "fr", "de", "es", "it"
+        ]
+    },
+    "claude-3-opus": {
+        "languages": [
+            "en", "fr", "de", "es", "it"
+        ]
+    },
+    "gemini-pro": {
+        "languages": [
+            "en", "fr", "de", "es", "it"
+        ]
+    }
+}

package/models/models.js

@@ -0,0 +1,32 @@
+// ----- IMPORTS ----- //
+
+const AliasToModel = require('./data/AliasToModel.json');
+const Models = require('./data/Models.json')
+
+// ----- MODEL INFORMATION ----- //
+
+// Get all info about a given model
+// Returns an object or null if invalid
+const getModelInfo = model => {
+    model = model?.toLowerCase();
+    modelName = AliasToModel[model] ? AliasToModel[model] : model;
+    return Models[modelName];
+}
+
+// Get all languages known to be compatible with a given LLM
+// Returns an array of languages codes, [] if unknown
+const getModelLanguages = model => {
+    return getModelInfo(model)?.languages || [];
+}
+
+// Returns true if a model is known to be compatible with a language
+// Returns false otherwise
+const isLanguageSupported = (model, code) => {
+    return getModelLanguages(model)?.includes(language);
+}
+
+module.exports = {
+    getModelInfo,
+    getModelLanguages,
+    isLanguageSupported
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "generaltranslation",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "A language toolkit for AI developers",
   "main": "index.js",
   "scripts": {

package/languages/639-3.js

@@ -1,188 +0,0 @@
-const CodeToLanguageTriletter = {
-    "abk": "Abkhazian",
-    "aar": "Afar",
-    "afr": "Afrikaans",
-    "aka": "Akan",
-    "sqi": "Albanian",
-    "amh": "Amharic",
-    "ara": "Arabic",
-    "arg": "Aragonese",
-    "hye": "Armenian",
-    "asm": "Assamese",
-    "ava": "Avaric",
-    "ave": "Avestan",
-    "aym": "Aymara",
-    "aze": "Azerbaijani",
-    "bam": "Bambara",
-    "bak": "Bashkir",
-    "eus": "Basque",
-    "bel": "Belarusian",
-    "ben": "Bengali",
-    "bis": "Bislama",
-    "bos": "Bosnian",
-    "bre": "Breton",
-    "bul": "Bulgarian",
-    "mya": "Burmese",
-    "cat": "Catalan",
-    "cha": "Chamorro",
-    "che": "Chechen",
-    "nya": "Chichewa",
-    "zho": "Chinese",
-    "chu": "Slavonic",
-    "chv": "Chuvash",
-    "cor": "Cornish",
-    "cos": "Corsican",
-    "cre": "Cree",
-    "hrv": "Croatian",
-    "ces": "Czech",
-    "dan": "Danish",
-    "div": "Divehi",
-    "nld": "Dutch",
-    "dzo": "Dzongkha",
-    "eng": "English",
-    "epo": "Esperanto",
-    "est": "Estonian",
-    "ewe": "Ewe",
-    "fao": "Faroese",
-    "fij": "Fijian",
-    "fin": "Finnish",
-    "fra": "French",
-    "fry": "Frisian",
-    "ful": "Fulah",
-    "gla": "Gaelic",
-    "glg": "Galician",
-    "lug": "Ganda",
-    "kat": "Georgian",
-    "deu": "German",
-    "ell": "Greek",
-    "kal": "Kalaallisut",
-    "grn": "Guarani",
-    "guj": "Gujarati",
-    "hat": "Haitian Creole",
-    "hau": "Hausa",
-    "heb": "Hebrew",
-    "her": "Herero",
-    "hin": "Hindi",
-    "hmo": "Hiri Motu",
-    "hun": "Hungarian",
-    "isl": "Icelandic",
-    "ido": "Ido",
-    "ibo": "Igbo",
-    "ind": "Indonesian",
-    "ina": "Interlingua",
-    "ile": "Interlingue",
-    "iku": "Inuktitut",
-    "ipk": "Inupiaq",
-    "gle": "Irish",
-    "ita": "Italian",
-    "jpn": "Japanese",
-    "jav": "Javanese",
-    "kan": "Kannada",
-    "kau": "Kanuri",
-    "kas": "Kashmiri",
-    "kaz": "Kazakh",
-    "khm": "Khmer",
-    "kik": "Kikuyu",
-    "kin": "Kinyarwanda",
-    "kir": "Kyrgyz",
-    "kom": "Komi",
-    "kon": "Kongo",
-    "kor": "Korean",
-    "kua": "Kuanyama",
-    "kur": "Kurdish",
-    "lao": "Lao",
-    "lat": "Latin",
-    "lav": "Latvian",
-    "lim": "Limburgan",
-    "lin": "Lingala",
-    "lit": "Lithuanian",
-    "lub": "Luba-Katanga",
-    "ltz": "Luxembourgish",
-    "mkd": "Macedonian",
-    "mlg": "Malagasy",
-    "msa": "Malay",
-    "mal": "Malayalam",
-    "mlt": "Maltese",
-    "glv": "Manx",
-    "mri": "Maori",
-    "mar": "Marathi",
-    "mah": "Marshallese",
-    "mon": "Mongolian",
-    "nau": "Nauru",
-    "nav": "Navajo",
-    "nde": "Northern Ndebele",
-    "nbl": "Southern Ndebele",
-    "ndo": "Ndonga",
-    "nep": "Nepali",
-    "nor": "Norwegian",
-    "nob": "Norwegian Bokmål",
-    "nno": "Norwegian Nynorsk",
-    "iii": "Nuosu",
-    "oci": "Occitan",
-    "oji": "Ojibwa",
-    "ori": "Oriya",
-    "orm": "Oromo",
-    "oss": "Ossetian",
-    "pli": "Pali",
-    "pus": "Pashto",
-    "fas": "Persian",
-    "pol": "Polish",
-    "por": "Portuguese",
-    "pan": "Punjabi",
-    "que": "Quechua",
-    "ron": "Romanian",
-    "roh": "Romansh",
-    "run": "Rundi",
-    "rus": "Russian",
-    "sme": "Northern Sami",
-    "smo": "Samoan",
-    "sag": "Sango",
-    "san": "Sanskrit",
-    "srd": "Sardinian",
-    "srp": "Serbian",
-    "sna": "Shona",
-    "snd": "Sindhi",
-    "sin": "Sinhalese",
-    "slk": "Slovak",
-    "slv": "Slovenian",
-    "som": "Somali",
-    "sot": "Sotho",
-    "spa": "Spanish",
-    "sun": "Sundanese",
-    "swa": "Swahili",
-    "ssw": "Swati",
-    "swe": "Swedish",
-    "tgl": "Tagalog",
-    "tah": "Tahitian",
-    "tgk": "Tajik",
-    "tam": "Tamil",
-    "tat": "Tatar",
-    "tel": "Telugu",
-    "tha": "Thai",
-    "bod": "Tibetan",
-    "tir": "Tigrinya",
-    "ton": "Tonga",
-    "tso": "Tsonga",
-    "tsn": "Tswana",
-    "tur": "Turkish",
-    "tuk": "Turkmen",
-    "twi": "Twi",
-    "uig": "Uyghur",
-    "ukr": "Ukrainian",
-    "urd": "Urdu",
-    "uzb": "Uzbek",
-    "ven": "Venda",
-    "vie": "Vietnamese",
-    "vol": "Volapük",
-    "wln": "Walloon",
-    "cym": "Welsh",
-    "wol": "Wolof",
-    "xho": "Xhosa",
-    "yid": "Yiddish",
-    "yor": "Yoruba",
-    "zha": "Zhuang",
-    "zul": "Zulu",
-    "grc": "Ancient Greek"
-}
-
-module.exports = { CodeToLanguageTriletter }