cld 0.1.0

data/encodings/public/encodings.h

@@ -0,0 +1,301 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ENCODINGS_PUBLIC_ENCODINGS_H_
+#define ENCODINGS_PUBLIC_ENCODINGS_H_
+
+// This interface defines the Encoding enum and various functions that
+// depend only on Encoding values.
+
+// A hash-function for Encoding, hash<Encoding>, is defined in
+// i18n/encodings/public/encodings-hash.h
+
+// On some Windows projects, UNICODE may be defined, which would prevent the
+// Encoding enum below from compiling. Note that this is a quick fix that does
+// not break any existing projects. The UNICODE enum may someday be changed
+// to something more specific and non-colliding, but this involves careful
+// testing of changes in many other projects.
+#undef UNICODE
+
+// NOTE: The Encoding enum must always start at 0. This assumption has
+// been made and used.
+
+#ifndef SWIG
+
+#include "encodings/proto/encodings.pb.h"
+
+// We must have this for compatibility.
+// COMMENTED OUT TO REDUCE DEPENDENCIES ON GOOGLE3 CODE
+//using namespace i18n::encodings;
+
+#else
+
+// Special proto SWIG workaround header file.
+#include "i18n/encodings/internal/encodings_proto_wrapper.h"
+
+#endif
+
+const int kNumEncodings = NUM_ENCODINGS;
+
+// some of the popular encoding aliases
+// TODO(jrm) Make these static const Encoding values instead of macros.
+#define LATIN1           ISO_8859_1
+#define LATIN2           ISO_8859_2
+#define LATIN3           ISO_8859_3
+#define LATIN4           ISO_8859_4
+#define CYRILLIC         ISO_8859_5
+#define ARABIC_ENCODING  ISO_8859_6     // avoiding the same name as language
+#define GREEK_ENCODING   ISO_8859_7     // avoiding the same name as language
+#define HEBREW_ENCODING  ISO_8859_8     // avoiding the same name as language
+#define LATIN5           ISO_8859_9
+#define LATIN6           ISO_8859_10
+#define KOREAN_HANGUL    KOREAN_EUC_KR
+
+// The default Encoding (LATIN1).
+Encoding default_encoding();
+
+
+
+// *************************************************************
+// Encoding predicates
+//   IsValidEncoding()
+//   IsEncEncCompatible
+//   IsSupersetOfAscii7Bit
+//   Is8BitEncoding
+//   IsCJKEncoding
+//   IsHebrewEncoding
+//   IsRightToLeftEncoding
+//   IsLogicalRightToLeftEncoding
+//   IsVisualRightToLeftEncoding
+//   IsIso2022Encoding
+//   IsIso2022JpOrVariant
+//   IsShiftJisOrVariant
+//   IsJapaneseCellPhoneCarrierSpecificEncoding
+// *************************************************************
+
+// IsValidEncoding
+// ===================================
+//
+// Function to check if the input language enum is within range.
+//
+
+bool IsValidEncoding(Encoding enc);
+
+//
+// IsEncEncCompatible
+// ------------------
+//
+// This function is to determine whether or not converting from the
+// first encoding to the second requires any changes to the underlying
+// text (e.g.  ASCII_7BIT is a subset of UTF8).
+//
+// TODO(someone more familiar with i18n): the current implementation
+// is likely incomplete.  It would be good to consider the full matrix
+// of all pairs of encodings and to fish out all compatible pairs.
+//
+bool IsEncEncCompatible(const Encoding from, const Encoding to);
+
+// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
+// encoding represent the same characters as they do in ISO_8859_1.
+
+// WARNING: This function does not currently return true for all encodings that
+// are supersets of Ascii 7-bit.
+bool IsSupersetOfAscii7Bit(Encoding e);
+
+// To be an 8-bit encoding means that there are fewer than 256 symbols.
+// Each byte determines a new character; there are no multi-byte sequences.
+
+// WARNING: This function does not currently return true for all encodings that
+// are 8-bit encodings.
+bool Is8BitEncoding(Encoding e);
+
+// IsCJKEncoding
+// -------------
+//
+// This function returns true if the encoding is either Chinese
+// (simplified or traditional), Japanese, or Korean. Note: UTF8 is not
+// considered a CJK encoding.
+bool IsCJKEncoding(Encoding e);
+
+// IsHebrewEncoding
+// -------------
+//
+// This function returns true if the encoding is a Hebrew specific
+// encoding (not UTF8, etc).
+bool IsHebrewEncoding(Encoding e);
+
+// IsRightToLeftEncoding
+// ---------------------
+//
+// Returns true if the encoding is a right-to-left encoding.
+//
+// Note that the name of this function is somewhat misleading. There is nothing
+// "right to left" about these encodings. They merely contain code points for
+// characters in RTL languages such as Hebrew and Arabic. But this is also
+// true for UTF-8.
+//
+// TODO(benjy): Get rid of this function. The only special-case we
+// should need to worry about are visual encodings. Anything we
+// need to do for all 'RTL' encodings we need to do for UTF-8 as well.
+bool IsRightToLeftEncoding(Encoding enc);
+
+// IsLogicalRightToLeftEncoding
+// ----------------------------
+//
+// Returns true if the encoding is a logical right-to-left encoding.
+// Logical right-to-left encodings are those that the browser renders
+// right-to-left and applies the BiDi algorithm to. Therefore the characters
+// appear in reading order in the file, and indexing, snippet generation etc.
+// should all just work with no special processing.
+//
+// TODO(benjy): Get rid of this function. The only special-case we
+// should need to worry about are visual encodings.
+bool IsLogicalRightToLeftEncoding(Encoding enc);
+
+// IsVisualRightToLeftEncoding
+// ---------------------------
+//
+// Returns true if the encoding is a visual right-to-left encoding.
+// Visual right-to-left encodings are those that the browser renders
+// left-to-right and does not apply the BiDi algorithm to. Therefore each
+// line appears in reverse order in the file, lines are manually wrapped
+// by abusing <br> or <p> tags, etc. Visual RTL encoding is a relic of
+// the prehistoric days when browsers couldn't render right-to-left, but
+// unfortunately some visual pages persist to this day. These documents require
+// special processing so that we don't index or snippet them with each line
+// reversed.
+bool IsVisualRightToLeftEncoding(Encoding enc);
+
+// IsIso2022Encoding
+// -----------------
+//
+// Returns true if the encoding is a kind of ISO 2022 such as
+// ISO-2022-JP.
+bool IsIso2022Encoding(Encoding enc);
+
+// IsIso2022JpOrVariant
+// --------------------
+//
+// Returns true if the encoding is ISO-2022-JP or a variant such as
+// KDDI's ISO-2022-JP.
+bool IsIso2022JpOrVariant(Encoding enc);
+
+// IsShiftJisOrVariant
+// --------------------
+//
+// Returns true if the encoding is Shift_JIS or a variant such as
+// KDDI's Shift_JIS.
+bool IsShiftJisOrVariant(Encoding enc);
+
+// IsJapanesCellPhoneCarrierSpecificEncoding
+// -----------------------------------------
+//
+// Returns true if it's Japanese cell phone carrier specific encoding
+// such as KDDI_SHIFT_JIS.
+bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc);
+
+
+
+// *************************************************************
+// ENCODING NAMES
+//
+// This interface defines a standard name for each valid encoding, and
+// a standard name for invalid encodings. (Some names use all upper
+// case, but others use mixed case.)
+//
+//   EncodingName() [Encoding to name]
+//   MimeEncodingName() [Encoding to name]
+//   EncodingFromName() [name to Encoding]
+//   EncodingNameAliasToEncoding() [name to Encoding]
+//   default_encoding_name()
+//   invalid_encoding_name()
+// *************************************************************
+
+// EncodingName
+// ------------
+//
+// Given the encoding, returns its standard name.
+// Return invalid_encoding_name() if the encoding is invalid.
+//
+const char* EncodingName(Encoding enc);
+
+//
+// MimeEncodingName
+// ----------------
+//
+// Return the "preferred MIME name" of an encoding.
+//
+// This name is suitable for using in HTTP headers, HTML tags,
+// and as the "charset" parameter of a MIME Content-Type.
+const char* MimeEncodingName(Encoding enc);
+
+
+// The maximum length of an encoding name
+const int kMaxEncodingNameSize = 50;
+
+// The standard name of the default encoding.
+const char* default_encoding_name();
+
+// The name used for an invalid encoding.
+const char* invalid_encoding_name();
+
+// EncodingFromName
+// ----------------
+//
+// If enc_name matches the standard name of an Encoding, using a
+// case-insensitive comparison, set *encoding to that Encoding and
+// return true.  Otherwise set *encoding to UNKNOWN_ENCODING and
+// return false.
+//
+// REQUIRES: encoding must not be NULL.
+//
+bool EncodingFromName(const char* enc_name, Encoding *encoding);
+
+//
+// EncodingNameAliasToEncoding
+// ---------------------------
+//
+// If enc_name matches the standard name or an alias of an Encoding,
+// using a case-insensitive comparison, return that
+// Encoding. Otherwise, return UNKNOWN_ENCODING.
+//
+// Aliases include most mime-encoding names (e.g., "ISO-8859-7" for
+// GREEK), alternate names (e.g., "cyrillic" for ISO_8859_5) and
+// common variations with hyphens and underscores (e.g., "koi8-u" and
+// "koi8u" for RUSSIAN_KOI8_R).
+
+Encoding EncodingNameAliasToEncoding(const char *enc_name);
+
+
+// *************************************************************
+// Miscellany
+// *************************************************************
+
+// PreferredWebOutputEncoding
+// --------------------------
+//
+// Some multi-byte encodings use byte values that coincide with the
+// ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
+// can misinterpret these, as indicated in an external XSS report from
+// 2007-02-15. Here, we map these dangerous encodings to safer ones. We
+// also use UTF8 instead of encodings that we don't support in our
+// output, and we generally try to be conservative in what we send out.
+// Where the client asks for single- or double-byte encodings that are
+// not as common, we substitute a more common single- or double-byte
+// encoding, if there is one, thereby preserving the client's intent
+// to use less space than UTF-8. This also means that characters
+// outside the destination set will be converted to HTML NCRs (&#NNN;)
+// if requested.
+Encoding PreferredWebOutputEncoding(Encoding enc);
+
+
+// InitEncodings
+// -------------
+//
+// Ensures the encodings module has been initialized.  Normally this happens
+// during InitGoogle, but this allows access for scripts that don't
+// support InitGoogle.
+void InitEncodings();
+
+#endif  // ENCODINGS_PUBLIC_ENCODINGS_H_

data/ext/cld/extconf.rb

@@ -0,0 +1,7 @@
+require "rake"
+home_dir = File.expand_path(File.join(File.dirname(__FILE__), "../../"))
+puts home_dir
+cmd = "cd #{home_dir}; ./build.sh"
+sh cmd
+sh "mv #{home_dir}/cld.so #{home_dir}/ext/cld/"
+sh "echo 'install:\n\tdate' > #{home_dir}/ext/cld/Makefile"

data/languages/internal/#languages.cc#

@@ -0,0 +1,337 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "languages/public/languages.h"
+
+#include "base/string_util.h"
+#include "encodings/compact_lang_det/win/cld_basictypes.h"
+
+
+Language default_language() {return ENGLISH;}
+
+
+// Language names and codes
+
+struct LanguageInfo {
+  const char * language_name_;
+  const char * language_code_639_1_;   // the ISO-639-1 code for the language
+  const char * language_code_639_2_;   // the ISO-639-2 code for the language
+  const char * language_code_other_;   // some nonstandard code for the language
+};
+
+static const LanguageInfo kLanguageInfoTable[] = {
+  { "ENGLISH",             "en", "eng", NULL},
+  { "DANISH",              "da", "dan", NULL},
+  { "DUTCH",               "nl", "dut", NULL},
+  { "FINNISH",             "fi", "fin", NULL},
+  { "FRENCH",              "fr", "fre", NULL},
+  { "GERMAN",              "de", "ger", NULL},
+  { "HEBREW",              "he", "heb", NULL},
+  { "ITALIAN",             "it", "ita", NULL},
+  { "Japanese",            "ja", "jpn", NULL},
+  { "Korean",              "ko", "kor", NULL},
+  { "NORWEGIAN",           "nb", "nor", NULL},
+  { "POLISH",              "pl", "pol", NULL},
+  { "PORTUGUESE",          "pt", "por", NULL},
+  { "RUSSIAN",             "ru", "rus", NULL},
+  { "SPANISH",             "es", "spa", NULL},
+  { "SWEDISH",             "sv", "swe", NULL},
+  { "Chinese",             "zh", "chi", "zh-CN"},
+  { "CZECH",               "cs", "cze", NULL},
+  { "GREEK",               "el", "gre", NULL},
+  { "ICELANDIC",           "is", "ice", NULL},
+  { "LATVIAN",             "lv", "lav", NULL},
+  { "LITHUANIAN",          "lt", "lit", NULL},
+  { "ROMANIAN",            "ro", "rum", NULL},
+  { "HUNGARIAN",           "hu", "hun", NULL},
+  { "ESTONIAN",            "et", "est", NULL},
+  // TODO: Although Teragram has two output names "TG_UNKNOWN_LANGUAGE"
+  // and "Unknown", they are essentially the same. Need to unify them.
+  // "un" and "ut" are invented by us, not from ISO-639.
+  //
+  { "TG_UNKNOWN_LANGUAGE", NULL, NULL, "ut"},
+  { "Unknown",             NULL, NULL, "un"},
+  { "BULGARIAN",           "bg", "bul", NULL},
+  { "CROATIAN",            "hr", "scr", NULL},
+  { "SERBIAN",             "sr", "scc", NULL},
+  { "IRISH",               "ga", "gle", NULL},
+  { "GALICIAN",            "gl", "glg", NULL},
+  // Impossible to tell Tagalog from Filipino at the moment.
+  // Use ISO 639-2 code for Filipino here.
+  { "TAGALOG",             NULL, "fil", NULL},
+  { "TURKISH",             "tr", "tur", NULL},
+  { "UKRAINIAN",           "uk", "ukr", NULL},
+  { "HINDI",               "hi", "hin", NULL},
+  { "MACEDONIAN",          "mk", "mac", NULL},
+  { "BENGALI",             "bn", "ben", NULL},
+  { "INDONESIAN",          "id", "ind", NULL},
+  { "LATIN",               "la", "lat", NULL},
+  { "MALAY",               "ms", "may", NULL},
+  { "MALAYALAM",           "ml", "mal", NULL},
+  { "WELSH",               "cy", "wel", NULL},
+  { "NEPALI",              "ne", "nep", NULL},
+  { "TELUGU",              "te", "tel", NULL},
+  { "ALBANIAN",            "sq", "alb", NULL},
+  { "TAMIL",               "ta", "tam", NULL},
+  { "BELARUSIAN",          "be", "bel", NULL},
+  { "JAVANESE",            "jw", "jav", NULL},
+  { "OCCITAN",             "oc", "oci", NULL},
+  { "URDU",                "ur", "urd", NULL},
+  { "BIHARI",              "bh", "bih", NULL},
+  { "GUJARATI",            "gu", "guj", NULL},
+  { "THAI",                "th", "tha", NULL},
+  { "ARABIC",              "ar", "ara", NULL},
+  { "CATALAN",             "ca", "cat", NULL},
+  { "ESPERANTO",           "eo", "epo", NULL},
+  { "BASQUE",              "eu", "baq", NULL},
+  { "INTERLINGUA",         "ia", "ina", NULL},
+  { "KANNADA",             "kn", "kan", NULL},
+  { "PUNJABI",             "pa", "pan", NULL},
+  { "SCOTS_GAELIC",        "gd", "gla", NULL},
+  { "SWAHILI",             "sw", "swa", NULL},
+  { "SLOVENIAN",           "sl", "slv", NULL},
+  { "MARATHI",             "mr", "mar", NULL},
+  { "MALTESE",             "mt", "mlt", NULL},
+  { "VIETNAMESE",          "vi", "vie", NULL},
+  { "FRISIAN",             "fy", "fry", NULL},
+  { "SLOVAK",              "sk", "slo", NULL},
+  { "ChineseT",
+    NULL,  NULL,  // We intentionally set these 2 fields to NULL to avoid
+                  // confusion between CHINESE_T and CHINESE.
+    "zh-TW"},
+  { "FAROESE",             "fo", "fao", NULL},
+  { "SUNDANESE",           "su", "sun", NULL},
+  { "UZBEK",               "uz", "uzb", NULL},
+  { "AMHARIC",             "am", "amh", NULL},
+  { "AZERBAIJANI",         "az", "aze", NULL},
+  { "GEORGIAN",            "ka", "geo", NULL},
+  { "TIGRINYA",            "ti", "tir", NULL},
+  { "PERSIAN",             "fa", "per", NULL},
+  { "BOSNIAN",             "bs", "bos", NULL},
+  { "SINHALESE",           "si", "sin", NULL},
+  { "NORWEGIAN_N",         "nn", "nno", NULL},
+  { "PORTUGUESE_P",        NULL, NULL, "pt-PT"},
+  { "PORTUGUESE_B",        NULL, NULL, "pt-BR"},
+  { "XHOSA",               "xh", "xho", NULL},
+  { "ZULU",                "zu", "zul", NULL},
+  { "GUARANI",             "gn", "grn", NULL},
+  { "SESOTHO",             "st", "sot", NULL},
+  { "TURKMEN",             "tk", "tuk", NULL},
+  { "KYRGYZ",              "ky", "kir", NULL},
+  { "BRETON",              "br", "bre", NULL},
+  { "TWI",                 "tw", "twi", NULL},
+  { "YIDDISH",             "yi", "yid", NULL},
+  { "SERBO_CROATIAN",      "sh", NULL, NULL},
+  { "SOMALI",              "so", "som", NULL},
+  { "UIGHUR",              "ug", "uig", NULL},
+  { "KURDISH",             "ku", "kur", NULL},
+  { "MONGOLIAN",           "mn", "mon", NULL},
+  { "ARMENIAN",            "hy", "arm", NULL},
+  { "LAOTHIAN",            "lo", "lao", NULL},
+  { "SINDHI",              "sd", "snd", NULL},
+  { "RHAETO_ROMANCE",      "rm", "roh", NULL},
+  { "AFRIKAANS",           "af", "afr", NULL},
+  { "LUXEMBOURGISH",       "lb", "ltz", NULL},
+  { "BURMESE",             "my", "bur", NULL},
+  // KHMER is known as Cambodian for Google user interfaces.
+  { "KHMER",               "km", "khm", NULL},
+  { "TIBETAN",             "bo", "tib", NULL},
+  { "DHIVEHI",             "dv", "div", NULL},
+  { "CHEROKEE",            NULL, "chr", NULL},
+  { "SYRIAC",              NULL, "syr", NULL},
+  { "LIMBU",               NULL, NULL, "sit-NP"},
+  { "ORIYA",               "or", "ori", NULL},
+  { "ASSAMESE",            "as", "asm", NULL},
+  { "CORSICAN",            "co", "cos", NULL},
+  { "INTERLINGUE",         "ie", "ine", NULL},
+  { "KAZAKH",              "kk", "kaz", NULL},
+  { "LINGALA",             "ln", "lin", NULL},
+  { "MOLDAVIAN",           "mo", "mol", NULL},
+  { "PASHTO",              "ps", "pus", NULL},
+  { "QUECHUA",             "qu", "que", NULL},
+  { "SHONA",               "sn", "sna", NULL},
+  { "TAJIK",               "tg", "tgk", NULL},
+  { "TATAR",               "tt", "tat", NULL},
+  { "TONGA",               "to", "tog", NULL},
+  { "YORUBA",              "yo", "yor", NULL},
+  { "CREOLES_AND_PIDGINS_ENGLISH_BASED", NULL, "cpe", NULL},
+  { "CREOLES_AND_PIDGINS_FRENCH_BASED",  NULL, "cpf", NULL},
+  { "CREOLES_AND_PIDGINS_PORTUGUESE_BASED", NULL, "cpp", NULL},
+  { "CREOLES_AND_PIDGINS_OTHER", NULL, "crp", NULL},
+  { "MAORI",               "mi", "mao", NULL},
+  { "WOLOF",               "wo", "wol", NULL},
+  { "ABKHAZIAN",           "ab", "abk", NULL},
+  { "AFAR",                "aa", "aar", NULL},
+  { "AYMARA",              "ay", "aym", NULL},
+  { "BASHKIR",             "ba", "bak", NULL},
+  { "BISLAMA",             "bi", "bis", NULL},
+  { "DZONGKHA",            "dz", "dzo", NULL},
+  { "FIJIAN",              "fj", "fij", NULL},
+  { "GREENLANDIC",         "kl", "kal", NULL},
+  { "HAUSA",               "ha", "hau", NULL},
+  { "HAITIAN_CREOLE",       "ht", NULL, NULL},
+  { "INUPIAK",             "ik", "ipk", NULL},
+  { "INUKTITUT",           "iu", "iku", NULL},
+  { "KASHMIRI",            "ks", "kas", NULL},
+  { "KINYARWANDA",         "rw", "kin", NULL},
+  { "MALAGASY",            "mg", "mlg", NULL},
+  { "NAURU",               "na", "nau", NULL},
+  { "OROMO",               "om", "orm", NULL},
+  { "RUNDI",               "rn", "run", NULL},
+  { "SAMOAN",              "sm", "smo", NULL},
+  { "SANGO",               "sg", "sag", NULL},
+  { "SANSKRIT",            "sa", "san", NULL},
+  { "SISWANT",             "ss", "ssw", NULL},
+  { "TSONGA",              "ts", "tso", NULL},
+  { "TSWANA",              "tn", "tsn", NULL},
+  { "VOLAPUK",             "vo", "vol", NULL},
+  { "ZHUANG",              "za", "zha", NULL},
+  { "KHASI",               NULL, "kha", NULL},
+  { "SCOTS",               NULL, "sco", NULL},
+  { "GANDA",               "lg", "lug", NULL},
+  { "MANX",                "gv", "glv", NULL},
+  { "MONTENEGRIN",         NULL, NULL, "sr-ME"},
+  { "XX",                  NULL, NULL, "XX"},
+};
+
+COMPILE_ASSERT(arraysize(kLanguageInfoTable) == NUM_LANGUAGES + 1,
+               kLanguageInfoTable_has_incorrect_length);
+
+
+// LANGUAGE NAMES
+
+const char* default_language_name() {
+  return kLanguageInfoTable[ENGLISH].language_name_;
+}
+
+static const char* const kInvalidLanguageName = "invalid_language";
+
+const char *invalid_language_name() {
+  return kInvalidLanguageName;
+}
+
+const char* LanguageName(Language lang) {
+  return IsValidLanguage(lang)
+      ? kLanguageInfoTable[lang].language_name_
+      : kInvalidLanguageName;
+}
+
+
+
+// LANGUAGE CODES
+
+
+// The space before invalid_language_code is intentional. It is used
+// to prevent it matching any two letter language code.
+//
+static const char* const kInvalidLanguageCode = " invalid_language_code";
+
+const char *invalid_language_code() {
+  return kInvalidLanguageCode;
+}
+
+const char * LanguageCode(Language lang) {
+  if (! IsValidLanguage(lang))
+    return kInvalidLanguageCode;
+  const LanguageInfo& info = kLanguageInfoTable[lang];
+  if (info.language_code_639_1_) {
+    return info.language_code_639_1_;
+  } else if (info.language_code_639_2_) {
+    return info.language_code_639_2_;
+  } else if (info.language_code_other_) {
+    return info.language_code_other_;
+  } else {
+    return kInvalidLanguageCode;
+  }
+}
+
+const char* default_language_code() {
+  return kLanguageInfoTable[ENGLISH].language_code_639_1_;
+}
+
+const char* LanguageCodeISO639_1(Language lang) {
+  if (! IsValidLanguage(lang))
+    return kInvalidLanguageCode;
+  if (const char* code = kLanguageInfoTable[lang].language_code_639_1_)
+    return code;
+  return kInvalidLanguageCode;
+}
+
+const char* LanguageCodeISO639_2(Language lang) {
+  if (! IsValidLanguage(lang))
+    return kInvalidLanguageCode;
+  if (const char* code = kLanguageInfoTable[lang].language_code_639_2_)
+    return code;
+  return kInvalidLanguageCode;
+}
+
+const char* LanguageCodeWithDialects(Language lang) {
+  if (lang == CHINESE)
+    return "zh-CN";
+  return LanguageCode(lang);
+}
+
+
+
+bool LanguageFromCode(const char* lang_code, Language *language) {
+  *language = UNKNOWN_LANGUAGE;
+  if ( lang_code == NULL ) return false;
+
+  for ( int i = 0 ; i < kNumLanguages ; i++ ) {
+    const LanguageInfo& info = kLanguageInfoTable[i];
+    if ((info.language_code_639_1_ &&
+         !base::strcasecmp(lang_code, info.language_code_639_1_)) ||
+        (info.language_code_639_2_ &&
+         !base::strcasecmp(lang_code, info.language_code_639_2_)) ||
+        (info.language_code_other_ &&
+         !base::strcasecmp(lang_code, info.language_code_other_))) {
+      *language = static_cast<Language>(i);
+      return true;
+    }
+  }
+
+  // For convenience, this function can also parse the non-standard
+  // five-letter language codes "zh-cn" and "zh-tw" which are used by
+  // front-ends such as GWS to distinguish Simplified from Traditional
+  // Chinese.
+  if (!base::strcasecmp(lang_code, "zh-cn") ||
+      !base::strcasecmp(lang_code, "zh_cn")) {
+    *language = CHINESE;
+    return true;
+  }
+  if (!base::strcasecmp(lang_code, "zh-tw") ||
+      !base::strcasecmp(lang_code, "zh_tw")) {
+    *language = CHINESE_T;
+    return true;
+  }
+  if (!base::strcasecmp(lang_code, "sr-me") ||
+      !base::strcasecmp(lang_code, "sr_me")) {
+    *language = MONTENEGRIN;
+    return true;
+  }
+
+  // Process language-code synonyms.
+  if (!base::strcasecmp(lang_code, "he")) {
+    *language = HEBREW;  // Use "iw".
+    return true;
+  }
+  if (!base::strcasecmp(lang_code, "in")) {
+    *language = INDONESIAN;  // Use "id".
+    return true;
+  }
+  if (!base::strcasecmp(lang_code, "ji")) {
+    *language = YIDDISH;  // Use "yi".
+    return true;
+  }
+
+  // Process language-detection synonyms.
+  // These distinct languages cannot be differentiated by our current
+  // language-detection algorithms.
+  if (!base::strcasecmp(lang_code, "fil")) {
+    *language = TAGALOG;
+    return true;
+  }
+
+  return false;
+}