cchardet 0.2.0

data/ext/uchardet/script/BuildLangModelLogs/LangTurkishModel.log

@@ -0,0 +1,113 @@
+= Logs of language model for Turkish (tr) =
+
+- Generated by BuildLangModel.py
+- Started: 2015-12-04 02:22:03.929245
+- Maximum depth: 3
+- Max number of pages: 50
+
+== Parsed pages ==
+
+Ana_Sayfa (revision 16293313)
+1048 (revision 12894005)
+1131 (revision 14840814)
+16. yüzyıl (revision 15185081)
+1859 (revision 16014427)
+1866 (revision 16120346)
+1869 (revision 12888270)
+1892 (revision 13955858)
+1895 (revision 15334635)
+1902 (revision 16283638)
+1906 (revision 15874323)
+1918 (revision 16099474)
+1926 (revision 16180584)
+1927 (revision 15370980)
+1940 (revision 15370990)
+1943 (revision 16091797)
+1944 (revision 16247827)
+1945 (revision 16281147)
+1948 (revision 15443886)
+1961 (revision 15799529)
+1964 (revision 16085332)
+1975 (revision 15006928)
+1980 (revision 16213240)
+1981 (revision 16295456)
+1983 (revision 16327128)
+1993 (revision 16300456)
+2002 (revision 16297206)
+2015 (revision 16328338)
+24 Ekim (revision 16213661)
+4 Aralık (revision 16341162)
+ABD (revision 16325951)
+ABD Senatosu (revision 15970439)
+Adam Horowitz (revision 14362106)
+Akçe (revision 16261547)
+Altın Takım (revision 13503001)
+American Broadcasting Company (revision 16055235)
+Amerika Birleşik Devletleri (revision 16325951)
+Ana Sayfa/Kardeş projeler (revision 16293313)
+Ana Sayfa/Kategoriler (revision 16293313)
+Aptullah Kuran (revision 15744893)
+Avrupa (revision 16299756)
+Ayasofya (revision 16305207)
+BM Güvenlik Konseyi (revision 16085518)
+Birleşmiş Milletler (revision 16258474)
+Budapeşte (revision 16219173)
+CIA (revision 16054325)
+Charlie Pace (revision 16129416)
+Cuma (revision 14197127)
+Desmond Hume (revision 16035300)
+Diğerleri (Lost) (revision 16329444)
+
+== End of Parsed pages ==
+
+- Wikipedia parsing ended at: 2015-12-04 02:24:44.728803
+
+48 characters appeared 267623 times.
+
+First 36 characters:
+[ 0] Char a: 12.311722086666691 %
+[ 1] Char e: 8.716365932673948 %
+[ 2] Char i: 8.507863673899479 %
+[ 3] Char n: 7.322987934519828 %
+[ 4] Char r: 6.979220769515326 %
+[ 5] Char l: 6.609297407173524 %
+[ 6] Char ı: 4.514933320379788 %
+[ 7] Char d: 4.3475336574210734 %
+[ 8] Char t: 4.2634601659797555 %
+[ 9] Char k: 4.240293248338147 %
+[10] Char s: 3.929781819948211 %
+[11] Char m: 3.429451130881875 %
+[12] Char u: 3.0998830444319063 %
+[13] Char y: 2.9212735826143494 %
+[14] Char o: 2.7135186437638024 %
+[15] Char b: 2.3129551645411643 %
+[16] Char ü: 1.8305601536489764 %
+[17] Char ş: 1.5988909772328985 %
+[18] Char z: 1.2267256551193282 %
+[19] Char h: 1.1983274980102607 %
+[20] Char v: 1.194964558352608 %
+[21] Char c: 1.143773143563894 %
+[22] Char g: 1.1004285879763698 %
+[23] Char p: 1.0178497363828969 %
+[24] Char ç: 0.8295251155543433 %
+[25] Char ğ: 0.8205572764672693 %
+[26] Char f: 0.7047226882592303 %
+[27] Char ö: 0.6710932916827029 %
+[28] Char j: 0.1296600068006113 %
+[29] Char w: 0.11359262843627041 %
+[30] Char â: 0.07846859201189733 %
+[31] Char î: 0.04147625577771716 %
+[32] Char x: 0.024287897527492032 %
+[33] Char é: 0.014946398478456635 %
+[34] Char q: 0.01083613889688106 %
+[35] Char û: 0.009341499049035397 %
+
+The first 36 characters have an accumulated ratio of 0.99980569681978.
+
+935 sequences found.
+
+First 512 (typical positive ratio): 0.991865243864388
+Next 512 (512-1024): 3.7365996196141585e-06
+Rest: 2.949029909160572e-17
+
+- Processing end: 2015-12-04 02:24:44.883537

data/ext/uchardet/script/BuildLangModelLogs/LangVietnameseModel.log

@@ -0,0 +1,121 @@
+= Logs of language model for Vietnamese (vi) =
+
+- Generated by BuildLangModel.py
+- Started: 2016-02-13 03:37:17.480303
+- Maximum depth: 3
+- Max number of pages: 40
+
+== Parsed pages ==
+
+Chữ_Quốc_ngữ (revision 22887853)
+1651 (revision 21455247)
+1773 (revision 21354755)
+1815 (revision 21361292)
+1838 (revision 21361314)
+1865 (revision 21361338)
+1869 (revision 21361342)
+1888 (revision 21389506)
+1902 (revision 21354811)
+1918 (revision 21354828)
+1919 (revision 21354829)
+1938 (revision 21354849)
+1945 (revision 21354857)
+22 tháng 2 (revision 21376086)
+26 tháng 11 (revision 22579845)
+28 tháng 12 (revision 22475308)
+A (revision 22549334)
+ASCII (revision 22528409)
+Alexandre de Rhodes (revision 22859954)
+Antonio Barbosa (revision 22145269)
+B (revision 22836557)
+BBC (revision 22863903)
+Biên khảo (revision 22531516)
+Bán nguyên âm (revision 22655600)
+Bình luận (revision 22117664)
+Bảng chữ cái Bồ Đào Nha (revision 22887853)
+Bảng chữ cái Hy Lạp (revision 21362081)
+Bảng chữ cái Latinh (revision 22442448)
+Bắc Kỳ (revision 22393289)
+Bồ Đào Nha (revision 22620858)
+C (revision 21341881)
+Cao Xuân Dục (revision 22620201)
+Chính tả (revision 22187359)
+Chính tả tiếng Việt (revision 20897580)
+Chữ Hán (revision 22889609)
+Chữ Nôm (revision 22781506)
+Chữ cái (revision 22169220)
+Công giáo (revision 22173119)
+D (revision 21447691)
+
+== End of Parsed pages ==
+
+- Wikipedia parsing ended at: 2016-02-13 03:42:06.560479
+
+101 characters appeared 222814 times.
+
+First 55 characters:
+[ 0] Char n: 11.262308472537633 %
+[ 1] Char h: 8.881398834902654 %
+[ 2] Char t: 7.022898022565907 %
+[ 3] Char c: 6.365398942615815 %
+[ 4] Char i: 6.198443544840091 %
+[ 5] Char g: 5.591210606155808 %
+[ 6] Char a: 3.5998635633308496 %
+[ 7] Char u: 2.8499106878382867 %
+[ 8] Char m: 2.615185760320267 %
+[ 9] Char o: 2.6012728105056238 %
+[10] Char đ: 2.222032726848403 %
+[11] Char r: 2.1102803234985235 %
+[12] Char à: 2.0447548179198796 %
+[13] Char v: 1.9437737305555307 %
+[14] Char l: 1.9119085874316697 %
+[15] Char á: 1.7539292863105551 %
+[16] Char p: 1.6453185167897888 %
+[17] Char b: 1.541195795596327 %
+[18] Char ư: 1.4397659033992478 %
+[19] Char s: 1.3760356171515256 %
+[20] Char y: 1.280440187779942 %
+[21] Char e: 1.2454334108269678 %
+[22] Char d: 1.1251537156552103 %
+[23] Char ế: 1.071745940560288 %
+[24] Char k: 1.0695019163966357 %
+[25] Char â: 0.9658280000359044 %
+[26] Char ữ: 0.9604423420431392 %
+[27] Char ê: 0.8374698178749989 %
+[28] Char ệ: 0.7459136319979893 %
+[29] Char ô: 0.7073164163831717 %
+[30] Char ạ: 0.6727584442629277 %
+[31] Char ộ: 0.6705144200992756 %
+[32] Char ố: 0.6476253736300233 %
+[33] Char ó: 0.6072329386842837 %
+[34] Char ả: 0.5484395055965963 %
+[35] Char ủ: 0.5475418959311353 %
+[36] Char q: 0.5138815334763525 %
+[37] Char ợ: 0.48560682901433483 %
+[38] Char ờ: 0.4851580241816044 %
+[39] Char ể: 0.4748355130288043 %
+[40] Char ớ: 0.4676546357051173 %
+[41] Char ấ: 0.418286104104769 %
+[42] Char ị: 0.40212913012647317 %
+[43] Char ầ: 0.3904602044754818 %
+[44] Char ọ: 0.3801376933226817 %
+[45] Char ề: 0.3787912788244904 %
+[46] Char ơ: 0.3590438661843511 %
+[47] Char í: 0.35679984202069887 %
+[48] Char ụ: 0.35276059852612496 %
+[49] Char ậ: 0.3469261357006292 %
+[50] Char ì: 0.32762752789322036 %
+[51] Char ă: 0.3253835037295682 %
+[52] Char ứ: 0.29665999443482005 %
+[53] Char ồ: 0.29665999443482005 %
+[54] Char x: 0.2939671654384374 %
+
+The first 55 characters have an accumulated ratio of 0.9603301408349568.
+
+1494 sequences found.
+
+First 512 (typical positive ratio): 0.9321889118082535
+Next 512 (512-1024): 0.009604423420431392
+Rest: 0.0068905733918831966
+
+- Processing end: 2016-02-13 03:42:07.174723

data/ext/uchardet/script/README

@@ -0,0 +1,63 @@
+# Supporting new or Updating languages #
+
+We generate statistical language data using Wikipedia as natural
+language text resource.
+
+Right now, we have automated scripts only to generate statistical data
+for single-byte encodings. Multi-byte encodings usually requires more
+in-depth knowledge of its specification.
+
+## New single-byte encoding ##
+
+Uchardet uses language data, and therefore rather than supporting a
+charset, we in fact support a couple (language, charset). So for
+instance if uchardet supports (French, ISO-8859-15), it should be able
+to recognize French text encoded in ISO-8859-15, but may fail at
+detecting ISO-8859-15 for non-supported languages.
+
+This is why, though less flexible, it also makes uchardet much more
+accurate than other detection system, as well as making it an efficient
+language recognition system.
+Since many single-byte charsets actually share the same layout (or very
+similar ones), it is actually impossible to have an accurate single-byte
+encoding detector for random text.
+
+Therefore you need to describe the language and the codepoint layouts of
+every charset you want to add support for.
+
+I recommend having a look at langs/fr.py which is heavily commented as
+a base of a new language description, and charsets/windows-1252.py as a
+base for a new charset layout (note that charset layouts can be shared
+between languages. If yours is already there, you have nothing to do).
+The important name in the charset file are:
+
+- `name`: an iconv-compatible name.
+- `charmap`: fill it with CTR (control character), SYM (symbol), NUM
+             (number), LET (letter), ILL (illegal codepoint).
+
+## Tools ##
+
+You must install Python 3 and the [`Wikipedia` Python
+tool](https://github.com/goldsmith/Wikipedia).
+
+## Run script ##
+
+Let's say you added (or modified) support for French (`fr`), run:
+
+> ./BuildLangModel.py fr --max-page=100 --max-depth=4
+
+The options can be changed to any value. Bigger values mean the script
+will process more data, so more processing time now, but uchardet may
+possibly be more accurate in the end.
+
+## Updating core code ##
+
+If you were only updating data for a language model, you have nothing
+else to do. Just build `uchardet` again and test it.
+
+If you were creating new models though, you will have to add these in
+src/nsSBCSGroupProber.cpp and src/nsSBCharSetProber.h, and increase the
+value of `NUM_OF_SBCS_PROBERS` in src/nsSBCSGroupProber.h.
+Finally add the new file in src/CMakeLists.txt.
+
+I will be looking to make this step more straightforward in the future.

data/ext/uchardet/script/charsets/codepoints.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# ##### BEGIN LICENSE BLOCK #####
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#          Jehan <jehan@girinstud.io>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ##### END LICENSE BLOCK #####
+
+# Illegal codepoints.
+ILL = 255
+# Control characters
+CTR = 254
+# Symbols and punctuations.
+SYM = 253
+# Return/Line feeds.
+RET = 252
+# Numbers 0-9.
+NUM = 251
+
+# Letters (should be all the rest).
+LET = 0

data/ext/uchardet/script/charsets/db.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# ##### BEGIN LICENSE BLOCK #####
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#          Jehan <jehan@girinstud.io>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ##### END LICENSE BLOCK #####
+
+import importlib
+import sys
+import os
+
+def load(charset_names):
+    '''
+    Load a list of charsets.
+
+    This function will return a dictionary of charsets from our
+    charset database.
+
+    :param charset_names: a list of supported charset names.
+    :return: a dictionary with all the loaded charsets.
+    :rtype: dict
+    '''
+    charsets = {}
+
+    # Temporarily change the search path for modules.
+    sys_path_backup = sys.path
+    current_dir = os.path.dirname(os.path.realpath(__file__))
+    sys.path = [current_dir + '/../charsets']
+
+    for name in charset_names:
+        try:
+            charset = importlib.import_module(name.lower())
+        except ImportError:
+            print('Unknown charset "{}": '
+                  'file "charsets/{}.py" does not exist.'.format(name, name.lower()))
+            exit(1)
+        charsets[charset.name] = charset
+    # Set back the default module paths.
+    sys.path = sys_path_backup
+    return charsets

data/ext/uchardet/script/charsets/ibm852.py

@@ -0,0 +1,72 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# ##### BEGIN LICENSE BLOCK #####
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#          Jehan <jehan@girinstud.io>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ##### END LICENSE BLOCK #####
+
+from codepoints import *
+
+name = 'IBM852'
+aliases = ['CP852']
+
+language = \
+{
+    'complete': [ 'bs', 'hr', 'cs', 'de', 'hu', 'pl', 'sr', 'sk', 'sl',
+                  'hsb', 'dsb', 'tk' ],
+    'incomplete': [ 'ro' ]
+}
+
+#   X0  X1  X2  X3  X4  X5  X6  X7  X8  X9  XA  XB  XC  XD  XE  XF   #
+charmap = \
+[
+    CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, # 0X
+    CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, # 1X
+    SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, # 2X
+    NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, # 3X
+    SYM,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET, # 4X
+    LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,SYM,SYM,SYM,SYM,SYM, # 5X
+    SYM,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET, # 6X
+    LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,SYM,SYM,SYM,SYM,CTR, # 7X
+    LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET, # 8X
+    LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,SYM,LET, # 9X
+    LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,SYM,LET,LET,LET,SYM,SYM, # AX
+    SYM,SYM,SYM,SYM,SYM,LET,LET,LET,LET,SYM,SYM,SYM,SYM,LET,LET,SYM, # BX
+    SYM,SYM,SYM,SYM,SYM,SYM,LET,LET,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, # CX
+    LET,LET,LET,LET,LET,LET,LET,LET,LET,SYM,SYM,SYM,SYM,LET,LET,SYM, # DX
+    LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,LET,SYM, # EX
+    SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,LET,LET,LET,SYM,SYM, # FX
+]