cchardet 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (317) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.gitmodules +3 -0
  4. data/.rubocop.yml +11 -0
  5. data/CHANGELOG.md +5 -0
  6. data/Gemfile +10 -0
  7. data/README.md +35 -0
  8. data/Rakefile +15 -0
  9. data/cchardet.gemspec +30 -0
  10. data/ext/cchardet/extconf.rb +26 -0
  11. data/ext/uchardet/.gitignore +1 -0
  12. data/ext/uchardet/.gitlab-ci.yml +106 -0
  13. data/ext/uchardet/AUTHORS +16 -0
  14. data/ext/uchardet/CMakeLists.txt +74 -0
  15. data/ext/uchardet/COPYING +1316 -0
  16. data/ext/uchardet/INSTALL +26 -0
  17. data/ext/uchardet/README.md +295 -0
  18. data/ext/uchardet/build-mac/uchardet.cpp +7 -0
  19. data/ext/uchardet/build-mac/uchardet.xcodeproj/project.pbxproj +543 -0
  20. data/ext/uchardet/build-mac/uchardet.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -0
  21. data/ext/uchardet/build-mac/uchardet.xcodeproj/project.xcworkspace/xcshareddata/uchardet.xccheckout +41 -0
  22. data/ext/uchardet/build-mac/uchardet.xcodeproj/xcshareddata/xcschemes/uchardet-ios.xcscheme +77 -0
  23. data/ext/uchardet/build-mac/uchardet.xcodeproj/xcshareddata/xcschemes/uchardet.xcscheme +77 -0
  24. data/ext/uchardet/doc/CMakeLists.txt +6 -0
  25. data/ext/uchardet/doc/README.maintainer +59 -0
  26. data/ext/uchardet/doc/uchardet.1 +18 -0
  27. data/ext/uchardet/script/BuildLangModel.py +533 -0
  28. data/ext/uchardet/script/BuildLangModelLogs/LangArabicModel.log +142 -0
  29. data/ext/uchardet/script/BuildLangModelLogs/LangCroatianModel.log +157 -0
  30. data/ext/uchardet/script/BuildLangModelLogs/LangCzechModel.log +161 -0
  31. data/ext/uchardet/script/BuildLangModelLogs/LangDanishModel.log +158 -0
  32. data/ext/uchardet/script/BuildLangModelLogs/LangEsperantoModel.log +110 -0
  33. data/ext/uchardet/script/BuildLangModelLogs/LangEstonianModel.log +159 -0
  34. data/ext/uchardet/script/BuildLangModelLogs/LangFinnishModel.log +156 -0
  35. data/ext/uchardet/script/BuildLangModelLogs/LangFrenchModel.log +116 -0
  36. data/ext/uchardet/script/BuildLangModelLogs/LangGermanModel.log +159 -0
  37. data/ext/uchardet/script/BuildLangModelLogs/LangGreekModel.log +272 -0
  38. data/ext/uchardet/script/BuildLangModelLogs/LangHungarianModel.log +109 -0
  39. data/ext/uchardet/script/BuildLangModelLogs/LangIrishModel.log +156 -0
  40. data/ext/uchardet/script/BuildLangModelLogs/LangItalianModel.log +162 -0
  41. data/ext/uchardet/script/BuildLangModelLogs/LangLatvianModel.log +162 -0
  42. data/ext/uchardet/script/BuildLangModelLogs/LangLithuanianModel.log +162 -0
  43. data/ext/uchardet/script/BuildLangModelLogs/LangMalteseModel.log +147 -0
  44. data/ext/uchardet/script/BuildLangModelLogs/LangPolishModel.log +154 -0
  45. data/ext/uchardet/script/BuildLangModelLogs/LangPortugueseModel.log +166 -0
  46. data/ext/uchardet/script/BuildLangModelLogs/LangRomanianModel.log +153 -0
  47. data/ext/uchardet/script/BuildLangModelLogs/LangSlovakModel.log +158 -0
  48. data/ext/uchardet/script/BuildLangModelLogs/LangSloveneModel.log +148 -0
  49. data/ext/uchardet/script/BuildLangModelLogs/LangSpanishModel.log +109 -0
  50. data/ext/uchardet/script/BuildLangModelLogs/LangSwedishModel.log +151 -0
  51. data/ext/uchardet/script/BuildLangModelLogs/LangThaiModel.log +141 -0
  52. data/ext/uchardet/script/BuildLangModelLogs/LangTurkishModel.log +113 -0
  53. data/ext/uchardet/script/BuildLangModelLogs/LangVietnameseModel.log +121 -0
  54. data/ext/uchardet/script/README +63 -0
  55. data/ext/uchardet/script/charsets/codepoints.py +53 -0
  56. data/ext/uchardet/script/charsets/db.py +73 -0
  57. data/ext/uchardet/script/charsets/ibm852.py +72 -0
  58. data/ext/uchardet/script/charsets/iso-8859-1.py +85 -0
  59. data/ext/uchardet/script/charsets/iso-8859-10.py +73 -0
  60. data/ext/uchardet/script/charsets/iso-8859-11.py +77 -0
  61. data/ext/uchardet/script/charsets/iso-8859-13.py +72 -0
  62. data/ext/uchardet/script/charsets/iso-8859-15.py +80 -0
  63. data/ext/uchardet/script/charsets/iso-8859-16.py +83 -0
  64. data/ext/uchardet/script/charsets/iso-8859-2.py +73 -0
  65. data/ext/uchardet/script/charsets/iso-8859-3.py +75 -0
  66. data/ext/uchardet/script/charsets/iso-8859-4.py +73 -0
  67. data/ext/uchardet/script/charsets/iso-8859-6.py +73 -0
  68. data/ext/uchardet/script/charsets/iso-8859-7.py +73 -0
  69. data/ext/uchardet/script/charsets/iso-8859-9.py +76 -0
  70. data/ext/uchardet/script/charsets/mac-centraleurope.py +72 -0
  71. data/ext/uchardet/script/charsets/tis-620.py +77 -0
  72. data/ext/uchardet/script/charsets/viscii.py +72 -0
  73. data/ext/uchardet/script/charsets/windows-1250.py +75 -0
  74. data/ext/uchardet/script/charsets/windows-1252.py +76 -0
  75. data/ext/uchardet/script/charsets/windows-1253.py +72 -0
  76. data/ext/uchardet/script/charsets/windows-1256.py +75 -0
  77. data/ext/uchardet/script/charsets/windows-1257.py +72 -0
  78. data/ext/uchardet/script/charsets/windows-1258.py +72 -0
  79. data/ext/uchardet/script/debug.sh +9 -0
  80. data/ext/uchardet/script/header-template.cpp +38 -0
  81. data/ext/uchardet/script/langs/ar.py +59 -0
  82. data/ext/uchardet/script/langs/cs.py +80 -0
  83. data/ext/uchardet/script/langs/da.py +69 -0
  84. data/ext/uchardet/script/langs/de.py +69 -0
  85. data/ext/uchardet/script/langs/el.py +55 -0
  86. data/ext/uchardet/script/langs/eo.py +67 -0
  87. data/ext/uchardet/script/langs/es.py +69 -0
  88. data/ext/uchardet/script/langs/et.py +57 -0
  89. data/ext/uchardet/script/langs/fi.py +60 -0
  90. data/ext/uchardet/script/langs/fr.py +79 -0
  91. data/ext/uchardet/script/langs/ga.py +60 -0
  92. data/ext/uchardet/script/langs/hr.py +59 -0
  93. data/ext/uchardet/script/langs/hu.py +66 -0
  94. data/ext/uchardet/script/langs/it.py +56 -0
  95. data/ext/uchardet/script/langs/lt.py +70 -0
  96. data/ext/uchardet/script/langs/lv.py +69 -0
  97. data/ext/uchardet/script/langs/mt.py +80 -0
  98. data/ext/uchardet/script/langs/pl.py +81 -0
  99. data/ext/uchardet/script/langs/pt.py +80 -0
  100. data/ext/uchardet/script/langs/ro.py +65 -0
  101. data/ext/uchardet/script/langs/sk.py +80 -0
  102. data/ext/uchardet/script/langs/sl.py +59 -0
  103. data/ext/uchardet/script/langs/sv.py +56 -0
  104. data/ext/uchardet/script/langs/th.py +55 -0
  105. data/ext/uchardet/script/langs/tr.py +67 -0
  106. data/ext/uchardet/script/langs/vi.py +64 -0
  107. data/ext/uchardet/script/release.sh +8 -0
  108. data/ext/uchardet/script/win32.sh +7 -0
  109. data/ext/uchardet/src/Big5Freq.tab +943 -0
  110. data/ext/uchardet/src/CMakeLists.txt +160 -0
  111. data/ext/uchardet/src/CharDistribution.cpp +109 -0
  112. data/ext/uchardet/src/CharDistribution.h +242 -0
  113. data/ext/uchardet/src/EUCKRFreq.tab +614 -0
  114. data/ext/uchardet/src/EUCTWFreq.tab +447 -0
  115. data/ext/uchardet/src/GB2312Freq.tab +491 -0
  116. data/ext/uchardet/src/JISFreq.tab +589 -0
  117. data/ext/uchardet/src/JpCntx.cpp +230 -0
  118. data/ext/uchardet/src/JpCntx.h +140 -0
  119. data/ext/uchardet/src/LangModels/LangArabicModel.cpp +267 -0
  120. data/ext/uchardet/src/LangModels/LangBulgarianModel.cpp +249 -0
  121. data/ext/uchardet/src/LangModels/LangCroatianModel.cpp +298 -0
  122. data/ext/uchardet/src/LangModels/LangCzechModel.cpp +285 -0
  123. data/ext/uchardet/src/LangModels/LangDanishModel.cpp +201 -0
  124. data/ext/uchardet/src/LangModels/LangEsperantoModel.cpp +142 -0
  125. data/ext/uchardet/src/LangModels/LangEstonianModel.cpp +268 -0
  126. data/ext/uchardet/src/LangModels/LangFinnishModel.cpp +297 -0
  127. data/ext/uchardet/src/LangModels/LangFrenchModel.cpp +209 -0
  128. data/ext/uchardet/src/LangModels/LangGermanModel.cpp +170 -0
  129. data/ext/uchardet/src/LangModels/LangGreekModel.cpp +231 -0
  130. data/ext/uchardet/src/LangModels/LangHebrewModel.cpp +220 -0
  131. data/ext/uchardet/src/LangModels/LangHungarianModel.cpp +171 -0
  132. data/ext/uchardet/src/LangModels/LangIrishModel.cpp +234 -0
  133. data/ext/uchardet/src/LangModels/LangItalianModel.cpp +269 -0
  134. data/ext/uchardet/src/LangModels/LangLatvianModel.cpp +210 -0
  135. data/ext/uchardet/src/LangModels/LangLithuanianModel.cpp +209 -0
  136. data/ext/uchardet/src/LangModels/LangMalteseModel.cpp +138 -0
  137. data/ext/uchardet/src/LangModels/LangPolishModel.cpp +304 -0
  138. data/ext/uchardet/src/LangModels/LangPortugueseModel.cpp +241 -0
  139. data/ext/uchardet/src/LangModels/LangRomanianModel.cpp +236 -0
  140. data/ext/uchardet/src/LangModels/LangRussianModel.cpp +367 -0
  141. data/ext/uchardet/src/LangModels/LangSlovakModel.cpp +293 -0
  142. data/ext/uchardet/src/LangModels/LangSloveneModel.cpp +264 -0
  143. data/ext/uchardet/src/LangModels/LangSpanishModel.cpp +204 -0
  144. data/ext/uchardet/src/LangModels/LangSwedishModel.cpp +266 -0
  145. data/ext/uchardet/src/LangModels/LangThaiModel.cpp +267 -0
  146. data/ext/uchardet/src/LangModels/LangTurkishModel.cpp +175 -0
  147. data/ext/uchardet/src/LangModels/LangVietnameseModel.cpp +249 -0
  148. data/ext/uchardet/src/nsBig5Prober.cpp +88 -0
  149. data/ext/uchardet/src/nsBig5Prober.h +75 -0
  150. data/ext/uchardet/src/nsCharSetProber.cpp +125 -0
  151. data/ext/uchardet/src/nsCharSetProber.h +77 -0
  152. data/ext/uchardet/src/nsCodingStateMachine.h +105 -0
  153. data/ext/uchardet/src/nsEUCJPProber.cpp +99 -0
  154. data/ext/uchardet/src/nsEUCJPProber.h +79 -0
  155. data/ext/uchardet/src/nsEUCKRProber.cpp +91 -0
  156. data/ext/uchardet/src/nsEUCKRProber.h +81 -0
  157. data/ext/uchardet/src/nsEUCTWProber.cpp +91 -0
  158. data/ext/uchardet/src/nsEUCTWProber.h +75 -0
  159. data/ext/uchardet/src/nsEscCharsetProber.cpp +101 -0
  160. data/ext/uchardet/src/nsEscCharsetProber.h +70 -0
  161. data/ext/uchardet/src/nsEscSM.cpp +267 -0
  162. data/ext/uchardet/src/nsGB2312Prober.cpp +96 -0
  163. data/ext/uchardet/src/nsGB2312Prober.h +77 -0
  164. data/ext/uchardet/src/nsHebrewProber.cpp +194 -0
  165. data/ext/uchardet/src/nsHebrewProber.h +177 -0
  166. data/ext/uchardet/src/nsLatin1Prober.cpp +182 -0
  167. data/ext/uchardet/src/nsLatin1Prober.h +73 -0
  168. data/ext/uchardet/src/nsMBCSGroupProber.cpp +242 -0
  169. data/ext/uchardet/src/nsMBCSGroupProber.h +81 -0
  170. data/ext/uchardet/src/nsMBCSSM.cpp +513 -0
  171. data/ext/uchardet/src/nsPkgInt.h +89 -0
  172. data/ext/uchardet/src/nsSBCSGroupProber.cpp +343 -0
  173. data/ext/uchardet/src/nsSBCSGroupProber.h +71 -0
  174. data/ext/uchardet/src/nsSBCharSetProber.cpp +160 -0
  175. data/ext/uchardet/src/nsSBCharSetProber.h +258 -0
  176. data/ext/uchardet/src/nsSJISProber.cpp +98 -0
  177. data/ext/uchardet/src/nsSJISProber.h +81 -0
  178. data/ext/uchardet/src/nsUTF8Prober.cpp +87 -0
  179. data/ext/uchardet/src/nsUTF8Prober.h +66 -0
  180. data/ext/uchardet/src/nsUniversalDetector.cpp +339 -0
  181. data/ext/uchardet/src/nsUniversalDetector.h +91 -0
  182. data/ext/uchardet/src/nscore.h +59 -0
  183. data/ext/uchardet/src/prmem.h +49 -0
  184. data/ext/uchardet/src/symbols.cmake +41 -0
  185. data/ext/uchardet/src/tools/CMakeLists.txt +23 -0
  186. data/ext/uchardet/src/tools/uchardet.cpp +254 -0
  187. data/ext/uchardet/src/uchardet.cpp +274 -0
  188. data/ext/uchardet/src/uchardet.h +136 -0
  189. data/ext/uchardet/test/CMakeLists.txt +47 -0
  190. data/ext/uchardet/test/ar/iso-8859-6.txt +3 -0
  191. data/ext/uchardet/test/ar/utf-8.txt +3 -0
  192. data/ext/uchardet/test/ar/windows-1256.txt +3 -0
  193. data/ext/uchardet/test/bg/windows-1251.txt +3 -0
  194. data/ext/uchardet/test/cs/ibm852.txt +4 -0
  195. data/ext/uchardet/test/cs/iso-8859-2.txt +4 -0
  196. data/ext/uchardet/test/cs/mac-centraleurope.txt +4 -0
  197. data/ext/uchardet/test/cs/utf-8.txt +4 -0
  198. data/ext/uchardet/test/cs/windows-1250.txt +4 -0
  199. data/ext/uchardet/test/da/iso-8859-1.txt +7 -0
  200. data/ext/uchardet/test/da/iso-8859-15.txt +10 -0
  201. data/ext/uchardet/test/da/utf-8.txt +10 -0
  202. data/ext/uchardet/test/da/windows-1252.txt +10 -0
  203. data/ext/uchardet/test/de/iso-8859-1.txt +11 -0
  204. data/ext/uchardet/test/de/windows-1252.txt +11 -0
  205. data/ext/uchardet/test/el/iso-8859-7.txt +3 -0
  206. data/ext/uchardet/test/el/utf-8.txt +3 -0
  207. data/ext/uchardet/test/el/windows-1253.txt +5 -0
  208. data/ext/uchardet/test/en/ascii.txt +4 -0
  209. data/ext/uchardet/test/eo/iso-8859-3.txt +7 -0
  210. data/ext/uchardet/test/es/iso-8859-1.txt +5 -0
  211. data/ext/uchardet/test/es/iso-8859-15.txt +5 -0
  212. data/ext/uchardet/test/es/utf-8.txt +5 -0
  213. data/ext/uchardet/test/es/windows-1252.txt +5 -0
  214. data/ext/uchardet/test/et/iso-8859-13.txt +6 -0
  215. data/ext/uchardet/test/et/iso-8859-15.txt +6 -0
  216. data/ext/uchardet/test/et/iso-8859-4.txt +6 -0
  217. data/ext/uchardet/test/et/utf-8.txt +6 -0
  218. data/ext/uchardet/test/et/windows-1252.txt +6 -0
  219. data/ext/uchardet/test/et/windows-1257.txt +6 -0
  220. data/ext/uchardet/test/fi/iso-8859-1.txt +8 -0
  221. data/ext/uchardet/test/fi/utf-8.txt +8 -0
  222. data/ext/uchardet/test/fr/iso-8859-1.txt +5 -0
  223. data/ext/uchardet/test/fr/iso-8859-15.txt +16 -0
  224. data/ext/uchardet/test/fr/utf-16.be +0 -0
  225. data/ext/uchardet/test/fr/utf-32.le +0 -0
  226. data/ext/uchardet/test/fr/utf-8.txt +14 -0
  227. data/ext/uchardet/test/fr/windows-1252.txt +3 -0
  228. data/ext/uchardet/test/ga/iso-8859-1.txt +6 -0
  229. data/ext/uchardet/test/ga/utf-8.txt +6 -0
  230. data/ext/uchardet/test/ga/windows-1252.txt +6 -0
  231. data/ext/uchardet/test/he/iso-8859-8.txt +2 -0
  232. data/ext/uchardet/test/he/utf-8.txt +3 -0
  233. data/ext/uchardet/test/he/windows-1255.txt +1 -0
  234. data/ext/uchardet/test/hr/ibm852.txt +4 -0
  235. data/ext/uchardet/test/hr/iso-8859-13.txt +4 -0
  236. data/ext/uchardet/test/hr/iso-8859-16.txt +4 -0
  237. data/ext/uchardet/test/hr/iso-8859-2.txt +4 -0
  238. data/ext/uchardet/test/hr/mac-centraleurope.txt +4 -0
  239. data/ext/uchardet/test/hr/utf-8.txt +4 -0
  240. data/ext/uchardet/test/hr/windows-1250.txt +4 -0
  241. data/ext/uchardet/test/hu/iso-8859-2.txt +3 -0
  242. data/ext/uchardet/test/hu/windows-1250.txt +1 -0
  243. data/ext/uchardet/test/it/iso-8859-1.txt +18 -0
  244. data/ext/uchardet/test/it/utf-8.txt +18 -0
  245. data/ext/uchardet/test/ja/euc-jp.txt +10 -0
  246. data/ext/uchardet/test/ja/iso-2022-jp.txt +8 -0
  247. data/ext/uchardet/test/ja/shift_jis.txt +1 -0
  248. data/ext/uchardet/test/ja/utf-16be.txt +0 -0
  249. data/ext/uchardet/test/ja/utf-16le.txt +0 -0
  250. data/ext/uchardet/test/ja/utf-8.txt +9 -0
  251. data/ext/uchardet/test/ko/iso-2022-kr.txt +8 -0
  252. data/ext/uchardet/test/ko/uhc.smi +16 -0
  253. data/ext/uchardet/test/ko/utf-16.le +0 -0
  254. data/ext/uchardet/test/ko/utf-32.be +0 -0
  255. data/ext/uchardet/test/ko/utf-8.txt +3 -0
  256. data/ext/uchardet/test/lt/iso-8859-10.txt +3 -0
  257. data/ext/uchardet/test/lt/iso-8859-13.txt +3 -0
  258. data/ext/uchardet/test/lt/iso-8859-4.txt +3 -0
  259. data/ext/uchardet/test/lt/utf-8.txt +3 -0
  260. data/ext/uchardet/test/lv/iso-8859-10.txt +6 -0
  261. data/ext/uchardet/test/lv/iso-8859-13.txt +6 -0
  262. data/ext/uchardet/test/lv/iso-8859-4.txt +6 -0
  263. data/ext/uchardet/test/lv/utf-8.txt +6 -0
  264. data/ext/uchardet/test/mt/iso-8859-3.txt +4 -0
  265. data/ext/uchardet/test/mt/utf-8.txt +4 -0
  266. data/ext/uchardet/test/pl/ibm852.txt +3 -0
  267. data/ext/uchardet/test/pl/iso-8859-13.txt +3 -0
  268. data/ext/uchardet/test/pl/iso-8859-16.txt +3 -0
  269. data/ext/uchardet/test/pl/iso-8859-2.txt +3 -0
  270. data/ext/uchardet/test/pl/mac-centraleurope.txt +3 -0
  271. data/ext/uchardet/test/pl/utf-8.txt +3 -0
  272. data/ext/uchardet/test/pl/windows-1250.txt +3 -0
  273. data/ext/uchardet/test/pt/iso-8859-1.txt +6 -0
  274. data/ext/uchardet/test/pt/utf-8.txt +6 -0
  275. data/ext/uchardet/test/ro/ibm852.txt +9 -0
  276. data/ext/uchardet/test/ro/iso-8859-16.txt +9 -0
  277. data/ext/uchardet/test/ro/utf-8.txt +9 -0
  278. data/ext/uchardet/test/ro/windows-1250.txt +9 -0
  279. data/ext/uchardet/test/ru/ibm855.txt +5 -0
  280. data/ext/uchardet/test/ru/ibm866.txt +11 -0
  281. data/ext/uchardet/test/ru/iso-8859-5.txt +3 -0
  282. data/ext/uchardet/test/ru/koi8-r.txt +1 -0
  283. data/ext/uchardet/test/ru/mac-cyrillic.txt +9 -0
  284. data/ext/uchardet/test/ru/windows-1251.txt +4 -0
  285. data/ext/uchardet/test/sk/ibm852.txt +3 -0
  286. data/ext/uchardet/test/sk/iso-8859-2.txt +3 -0
  287. data/ext/uchardet/test/sk/mac-centraleurope.txt +3 -0
  288. data/ext/uchardet/test/sk/utf-8.txt +3 -0
  289. data/ext/uchardet/test/sk/windows-1250.txt +3 -0
  290. data/ext/uchardet/test/sl/ibm852.txt +9 -0
  291. data/ext/uchardet/test/sl/iso-8859-16.txt +9 -0
  292. data/ext/uchardet/test/sl/iso-8859-2.txt +9 -0
  293. data/ext/uchardet/test/sl/mac-centraleurope.txt +9 -0
  294. data/ext/uchardet/test/sl/utf-8.txt +9 -0
  295. data/ext/uchardet/test/sl/windows-1250.txt +9 -0
  296. data/ext/uchardet/test/sv/iso-8859-1.txt +10 -0
  297. data/ext/uchardet/test/sv/utf-8.txt +10 -0
  298. data/ext/uchardet/test/sv/windows-1252.txt +10 -0
  299. data/ext/uchardet/test/th/iso-8859-11.txt +5 -0
  300. data/ext/uchardet/test/th/tis-620.txt +5 -0
  301. data/ext/uchardet/test/th/utf-8.txt +1 -0
  302. data/ext/uchardet/test/tr/iso-8859-3.txt +13 -0
  303. data/ext/uchardet/test/tr/iso-8859-9.txt +13 -0
  304. data/ext/uchardet/test/uchardet-tests.c +130 -0
  305. data/ext/uchardet/test/vi/utf-8.txt +4 -0
  306. data/ext/uchardet/test/vi/viscii.txt +4 -0
  307. data/ext/uchardet/test/vi/windows-1258.txt +4 -0
  308. data/ext/uchardet/test/zh/big5.txt +1 -0
  309. data/ext/uchardet/test/zh/euc-tw.txt +1 -0
  310. data/ext/uchardet/test/zh/gb18030.txt +1 -0
  311. data/ext/uchardet/test/zh/utf-8.txt +1 -0
  312. data/ext/uchardet/uchardet.doap +51 -0
  313. data/ext/uchardet/uchardet.pc.in +10 -0
  314. data/lib/cchardet.rb +56 -0
  315. data/lib/cchardet/lib_finder.rb +32 -0
  316. data/lib/cchardet/version.rb +5 -0
  317. metadata +362 -0
data/ext/uchardet/script/BuildLangModelLogs/LangLithuanianModel.log ADDED
@@ -0,0 +1,162 @@
1
+ = Logs of language model for Lithuanian (lt) =
2
+
3
+ - Generated by BuildLangModel.py
4
+ - Started: 2016-09-21 00:23:03.857157
5
+ - Maximum depth: 5
6
+ - Max number of pages: 100
7
+
8
+ == Parsed pages ==
9
+
10
+ Karūna (laivas) (revision 5080379)
11
+ 1650 (revision 4990868)
12
+ 1654 (revision 4991037)
13
+ 1664 (revision 4991048)
14
+ 1665 (revision 4991050)
15
+ 1668 (revision 4991052)
16
+ 1669 (revision 4991053)
17
+ 1672 (revision 4991056)
18
+ 1676 (revision 4991060)
19
+ 1718 (revision 4990914)
20
+ 1909 (revision 4990667)
21
+ 1928 (revision 4990262)
22
+ 1932 (revision 4990613)
23
+ 1956 (revision 4990635)
24
+ 1980 (revision 4990655)
25
+ Baltijos jūra (revision 5052833)
26
+ Burinis laivas (revision 4657401)
27
+ Flagmanas (laivas) (revision 5005271)
28
+ Grimzlė (revision 4487052)
29
+ Kalmaras (Švedija) (revision 4978519)
30
+ Karo laivas (revision 4726931)
31
+ Karolis XI (revision 4944621)
32
+ Karolis XII (revision 4915230)
33
+ Kilis (revision 4325533)
34
+ Koordinačių sistema (revision 5033980)
35
+ Laivo vėliava (revision 4986001)
36
+ Liepos 1 d. (revision 4910200)
37
+ Nyderlandai (revision 5080140)
38
+ Rugpjūčio 10 (revision 4910281)
39
+ Varytuvas (revision 4620792)
40
+ Vaza (laivas) (revision 5079282)
41
+ XVIII a. (revision 4896219)
42
+ XVII a. (revision 4768242)
43
+ Švedija (revision 5057665)
44
+ Švedų kalba (revision 4687559)
45
+ 1590 (revision 4990983)
46
+ 1596 (revision 4990989)
47
+ 1608 (revision 4991000)
48
+ 1610 (revision 4991002)
49
+ 1623 m. (revision 4991015)
50
+ 1634 m. (revision 4991026)
51
+ 1643 m. (revision 4990870)
52
+ 1644 m. (revision 4990872)
53
+ 1645 m. (revision 4990873)
54
+ 1646 m. (revision 4990874)
55
+ 1647 m. (revision 4913295)
56
+ 1648 m. (revision 4990875)
57
+ 1649 m. (revision 4990876)
58
+ 1651 m. (revision 4991035)
59
+ 1652 m. (revision 4991072)
60
+ 1653 m. (revision 4991036)
61
+ 1654 m. (revision 4991037)
62
+ 1655 m. (revision 4991038)
63
+ 1662 m. (revision 4991046)
64
+ 1668 m. (revision 4991052)
65
+ 1677 m. (revision 4991061)
66
+ 1702 (revision 4990595)
67
+ 1704 (revision 4990863)
68
+ 1722 (revision 4990918)
69
+ 1723 (revision 4990919)
70
+ 1737 (revision 4990931)
71
+ 2 tūkstantmetis (revision 4296407)
72
+ ATR (revision 5078529)
73
+ Abiejų Tautų Respublika (revision 5078529)
74
+ Adomas Freitagas (revision 4362991)
75
+ Anglų kalba (revision 4911240)
76
+ Armėnų kalendorius (revision 4817534)
77
+ Bahajų kalendorius (revision 4706296)
78
+ Bajorai (revision 5006456)
79
+ Berberų kalendorius (revision 4926904)
80
+ Birželio 21 (revision 4910142)
81
+ Bizantijos kalendorius (revision 4927623)
82
+ Budistų kalendorius (revision 4705734)
83
+ Dešimtmetis (revision 4296419)
84
+ Dominikonai (revision 4921895)
85
+ Dominikonų ordinas (revision 4921895)
86
+ Džohoro sultonatas (revision 4934526)
87
+ Džu Ihai (revision 4991072)
88
+ Džu Joulang (revision 4991072)
89
+ Emanuelis Vladislovas Tiškevičius Logoiskis (revision 4939239)
90
+ Filosofas (revision 5078172)
91
+ Gegužės 26 (revision 4910130)
92
+ Grafas (titulas) (revision 5008057)
93
+ Grigaliaus kalendorius (revision 5000317)
94
+ Hebrajų kalendorius (revision 4728592)
95
+ Imperatorius Go-Komijas (revision 4907057)
96
+ Inocentas X (revision 4905150)
97
+ Iraniečių kalendorius (revision 4964854)
98
+ Isaac Titsingh (revision 4990745)
99
+ Japonija (revision 5035249)
100
+ Japonijos imperatorius (revision 4720428)
101
+ Japonų kalendorius (revision 4956765)
102
+ John Churchill (revision 4903704)
103
+ Jonas Kazimieras Vaza (revision 5037754)
104
+ Jurgis Kasakauskis (revision 5047829)
105
+ Jurgis Kazimieras Ancuta (revision 5059404)
106
+ Jurgis Mikalojus Tiškevičius (revision 4939554)
107
+
108
+ == End of Parsed pages ==
109
+
110
+ - Wikipedia parsing ended at: 2016-09-21 00:25:34.773941
111
+
112
+ 60 characters appeared 353051 times.
113
+
114
+ First 38 characters:
115
+ [ 0] Char i: 13.032394753165974 %
116
+ [ 1] Char a: 11.167225131779828 %
117
+ [ 2] Char s: 8.586578143101137 %
118
+ [ 3] Char o: 7.018815978428046 %
119
+ [ 4] Char e: 5.525830545728521 %
120
+ [ 5] Char r: 5.469181506354606 %
121
+ [ 6] Char n: 5.142599794363987 %
122
+ [ 7] Char t: 5.105777918770942 %
123
+ [ 8] Char u: 4.270487833202568 %
124
+ [ 9] Char k: 3.9617505686147325 %
125
+ [10] Char l: 3.9051015292408184 %
126
+ [11] Char m: 3.359854525266888 %
127
+ [12] Char d: 3.0372382460324427 %
128
+ [13] Char v: 2.7270847554602593 %
129
+ [14] Char j: 2.4472385009531203 %
130
+ [15] Char p: 2.329125253858508 %
131
+ [16] Char g: 1.9427788053284087 %
132
+ [17] Char ė: 1.5657794482950054 %
133
+ [18] Char b: 1.5074309377398734 %
134
+ [19] Char y: 1.2236192504765602 %
135
+ [20] Char ų: 1.181698961339863 %
136
+ [21] Char š: 0.9630336693565519 %
137
+ [22] Char ž: 0.8171623929687212 %
138
+ [23] Char c: 0.5959478942135839 %
139
+ [24] Char č: 0.48010060869392807 %
140
+ [25] Char f: 0.428266737666796 %
141
+ [26] Char h: 0.42515104050123065 %
142
+ [27] Char z: 0.4010751987673169 %
143
+ [28] Char ū: 0.3685020011273159 %
144
+ [29] Char ą: 0.3526402701026197 %
145
+ [30] Char į: 0.29004308159444386 %
146
+ [31] Char ę: 0.14813723796278724 %
147
+ [32] Char x: 0.08752276583269838 %
148
+ [33] Char w: 0.059198246145740985 %
149
+ [34] Char ō: 0.01812769259965274 %
150
+ [35] Char ö: 0.008780601102956797 %
151
+ [36] Char é: 0.0076476203154785 %
152
+ [37] Char q: 0.007364375118608926 %
153
+
154
+ The first 38 characters have an accumulated ratio of 0.9996629382157253.
155
+
156
+ 1016 sequences found.
157
+
158
+ First 512 (typical positive ratio): 0.9928710196247589
159
+ Next 512 (512-1024): 0.008171623929687212
160
+ Rest: -4.85722573273506e-17
161
+
162
+ - Processing end: 2016-09-21 00:25:34.935858
data/ext/uchardet/script/BuildLangModelLogs/LangMalteseModel.log ADDED
@@ -0,0 +1,147 @@
1
+ = Logs of language model for Maltese (mt) =
2
+
3
+ - Generated by BuildLangModel.py
4
+ - Started: 2016-09-21 02:05:23.411546
5
+ - Maximum depth: 5
6
+ - Max number of pages: 100
7
+
8
+ == Parsed pages ==
9
+
10
+ Unjoni Ewropea (revision 246298)
11
+ 1951 (revision 229183)
12
+ 1952 (revision 229184)
13
+ 1957 (revision 229188)
14
+ 1958 (revision 229189)
15
+ 1973 (revision 223536)
16
+ 1979 (revision 243876)
17
+ 1981 (revision 205545)
18
+ 1985 (revision 216368)
19
+ 1986 (revision 231433)
20
+ 1990 (revision 237666)
21
+ 1992 (revision 244087)
22
+ 1995 (revision 214650)
23
+ 1 ta' Mejju (revision 245374)
24
+ 2007 (revision 214851)
25
+ 2013 (revision 245606)
26
+ Albanija (revision 243079)
27
+ Awstrija (revision 243627)
28
+ Awtonomija (revision 245824)
29
+ Ażores (revision 246298)
30
+ Bank Ċentrali Ewropew (revision 246298)
31
+ Belt kapitali (revision 237400)
32
+ Belġju (revision 244363)
33
+ Brussell (revision 243311)
34
+ Bulgarija (revision 243622)
35
+ Danimarka (revision 244419)
36
+ De facto (revision 215102)
37
+ Estonja (revision 243826)
38
+ European Free Trade Association (revision 246298)
39
+ Ewropa (revision 244177)
40
+ Ex Repubblika Jugoslava tal-Maċedonja (revision 246298)
41
+ Federazzjoni (revision 246226)
42
+ Finlandja (revision 245824)
43
+ Frankfurt (revision 243576)
44
+ Franza (revision 244461)
45
+ Greċja (revision 244423)
46
+ Groenlandja (revision 243829)
47
+ Indja (revision 244873)
48
+ Islanda (revision 243771)
49
+ Isle of Man (revision 246298)
50
+ Istitut tal-Unjoni Ewropea għall-Istudji dwar is-Sigurtà (revision 244412)
51
+ Italja (revision 246323)
52
+ Kilometru kwadru (revision 244871)
53
+ Komunitajiet Ewropej (revision 246298)
54
+ Komunità Ekonomika Ewropea (revision 246298)
55
+ Kroazja (revision 245711)
56
+ Kummissjoni Ewropea (revision 243311)
57
+ Kunsill Ewropew (revision 246298)
58
+ Kunsill tal-Ewropa (revision 243334)
59
+ Kunsill tal-Unjoni Ewropea (revision 243311)
60
+ Latvja (revision 245746)
61
+ Lista ta' pajjiżi skont id-daqs (revision 244419)
62
+ Lista ta' pajjiżi skont il-popolazzjoni (revision 246128)
63
+ Litwanja (revision 243114)
64
+ Liġijiet tal-Unjoni Ewropea (revision 246298)
65
+ Lussemburgu (revision 244239)
66
+ Lussemburgu (belt) (revision 243587)
67
+ Madejra (revision 243625)
68
+ Malta (revision 247210)
69
+ Montenegro (revision 243930)
70
+ Norveġja (revision 243829)
71
+ Olanda (revision 243989)
72
+ Organizzazzjoni Internazzjonali (revision 246724)
73
+ Pajjiżi l-Baxxi (revision 243989)
74
+ Pajjiżi membri tal-Unjoni Ewropea (revision 243625)
75
+ Pajjiżi ġirien li jdawru l-Unjoni Ewropea (revision 246298)
76
+ Parlament Ewropew (revision 243907)
77
+ Patt ta' Stabilità u Tkabbir (revision 246298)
78
+ Politika agrikola komuni (revision 244363)
79
+ Politika reġjonali tal-Unjoni Ewropea (revision 246298)
80
+ Polonja (revision 244530)
81
+ Portugall (revision 243625)
82
+ Relazzjonijiet ta' terzi pajjiżi ma l-UE (revision 246298)
83
+ Renju Unit (revision 247318)
84
+ Repubblika Federali tal-Ġermanja (revision 244859)
85
+ Repubblika tal-Irlanda (revision 243686)
86
+ Repubblika Ċeka (revision 246832)
87
+ Rumanija (revision 243623)
88
+ Segretarjat tal-Parlament Ewropew (revision 246298)
89
+ Serbja (revision 243728)
90
+ Slovakkja (revision 243831)
91
+ Slovenja (revision 244588)
92
+ Spanja (revision 246856)
93
+ Stati Uniti tal-Amerika (revision 243926)
94
+ Stati membri tal-Unjoni Ewropea (revision 243114)
95
+ Strasburgu (revision 243503)
96
+ Sui generis (revision 247150)
97
+ Suq komuni (revision 246298)
98
+ Svezja (revision 244871)
99
+
100
+ == End of Parsed pages ==
101
+
102
+ - Wikipedia parsing ended at: 2016-09-21 02:07:45.508113
103
+
104
+ 48 characters appeared 474337 times.
105
+
106
+ First 31 characters:
107
+ [ 0] Char a: 12.326257492036252 %
108
+ [ 1] Char i: 12.069899670487438 %
109
+ [ 2] Char t: 8.064941170518008 %
110
+ [ 3] Char l: 7.795301652622502 %
111
+ [ 4] Char e: 6.615971345267184 %
112
+ [ 5] Char n: 6.128132530247482 %
113
+ [ 6] Char r: 5.579577389071483 %
114
+ [ 7] Char u: 4.376424356522894 %
115
+ [ 8] Char o: 3.8337721915009797 %
116
+ [ 9] Char j: 3.7378488289971057 %
117
+ [10] Char m: 3.6084049947611088 %
118
+ [11] Char s: 3.3533120966738834 %
119
+ [12] Char k: 2.588033402412209 %
120
+ [13] Char d: 2.3173397816320462 %
121
+ [14] Char p: 2.0555006250830106 %
122
+ [15] Char b: 2.017131280081461 %
123
+ [16] Char f: 2.004692866042497 %
124
+ [17] Char ħ: 1.6372326004507345 %
125
+ [18] Char w: 1.4801712706366992 %
126
+ [19] Char g: 1.4763765002519307 %
127
+ [20] Char z: 1.3150987588992635 %
128
+ [21] Char ż: 0.9910675321554084 %
129
+ [22] Char h: 0.9750451683086075 %
130
+ [23] Char ġ: 0.7640137708000851 %
131
+ [24] Char ċ: 0.6723068198348432 %
132
+ [25] Char x: 0.5892435125237964 %
133
+ [26] Char v: 0.5668965313690478 %
134
+ [27] Char q: 0.5647883255997318 %
135
+ [28] Char c: 0.2759641352034524 %
136
+ [29] Char à: 0.10730767365817974 %
137
+ [30] Char y: 0.059029761540845424 %
138
+
139
+ The first 31 characters have an accumulated ratio of 0.9994708403519017.
140
+
141
+ 870 sequences found.
142
+
143
+ First 512 (typical positive ratio): 0.9959115850692665
144
+ Next 512 (512-1024): 2.108205769315908e-06
145
+ Rest: -4.423544863740858e-17
146
+
147
+ - Processing end: 2016-09-21 02:07:45.646198
data/ext/uchardet/script/BuildLangModelLogs/LangPolishModel.log ADDED
@@ -0,0 +1,154 @@
1
+ = Logs of language model for Polish (pl) =
2
+
3
+ - Generated by BuildLangModel.py
4
+ - Started: 2016-09-21 17:06:43.735784
5
+ - Maximum depth: 5
6
+ - Max number of pages: 100
7
+
8
+ == Parsed pages ==
9
+
10
+ Krasnyj Krym (revision 46884814)
11
+ 1913 (revision 46708474)
12
+ 1915 (revision 46743905)
13
+ 1917 (revision 46559521)
14
+ 1925 (revision 46809935)
15
+ 1928 (revision 46875978)
16
+ 1929 (revision 46760445)
17
+ 1935 (revision 46487358)
18
+ 1936 (revision 46874348)
19
+ 1939 (revision 46789269)
20
+ 1941 (revision 46856112)
21
+ 1942 (revision 46851808)
22
+ 1943 (revision 46768330)
23
+ 1944 (revision 46866229)
24
+ 1949 (revision 46882598)
25
+ 1953 (revision 46437607)
26
+ 1957 (revision 46591716)
27
+ 1959 (revision 46255886)
28
+ Admirał Butakow (revision 45993412)
29
+ Admirał Spiridow (revision 45993412)
30
+ Aparat torpedowy (revision 46633263)
31
+ Askold (revision 45787848)
32
+ Avro 504 (revision 44668646)
33
+ Ałmaz (1903) (revision 46472283)
34
+ Batumi (revision 46594611)
35
+ Bomba głębinowa (revision 46011227)
36
+ Brest (revision 45771242)
37
+ Burta (revision 45569092)
38
+ Cagliari (revision 46235605)
39
+ Cesariewicz (revision 40031486)
40
+ Czerwona Ukraina (revision 45993524)
41
+ Daty nowego i starego porządku (revision 45622575)
42
+ Drednot (revision 45789788)
43
+ Działo przeciwlotnicze (revision 45160162)
44
+ Flota Bałtycka Marynarki Wojennej Rosji (revision 45700667)
45
+ Gromoboj (revision 44328986)
46
+ Hulk (okręt) (revision 46020688)
47
+ II wojna światowa (revision 46871591)
48
+ I wojna światowa (revision 46869119)
49
+ Imperator Nikołaj I (okręt lotniczy) (revision 45520638)
50
+ Imperium Rosyjskie (revision 46604959)
51
+ Impierator Nikołaj I (1916) (revision 46534166)
52
+ Język rosyjski (revision 46433952)
53
+ Kanonierka (revision 41091952)
54
+ Kanonierki typu Ardagan (revision 46534166)
55
+ Kanonierki typu Bobr (revision 45788694)
56
+ Kanonierki typu Chiwiniec (revision 46534166)
57
+ Kanonierki typu Groziaszczij (revision 46534166)
58
+ Kanonierki typu Mandżur (revision 46534166)
59
+ Karabin maszynowy DSzK (revision 45587452)
60
+ Karabin maszynowy Vickers 12,7 mm (revision 44572918)
61
+ Kocioł parowy (revision 46716473)
62
+ Konstrukcyjna linia wodna (revision 37082620)
63
+ Kontrtorpedowce typu Biesstrasznyj (revision 46534166)
64
+ Kontrtorpedowce typu Brawyj (revision 46534166)
65
+ Kontrtorpedowce typu Grozowoj (revision 46534166)
66
+ Kontrtorpedowce typu Prytkij (revision 46534166)
67
+ Koń mechaniczny (revision 44722357)
68
+ Krab (1915) (revision 42791389)
69
+ Kronsztad (revision 46425497)
70
+ Krążownik lekki (revision 40661490)
71
+ Krążownik liniowy (revision 40601776)
72
+ Krążownik pancernopokładowy (revision 40055901)
73
+ Krążownik pancerny (revision 40324458)
74
+ Krążowniki lekkie typu Swietłana (revision 45993412)
75
+ Krążowniki liniowe typu Borodino (revision 45990866)
76
+ Krążowniki typu Admirał Nachimow (revision 45993521)
77
+ Krążowniki typu Bajan (revision 45991279)
78
+ Krążowniki typu Diana (revision 45991349)
79
+ Krążowniki typu Izumrud (revision 45991349)
80
+ Lend-Lease Act (revision 46877263)
81
+ Marynarka Wojenna Związku Socjalistycznych Republik Radzieckich (revision 45795993)
82
+ Maszyna sterowa (revision 28497888)
83
+ Mecidiye (1903) (revision 43956539)
84
+ Mila morska (revision 45754209)
85
+ Mina morska (revision 45781427)
86
+ Morze Czarne (revision 46729213)
87
+ Nadbudówka (revision 45292731)
88
+ Neapol (revision 46823083)
89
+ Niszczyciel (revision 45799132)
90
+ Niszczyciele rakietowe projektu 61 (revision 46498775)
91
+ Niszczyciele typu Finn (revision 46620140)
92
+ Niszczyciele typu Lejtienant Szestakow (revision 46620140)
93
+ Niszczyciele typu Ochotnik (revision 46620140)
94
+ Niszczyciele typu Ukraina (revision 46620140)
95
+ Noworosyjsk (revision 44721836)
96
+ Odessa (revision 45629804)
97
+ Oerlikon 20 mm (revision 45493862)
98
+ Okres międzywojenny (revision 46668249)
99
+ Okręt-baza wodnosamolotów (revision 45115462)
100
+
101
+ == End of Parsed pages ==
102
+
103
+ - Wikipedia parsing ended at: 2016-09-21 17:21:04.404471
104
+
105
+ 78 characters appeared 1159291 times.
106
+
107
+ First 37 characters:
108
+ [ 0] Char a: 9.685575062689178 %
109
+ [ 1] Char i: 8.815819324052374 %
110
+ [ 2] Char o: 7.920185699707839 %
111
+ [ 3] Char e: 6.871613770830621 %
112
+ [ 4] Char r: 5.8672067668945935 %
113
+ [ 5] Char n: 5.763608964444647 %
114
+ [ 6] Char s: 4.736688199942896 %
115
+ [ 7] Char k: 4.722196583946568 %
116
+ [ 8] Char z: 4.519227700378939 %
117
+ [ 9] Char w: 4.279512219106333 %
118
+ [10] Char t: 4.0191806888865695 %
119
+ [11] Char c: 3.6891513864939864 %
120
+ [12] Char y: 3.565282573572986 %
121
+ [13] Char p: 3.0190004062828053 %
122
+ [14] Char d: 2.851052928039638 %
123
+ [15] Char l: 2.7930002044352973 %
124
+ [16] Char m: 2.7530620008263673 %
125
+ [17] Char u: 2.348504387595522 %
126
+ [18] Char j: 1.881236031332944 %
127
+ [19] Char ł: 1.6885320424293815 %
128
+ [20] Char b: 1.394559260789569 %
129
+ [21] Char g: 1.3928340684090534 %
130
+ [22] Char h: 1.163901039514669 %
131
+ [23] Char ę: 0.8066136975099435 %
132
+ [24] Char ó: 0.5971753425153823 %
133
+ [25] Char ą: 0.563275312238256 %
134
+ [26] Char f: 0.5245447432956868 %
135
+ [27] Char ż: 0.4545019326467643 %
136
+ [28] Char ś: 0.39567287247119143 %
137
+ [29] Char ń: 0.3857530162832283 %
138
+ [30] Char ć: 0.1397405828217419 %
139
+ [31] Char v: 0.12455888987320698 %
140
+ [32] Char ź: 0.10204512930748191 %
141
+ [33] Char x: 0.05468859846233603 %
142
+ [34] Char é: 0.020961087423261287 %
143
+ [35] Char á: 0.01707940456710179 %
144
+ [36] Char q: 0.011386269711401192 %
145
+
146
+ The first 37 characters have an accumulated ratio of 0.9993892818972973.
147
+
148
+ 1321 sequences found.
149
+
150
+ First 512 (typical positive ratio): 0.9894531815946438
151
+ Next 512 (512-1024): 1.7251923805153322e-06
152
+ Rest: 0.0003530230403650733
153
+
154
+ - Processing end: 2016-09-21 17:21:04.878014