cchardet 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (317) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.gitmodules +3 -0
  4. data/.rubocop.yml +11 -0
  5. data/CHANGELOG.md +5 -0
  6. data/Gemfile +10 -0
  7. data/README.md +35 -0
  8. data/Rakefile +15 -0
  9. data/cchardet.gemspec +30 -0
  10. data/ext/cchardet/extconf.rb +26 -0
  11. data/ext/uchardet/.gitignore +1 -0
  12. data/ext/uchardet/.gitlab-ci.yml +106 -0
  13. data/ext/uchardet/AUTHORS +16 -0
  14. data/ext/uchardet/CMakeLists.txt +74 -0
  15. data/ext/uchardet/COPYING +1316 -0
  16. data/ext/uchardet/INSTALL +26 -0
  17. data/ext/uchardet/README.md +295 -0
  18. data/ext/uchardet/build-mac/uchardet.cpp +7 -0
  19. data/ext/uchardet/build-mac/uchardet.xcodeproj/project.pbxproj +543 -0
  20. data/ext/uchardet/build-mac/uchardet.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -0
  21. data/ext/uchardet/build-mac/uchardet.xcodeproj/project.xcworkspace/xcshareddata/uchardet.xccheckout +41 -0
  22. data/ext/uchardet/build-mac/uchardet.xcodeproj/xcshareddata/xcschemes/uchardet-ios.xcscheme +77 -0
  23. data/ext/uchardet/build-mac/uchardet.xcodeproj/xcshareddata/xcschemes/uchardet.xcscheme +77 -0
  24. data/ext/uchardet/doc/CMakeLists.txt +6 -0
  25. data/ext/uchardet/doc/README.maintainer +59 -0
  26. data/ext/uchardet/doc/uchardet.1 +18 -0
  27. data/ext/uchardet/script/BuildLangModel.py +533 -0
  28. data/ext/uchardet/script/BuildLangModelLogs/LangArabicModel.log +142 -0
  29. data/ext/uchardet/script/BuildLangModelLogs/LangCroatianModel.log +157 -0
  30. data/ext/uchardet/script/BuildLangModelLogs/LangCzechModel.log +161 -0
  31. data/ext/uchardet/script/BuildLangModelLogs/LangDanishModel.log +158 -0
  32. data/ext/uchardet/script/BuildLangModelLogs/LangEsperantoModel.log +110 -0
  33. data/ext/uchardet/script/BuildLangModelLogs/LangEstonianModel.log +159 -0
  34. data/ext/uchardet/script/BuildLangModelLogs/LangFinnishModel.log +156 -0
  35. data/ext/uchardet/script/BuildLangModelLogs/LangFrenchModel.log +116 -0
  36. data/ext/uchardet/script/BuildLangModelLogs/LangGermanModel.log +159 -0
  37. data/ext/uchardet/script/BuildLangModelLogs/LangGreekModel.log +272 -0
  38. data/ext/uchardet/script/BuildLangModelLogs/LangHungarianModel.log +109 -0
  39. data/ext/uchardet/script/BuildLangModelLogs/LangIrishModel.log +156 -0
  40. data/ext/uchardet/script/BuildLangModelLogs/LangItalianModel.log +162 -0
  41. data/ext/uchardet/script/BuildLangModelLogs/LangLatvianModel.log +162 -0
  42. data/ext/uchardet/script/BuildLangModelLogs/LangLithuanianModel.log +162 -0
  43. data/ext/uchardet/script/BuildLangModelLogs/LangMalteseModel.log +147 -0
  44. data/ext/uchardet/script/BuildLangModelLogs/LangPolishModel.log +154 -0
  45. data/ext/uchardet/script/BuildLangModelLogs/LangPortugueseModel.log +166 -0
  46. data/ext/uchardet/script/BuildLangModelLogs/LangRomanianModel.log +153 -0
  47. data/ext/uchardet/script/BuildLangModelLogs/LangSlovakModel.log +158 -0
  48. data/ext/uchardet/script/BuildLangModelLogs/LangSloveneModel.log +148 -0
  49. data/ext/uchardet/script/BuildLangModelLogs/LangSpanishModel.log +109 -0
  50. data/ext/uchardet/script/BuildLangModelLogs/LangSwedishModel.log +151 -0
  51. data/ext/uchardet/script/BuildLangModelLogs/LangThaiModel.log +141 -0
  52. data/ext/uchardet/script/BuildLangModelLogs/LangTurkishModel.log +113 -0
  53. data/ext/uchardet/script/BuildLangModelLogs/LangVietnameseModel.log +121 -0
  54. data/ext/uchardet/script/README +63 -0
  55. data/ext/uchardet/script/charsets/codepoints.py +53 -0
  56. data/ext/uchardet/script/charsets/db.py +73 -0
  57. data/ext/uchardet/script/charsets/ibm852.py +72 -0
  58. data/ext/uchardet/script/charsets/iso-8859-1.py +85 -0
  59. data/ext/uchardet/script/charsets/iso-8859-10.py +73 -0
  60. data/ext/uchardet/script/charsets/iso-8859-11.py +77 -0
  61. data/ext/uchardet/script/charsets/iso-8859-13.py +72 -0
  62. data/ext/uchardet/script/charsets/iso-8859-15.py +80 -0
  63. data/ext/uchardet/script/charsets/iso-8859-16.py +83 -0
  64. data/ext/uchardet/script/charsets/iso-8859-2.py +73 -0
  65. data/ext/uchardet/script/charsets/iso-8859-3.py +75 -0
  66. data/ext/uchardet/script/charsets/iso-8859-4.py +73 -0
  67. data/ext/uchardet/script/charsets/iso-8859-6.py +73 -0
  68. data/ext/uchardet/script/charsets/iso-8859-7.py +73 -0
  69. data/ext/uchardet/script/charsets/iso-8859-9.py +76 -0
  70. data/ext/uchardet/script/charsets/mac-centraleurope.py +72 -0
  71. data/ext/uchardet/script/charsets/tis-620.py +77 -0
  72. data/ext/uchardet/script/charsets/viscii.py +72 -0
  73. data/ext/uchardet/script/charsets/windows-1250.py +75 -0
  74. data/ext/uchardet/script/charsets/windows-1252.py +76 -0
  75. data/ext/uchardet/script/charsets/windows-1253.py +72 -0
  76. data/ext/uchardet/script/charsets/windows-1256.py +75 -0
  77. data/ext/uchardet/script/charsets/windows-1257.py +72 -0
  78. data/ext/uchardet/script/charsets/windows-1258.py +72 -0
  79. data/ext/uchardet/script/debug.sh +9 -0
  80. data/ext/uchardet/script/header-template.cpp +38 -0
  81. data/ext/uchardet/script/langs/ar.py +59 -0
  82. data/ext/uchardet/script/langs/cs.py +80 -0
  83. data/ext/uchardet/script/langs/da.py +69 -0
  84. data/ext/uchardet/script/langs/de.py +69 -0
  85. data/ext/uchardet/script/langs/el.py +55 -0
  86. data/ext/uchardet/script/langs/eo.py +67 -0
  87. data/ext/uchardet/script/langs/es.py +69 -0
  88. data/ext/uchardet/script/langs/et.py +57 -0
  89. data/ext/uchardet/script/langs/fi.py +60 -0
  90. data/ext/uchardet/script/langs/fr.py +79 -0
  91. data/ext/uchardet/script/langs/ga.py +60 -0
  92. data/ext/uchardet/script/langs/hr.py +59 -0
  93. data/ext/uchardet/script/langs/hu.py +66 -0
  94. data/ext/uchardet/script/langs/it.py +56 -0
  95. data/ext/uchardet/script/langs/lt.py +70 -0
  96. data/ext/uchardet/script/langs/lv.py +69 -0
  97. data/ext/uchardet/script/langs/mt.py +80 -0
  98. data/ext/uchardet/script/langs/pl.py +81 -0
  99. data/ext/uchardet/script/langs/pt.py +80 -0
  100. data/ext/uchardet/script/langs/ro.py +65 -0
  101. data/ext/uchardet/script/langs/sk.py +80 -0
  102. data/ext/uchardet/script/langs/sl.py +59 -0
  103. data/ext/uchardet/script/langs/sv.py +56 -0
  104. data/ext/uchardet/script/langs/th.py +55 -0
  105. data/ext/uchardet/script/langs/tr.py +67 -0
  106. data/ext/uchardet/script/langs/vi.py +64 -0
  107. data/ext/uchardet/script/release.sh +8 -0
  108. data/ext/uchardet/script/win32.sh +7 -0
  109. data/ext/uchardet/src/Big5Freq.tab +943 -0
  110. data/ext/uchardet/src/CMakeLists.txt +160 -0
  111. data/ext/uchardet/src/CharDistribution.cpp +109 -0
  112. data/ext/uchardet/src/CharDistribution.h +242 -0
  113. data/ext/uchardet/src/EUCKRFreq.tab +614 -0
  114. data/ext/uchardet/src/EUCTWFreq.tab +447 -0
  115. data/ext/uchardet/src/GB2312Freq.tab +491 -0
  116. data/ext/uchardet/src/JISFreq.tab +589 -0
  117. data/ext/uchardet/src/JpCntx.cpp +230 -0
  118. data/ext/uchardet/src/JpCntx.h +140 -0
  119. data/ext/uchardet/src/LangModels/LangArabicModel.cpp +267 -0
  120. data/ext/uchardet/src/LangModels/LangBulgarianModel.cpp +249 -0
  121. data/ext/uchardet/src/LangModels/LangCroatianModel.cpp +298 -0
  122. data/ext/uchardet/src/LangModels/LangCzechModel.cpp +285 -0
  123. data/ext/uchardet/src/LangModels/LangDanishModel.cpp +201 -0
  124. data/ext/uchardet/src/LangModels/LangEsperantoModel.cpp +142 -0
  125. data/ext/uchardet/src/LangModels/LangEstonianModel.cpp +268 -0
  126. data/ext/uchardet/src/LangModels/LangFinnishModel.cpp +297 -0
  127. data/ext/uchardet/src/LangModels/LangFrenchModel.cpp +209 -0
  128. data/ext/uchardet/src/LangModels/LangGermanModel.cpp +170 -0
  129. data/ext/uchardet/src/LangModels/LangGreekModel.cpp +231 -0
  130. data/ext/uchardet/src/LangModels/LangHebrewModel.cpp +220 -0
  131. data/ext/uchardet/src/LangModels/LangHungarianModel.cpp +171 -0
  132. data/ext/uchardet/src/LangModels/LangIrishModel.cpp +234 -0
  133. data/ext/uchardet/src/LangModels/LangItalianModel.cpp +269 -0
  134. data/ext/uchardet/src/LangModels/LangLatvianModel.cpp +210 -0
  135. data/ext/uchardet/src/LangModels/LangLithuanianModel.cpp +209 -0
  136. data/ext/uchardet/src/LangModels/LangMalteseModel.cpp +138 -0
  137. data/ext/uchardet/src/LangModels/LangPolishModel.cpp +304 -0
  138. data/ext/uchardet/src/LangModels/LangPortugueseModel.cpp +241 -0
  139. data/ext/uchardet/src/LangModels/LangRomanianModel.cpp +236 -0
  140. data/ext/uchardet/src/LangModels/LangRussianModel.cpp +367 -0
  141. data/ext/uchardet/src/LangModels/LangSlovakModel.cpp +293 -0
  142. data/ext/uchardet/src/LangModels/LangSloveneModel.cpp +264 -0
  143. data/ext/uchardet/src/LangModels/LangSpanishModel.cpp +204 -0
  144. data/ext/uchardet/src/LangModels/LangSwedishModel.cpp +266 -0
  145. data/ext/uchardet/src/LangModels/LangThaiModel.cpp +267 -0
  146. data/ext/uchardet/src/LangModels/LangTurkishModel.cpp +175 -0
  147. data/ext/uchardet/src/LangModels/LangVietnameseModel.cpp +249 -0
  148. data/ext/uchardet/src/nsBig5Prober.cpp +88 -0
  149. data/ext/uchardet/src/nsBig5Prober.h +75 -0
  150. data/ext/uchardet/src/nsCharSetProber.cpp +125 -0
  151. data/ext/uchardet/src/nsCharSetProber.h +77 -0
  152. data/ext/uchardet/src/nsCodingStateMachine.h +105 -0
  153. data/ext/uchardet/src/nsEUCJPProber.cpp +99 -0
  154. data/ext/uchardet/src/nsEUCJPProber.h +79 -0
  155. data/ext/uchardet/src/nsEUCKRProber.cpp +91 -0
  156. data/ext/uchardet/src/nsEUCKRProber.h +81 -0
  157. data/ext/uchardet/src/nsEUCTWProber.cpp +91 -0
  158. data/ext/uchardet/src/nsEUCTWProber.h +75 -0
  159. data/ext/uchardet/src/nsEscCharsetProber.cpp +101 -0
  160. data/ext/uchardet/src/nsEscCharsetProber.h +70 -0
  161. data/ext/uchardet/src/nsEscSM.cpp +267 -0
  162. data/ext/uchardet/src/nsGB2312Prober.cpp +96 -0
  163. data/ext/uchardet/src/nsGB2312Prober.h +77 -0
  164. data/ext/uchardet/src/nsHebrewProber.cpp +194 -0
  165. data/ext/uchardet/src/nsHebrewProber.h +177 -0
  166. data/ext/uchardet/src/nsLatin1Prober.cpp +182 -0
  167. data/ext/uchardet/src/nsLatin1Prober.h +73 -0
  168. data/ext/uchardet/src/nsMBCSGroupProber.cpp +242 -0
  169. data/ext/uchardet/src/nsMBCSGroupProber.h +81 -0
  170. data/ext/uchardet/src/nsMBCSSM.cpp +513 -0
  171. data/ext/uchardet/src/nsPkgInt.h +89 -0
  172. data/ext/uchardet/src/nsSBCSGroupProber.cpp +343 -0
  173. data/ext/uchardet/src/nsSBCSGroupProber.h +71 -0
  174. data/ext/uchardet/src/nsSBCharSetProber.cpp +160 -0
  175. data/ext/uchardet/src/nsSBCharSetProber.h +258 -0
  176. data/ext/uchardet/src/nsSJISProber.cpp +98 -0
  177. data/ext/uchardet/src/nsSJISProber.h +81 -0
  178. data/ext/uchardet/src/nsUTF8Prober.cpp +87 -0
  179. data/ext/uchardet/src/nsUTF8Prober.h +66 -0
  180. data/ext/uchardet/src/nsUniversalDetector.cpp +339 -0
  181. data/ext/uchardet/src/nsUniversalDetector.h +91 -0
  182. data/ext/uchardet/src/nscore.h +59 -0
  183. data/ext/uchardet/src/prmem.h +49 -0
  184. data/ext/uchardet/src/symbols.cmake +41 -0
  185. data/ext/uchardet/src/tools/CMakeLists.txt +23 -0
  186. data/ext/uchardet/src/tools/uchardet.cpp +254 -0
  187. data/ext/uchardet/src/uchardet.cpp +274 -0
  188. data/ext/uchardet/src/uchardet.h +136 -0
  189. data/ext/uchardet/test/CMakeLists.txt +47 -0
  190. data/ext/uchardet/test/ar/iso-8859-6.txt +3 -0
  191. data/ext/uchardet/test/ar/utf-8.txt +3 -0
  192. data/ext/uchardet/test/ar/windows-1256.txt +3 -0
  193. data/ext/uchardet/test/bg/windows-1251.txt +3 -0
  194. data/ext/uchardet/test/cs/ibm852.txt +4 -0
  195. data/ext/uchardet/test/cs/iso-8859-2.txt +4 -0
  196. data/ext/uchardet/test/cs/mac-centraleurope.txt +4 -0
  197. data/ext/uchardet/test/cs/utf-8.txt +4 -0
  198. data/ext/uchardet/test/cs/windows-1250.txt +4 -0
  199. data/ext/uchardet/test/da/iso-8859-1.txt +7 -0
  200. data/ext/uchardet/test/da/iso-8859-15.txt +10 -0
  201. data/ext/uchardet/test/da/utf-8.txt +10 -0
  202. data/ext/uchardet/test/da/windows-1252.txt +10 -0
  203. data/ext/uchardet/test/de/iso-8859-1.txt +11 -0
  204. data/ext/uchardet/test/de/windows-1252.txt +11 -0
  205. data/ext/uchardet/test/el/iso-8859-7.txt +3 -0
  206. data/ext/uchardet/test/el/utf-8.txt +3 -0
  207. data/ext/uchardet/test/el/windows-1253.txt +5 -0
  208. data/ext/uchardet/test/en/ascii.txt +4 -0
  209. data/ext/uchardet/test/eo/iso-8859-3.txt +7 -0
  210. data/ext/uchardet/test/es/iso-8859-1.txt +5 -0
  211. data/ext/uchardet/test/es/iso-8859-15.txt +5 -0
  212. data/ext/uchardet/test/es/utf-8.txt +5 -0
  213. data/ext/uchardet/test/es/windows-1252.txt +5 -0
  214. data/ext/uchardet/test/et/iso-8859-13.txt +6 -0
  215. data/ext/uchardet/test/et/iso-8859-15.txt +6 -0
  216. data/ext/uchardet/test/et/iso-8859-4.txt +6 -0
  217. data/ext/uchardet/test/et/utf-8.txt +6 -0
  218. data/ext/uchardet/test/et/windows-1252.txt +6 -0
  219. data/ext/uchardet/test/et/windows-1257.txt +6 -0
  220. data/ext/uchardet/test/fi/iso-8859-1.txt +8 -0
  221. data/ext/uchardet/test/fi/utf-8.txt +8 -0
  222. data/ext/uchardet/test/fr/iso-8859-1.txt +5 -0
  223. data/ext/uchardet/test/fr/iso-8859-15.txt +16 -0
  224. data/ext/uchardet/test/fr/utf-16.be +0 -0
  225. data/ext/uchardet/test/fr/utf-32.le +0 -0
  226. data/ext/uchardet/test/fr/utf-8.txt +14 -0
  227. data/ext/uchardet/test/fr/windows-1252.txt +3 -0
  228. data/ext/uchardet/test/ga/iso-8859-1.txt +6 -0
  229. data/ext/uchardet/test/ga/utf-8.txt +6 -0
  230. data/ext/uchardet/test/ga/windows-1252.txt +6 -0
  231. data/ext/uchardet/test/he/iso-8859-8.txt +2 -0
  232. data/ext/uchardet/test/he/utf-8.txt +3 -0
  233. data/ext/uchardet/test/he/windows-1255.txt +1 -0
  234. data/ext/uchardet/test/hr/ibm852.txt +4 -0
  235. data/ext/uchardet/test/hr/iso-8859-13.txt +4 -0
  236. data/ext/uchardet/test/hr/iso-8859-16.txt +4 -0
  237. data/ext/uchardet/test/hr/iso-8859-2.txt +4 -0
  238. data/ext/uchardet/test/hr/mac-centraleurope.txt +4 -0
  239. data/ext/uchardet/test/hr/utf-8.txt +4 -0
  240. data/ext/uchardet/test/hr/windows-1250.txt +4 -0
  241. data/ext/uchardet/test/hu/iso-8859-2.txt +3 -0
  242. data/ext/uchardet/test/hu/windows-1250.txt +1 -0
  243. data/ext/uchardet/test/it/iso-8859-1.txt +18 -0
  244. data/ext/uchardet/test/it/utf-8.txt +18 -0
  245. data/ext/uchardet/test/ja/euc-jp.txt +10 -0
  246. data/ext/uchardet/test/ja/iso-2022-jp.txt +8 -0
  247. data/ext/uchardet/test/ja/shift_jis.txt +1 -0
  248. data/ext/uchardet/test/ja/utf-16be.txt +0 -0
  249. data/ext/uchardet/test/ja/utf-16le.txt +0 -0
  250. data/ext/uchardet/test/ja/utf-8.txt +9 -0
  251. data/ext/uchardet/test/ko/iso-2022-kr.txt +8 -0
  252. data/ext/uchardet/test/ko/uhc.smi +16 -0
  253. data/ext/uchardet/test/ko/utf-16.le +0 -0
  254. data/ext/uchardet/test/ko/utf-32.be +0 -0
  255. data/ext/uchardet/test/ko/utf-8.txt +3 -0
  256. data/ext/uchardet/test/lt/iso-8859-10.txt +3 -0
  257. data/ext/uchardet/test/lt/iso-8859-13.txt +3 -0
  258. data/ext/uchardet/test/lt/iso-8859-4.txt +3 -0
  259. data/ext/uchardet/test/lt/utf-8.txt +3 -0
  260. data/ext/uchardet/test/lv/iso-8859-10.txt +6 -0
  261. data/ext/uchardet/test/lv/iso-8859-13.txt +6 -0
  262. data/ext/uchardet/test/lv/iso-8859-4.txt +6 -0
  263. data/ext/uchardet/test/lv/utf-8.txt +6 -0
  264. data/ext/uchardet/test/mt/iso-8859-3.txt +4 -0
  265. data/ext/uchardet/test/mt/utf-8.txt +4 -0
  266. data/ext/uchardet/test/pl/ibm852.txt +3 -0
  267. data/ext/uchardet/test/pl/iso-8859-13.txt +3 -0
  268. data/ext/uchardet/test/pl/iso-8859-16.txt +3 -0
  269. data/ext/uchardet/test/pl/iso-8859-2.txt +3 -0
  270. data/ext/uchardet/test/pl/mac-centraleurope.txt +3 -0
  271. data/ext/uchardet/test/pl/utf-8.txt +3 -0
  272. data/ext/uchardet/test/pl/windows-1250.txt +3 -0
  273. data/ext/uchardet/test/pt/iso-8859-1.txt +6 -0
  274. data/ext/uchardet/test/pt/utf-8.txt +6 -0
  275. data/ext/uchardet/test/ro/ibm852.txt +9 -0
  276. data/ext/uchardet/test/ro/iso-8859-16.txt +9 -0
  277. data/ext/uchardet/test/ro/utf-8.txt +9 -0
  278. data/ext/uchardet/test/ro/windows-1250.txt +9 -0
  279. data/ext/uchardet/test/ru/ibm855.txt +5 -0
  280. data/ext/uchardet/test/ru/ibm866.txt +11 -0
  281. data/ext/uchardet/test/ru/iso-8859-5.txt +3 -0
  282. data/ext/uchardet/test/ru/koi8-r.txt +1 -0
  283. data/ext/uchardet/test/ru/mac-cyrillic.txt +9 -0
  284. data/ext/uchardet/test/ru/windows-1251.txt +4 -0
  285. data/ext/uchardet/test/sk/ibm852.txt +3 -0
  286. data/ext/uchardet/test/sk/iso-8859-2.txt +3 -0
  287. data/ext/uchardet/test/sk/mac-centraleurope.txt +3 -0
  288. data/ext/uchardet/test/sk/utf-8.txt +3 -0
  289. data/ext/uchardet/test/sk/windows-1250.txt +3 -0
  290. data/ext/uchardet/test/sl/ibm852.txt +9 -0
  291. data/ext/uchardet/test/sl/iso-8859-16.txt +9 -0
  292. data/ext/uchardet/test/sl/iso-8859-2.txt +9 -0
  293. data/ext/uchardet/test/sl/mac-centraleurope.txt +9 -0
  294. data/ext/uchardet/test/sl/utf-8.txt +9 -0
  295. data/ext/uchardet/test/sl/windows-1250.txt +9 -0
  296. data/ext/uchardet/test/sv/iso-8859-1.txt +10 -0
  297. data/ext/uchardet/test/sv/utf-8.txt +10 -0
  298. data/ext/uchardet/test/sv/windows-1252.txt +10 -0
  299. data/ext/uchardet/test/th/iso-8859-11.txt +5 -0
  300. data/ext/uchardet/test/th/tis-620.txt +5 -0
  301. data/ext/uchardet/test/th/utf-8.txt +1 -0
  302. data/ext/uchardet/test/tr/iso-8859-3.txt +13 -0
  303. data/ext/uchardet/test/tr/iso-8859-9.txt +13 -0
  304. data/ext/uchardet/test/uchardet-tests.c +130 -0
  305. data/ext/uchardet/test/vi/utf-8.txt +4 -0
  306. data/ext/uchardet/test/vi/viscii.txt +4 -0
  307. data/ext/uchardet/test/vi/windows-1258.txt +4 -0
  308. data/ext/uchardet/test/zh/big5.txt +1 -0
  309. data/ext/uchardet/test/zh/euc-tw.txt +1 -0
  310. data/ext/uchardet/test/zh/gb18030.txt +1 -0
  311. data/ext/uchardet/test/zh/utf-8.txt +1 -0
  312. data/ext/uchardet/uchardet.doap +51 -0
  313. data/ext/uchardet/uchardet.pc.in +10 -0
  314. data/lib/cchardet.rb +56 -0
  315. data/lib/cchardet/lib_finder.rb +32 -0
  316. data/lib/cchardet/version.rb +5 -0
  317. metadata +362 -0
data/ext/uchardet/script/BuildLangModelLogs/LangLithuanianModel.log ADDED
@@ -0,0 +1,162 @@
1
+ = Logs of language model for Lithuanian (lt) =
2
+
3
+ - Generated by BuildLangModel.py
4
+ - Started: 2016-09-21 00:23:03.857157
5
+ - Maximum depth: 5
6
+ - Max number of pages: 100
7
+
8
+ == Parsed pages ==
9
+
10
+ Karūna (laivas) (revision 5080379)
11
+ 1650 (revision 4990868)
12
+ 1654 (revision 4991037)
13
+ 1664 (revision 4991048)
14
+ 1665 (revision 4991050)
15
+ 1668 (revision 4991052)
16
+ 1669 (revision 4991053)
17
+ 1672 (revision 4991056)
18
+ 1676 (revision 4991060)
19
+ 1718 (revision 4990914)
20
+ 1909 (revision 4990667)
21
+ 1928 (revision 4990262)
22
+ 1932 (revision 4990613)
23
+ 1956 (revision 4990635)
24
+ 1980 (revision 4990655)
25
+ Baltijos jūra (revision 5052833)
26
+ Burinis laivas (revision 4657401)
27
+ Flagmanas (laivas) (revision 5005271)
28
+ Grimzlė (revision 4487052)
29
+ Kalmaras (Švedija) (revision 4978519)
30
+ Karo laivas (revision 4726931)
31
+ Karolis XI (revision 4944621)
32
+ Karolis XII (revision 4915230)
33
+ Kilis (revision 4325533)
34
+ Koordinačių sistema (revision 5033980)
35
+ Laivo vėliava (revision 4986001)
36
+ Liepos 1 d. (revision 4910200)
37
+ Nyderlandai (revision 5080140)
38
+ Rugpjūčio 10 (revision 4910281)
39
+ Varytuvas (revision 4620792)
40
+ Vaza (laivas) (revision 5079282)
41
+ XVIII a. (revision 4896219)
42
+ XVII a. (revision 4768242)
43
+ Švedija (revision 5057665)
44
+ Švedų kalba (revision 4687559)
45
+ 1590 (revision 4990983)
46
+ 1596 (revision 4990989)
47
+ 1608 (revision 4991000)
48
+ 1610 (revision 4991002)
49
+ 1623 m. (revision 4991015)
50
+ 1634 m. (revision 4991026)
51
+ 1643 m. (revision 4990870)
52
+ 1644 m. (revision 4990872)
53
+ 1645 m. (revision 4990873)
54
+ 1646 m. (revision 4990874)
55
+ 1647 m. (revision 4913295)
56
+ 1648 m. (revision 4990875)
57
+ 1649 m. (revision 4990876)
58
+ 1651 m. (revision 4991035)
59
+ 1652 m. (revision 4991072)
60
+ 1653 m. (revision 4991036)
61
+ 1654 m. (revision 4991037)
62
+ 1655 m. (revision 4991038)
63
+ 1662 m. (revision 4991046)
64
+ 1668 m. (revision 4991052)
65
+ 1677 m. (revision 4991061)
66
+ 1702 (revision 4990595)
67
+ 1704 (revision 4990863)
68
+ 1722 (revision 4990918)
69
+ 1723 (revision 4990919)
70
+ 1737 (revision 4990931)
71
+ 2 tūkstantmetis (revision 4296407)
72
+ ATR (revision 5078529)
73
+ Abiejų Tautų Respublika (revision 5078529)
74
+ Adomas Freitagas (revision 4362991)
75
+ Anglų kalba (revision 4911240)
76
+ Armėnų kalendorius (revision 4817534)
77
+ Bahajų kalendorius (revision 4706296)
78
+ Bajorai (revision 5006456)
79
+ Berberų kalendorius (revision 4926904)
80
+ Birželio 21 (revision 4910142)
81
+ Bizantijos kalendorius (revision 4927623)
82
+ Budistų kalendorius (revision 4705734)
83
+ Dešimtmetis (revision 4296419)
84
+ Dominikonai (revision 4921895)
85
+ Dominikonų ordinas (revision 4921895)
86
+ Džohoro sultonatas (revision 4934526)
87
+ Džu Ihai (revision 4991072)
88
+ Džu Joulang (revision 4991072)
89
+ Emanuelis Vladislovas Tiškevičius Logoiskis (revision 4939239)
90
+ Filosofas (revision 5078172)
91
+ Gegužės 26 (revision 4910130)
92
+ Grafas (titulas) (revision 5008057)
93
+ Grigaliaus kalendorius (revision 5000317)
94
+ Hebrajų kalendorius (revision 4728592)
95
+ Imperatorius Go-Komijas (revision 4907057)
96
+ Inocentas X (revision 4905150)
97
+ Iraniečių kalendorius (revision 4964854)
98
+ Isaac Titsingh (revision 4990745)
99
+ Japonija (revision 5035249)
100
+ Japonijos imperatorius (revision 4720428)
101
+ Japonų kalendorius (revision 4956765)
102
+ John Churchill (revision 4903704)
103
+ Jonas Kazimieras Vaza (revision 5037754)
104
+ Jurgis Kasakauskis (revision 5047829)
105
+ Jurgis Kazimieras Ancuta (revision 5059404)
106
+ Jurgis Mikalojus Tiškevičius (revision 4939554)
107
+
108
+ == End of Parsed pages ==
109
+
110
+ - Wikipedia parsing ended at: 2016-09-21 00:25:34.773941
111
+
112
+ 60 characters appeared 353051 times.
113
+
114
+ First 38 characters:
115
+ [ 0] Char i: 13.032394753165974 %
116
+ [ 1] Char a: 11.167225131779828 %
117
+ [ 2] Char s: 8.586578143101137 %
118
+ [ 3] Char o: 7.018815978428046 %
119
+ [ 4] Char e: 5.525830545728521 %
120
+ [ 5] Char r: 5.469181506354606 %
121
+ [ 6] Char n: 5.142599794363987 %
122
+ [ 7] Char t: 5.105777918770942 %
123
+ [ 8] Char u: 4.270487833202568 %
124
+ [ 9] Char k: 3.9617505686147325 %
125
+ [10] Char l: 3.9051015292408184 %
126
+ [11] Char m: 3.359854525266888 %
127
+ [12] Char d: 3.0372382460324427 %
128
+ [13] Char v: 2.7270847554602593 %
129
+ [14] Char j: 2.4472385009531203 %
130
+ [15] Char p: 2.329125253858508 %
131
+ [16] Char g: 1.9427788053284087 %
132
+ [17] Char ė: 1.5657794482950054 %
133
+ [18] Char b: 1.5074309377398734 %
134
+ [19] Char y: 1.2236192504765602 %
135
+ [20] Char ų: 1.181698961339863 %
136
+ [21] Char š: 0.9630336693565519 %
137
+ [22] Char ž: 0.8171623929687212 %
138
+ [23] Char c: 0.5959478942135839 %
139
+ [24] Char č: 0.48010060869392807 %
140
+ [25] Char f: 0.428266737666796 %
141
+ [26] Char h: 0.42515104050123065 %
142
+ [27] Char z: 0.4010751987673169 %
143
+ [28] Char ū: 0.3685020011273159 %
144
+ [29] Char ą: 0.3526402701026197 %
145
+ [30] Char į: 0.29004308159444386 %
146
+ [31] Char ę: 0.14813723796278724 %
147
+ [32] Char x: 0.08752276583269838 %
148
+ [33] Char w: 0.059198246145740985 %
149
+ [34] Char ō: 0.01812769259965274 %
150
+ [35] Char ö: 0.008780601102956797 %
151
+ [36] Char é: 0.0076476203154785 %
152
+ [37] Char q: 0.007364375118608926 %
153
+
154
+ The first 38 characters have an accumulated ratio of 0.9996629382157253.
155
+
156
+ 1016 sequences found.
157
+
158
+ First 512 (typical positive ratio): 0.9928710196247589
159
+ Next 512 (512-1024): 0.008171623929687212
160
+ Rest: -4.85722573273506e-17
161
+
162
+ - Processing end: 2016-09-21 00:25:34.935858
data/ext/uchardet/script/BuildLangModelLogs/LangMalteseModel.log ADDED
@@ -0,0 +1,147 @@
1
+ = Logs of language model for Maltese (mt) =
2
+
3
+ - Generated by BuildLangModel.py
4
+ - Started: 2016-09-21 02:05:23.411546
5
+ - Maximum depth: 5
6
+ - Max number of pages: 100
7
+
8
+ == Parsed pages ==
9
+
10
+ Unjoni Ewropea (revision 246298)
11
+ 1951 (revision 229183)
12
+ 1952 (revision 229184)
13
+ 1957 (revision 229188)
14
+ 1958 (revision 229189)
15
+ 1973 (revision 223536)
16
+ 1979 (revision 243876)
17
+ 1981 (revision 205545)
18
+ 1985 (revision 216368)
19
+ 1986 (revision 231433)
20
+ 1990 (revision 237666)
21
+ 1992 (revision 244087)
22
+ 1995 (revision 214650)
23
+ 1 ta' Mejju (revision 245374)
24
+ 2007 (revision 214851)
25
+ 2013 (revision 245606)
26
+ Albanija (revision 243079)
27
+ Awstrija (revision 243627)
28
+ Awtonomija (revision 245824)
29
+ Ażores (revision 246298)
30
+ Bank Ċentrali Ewropew (revision 246298)
31
+ Belt kapitali (revision 237400)
32
+ Belġju (revision 244363)
33
+ Brussell (revision 243311)
34
+ Bulgarija (revision 243622)
35
+ Danimarka (revision 244419)
36
+ De facto (revision 215102)
37
+ Estonja (revision 243826)
38
+ European Free Trade Association (revision 246298)
39
+ Ewropa (revision 244177)
40
+ Ex Repubblika Jugoslava tal-Maċedonja (revision 246298)
41
+ Federazzjoni (revision 246226)
42
+ Finlandja (revision 245824)
43
+ Frankfurt (revision 243576)
44
+ Franza (revision 244461)
45
+ Greċja (revision 244423)
46
+ Groenlandja (revision 243829)
47
+ Indja (revision 244873)
48
+ Islanda (revision 243771)
49
+ Isle of Man (revision 246298)
50
+ Istitut tal-Unjoni Ewropea għall-Istudji dwar is-Sigurtà (revision 244412)
51
+ Italja (revision 246323)
52
+ Kilometru kwadru (revision 244871)
53
+ Komunitajiet Ewropej (revision 246298)
54
+ Komunità Ekonomika Ewropea (revision 246298)
55
+ Kroazja (revision 245711)
56
+ Kummissjoni Ewropea (revision 243311)
57
+ Kunsill Ewropew (revision 246298)
58
+ Kunsill tal-Ewropa (revision 243334)
59
+ Kunsill tal-Unjoni Ewropea (revision 243311)
60
+ Latvja (revision 245746)
61
+ Lista ta' pajjiżi skont id-daqs (revision 244419)
62
+ Lista ta' pajjiżi skont il-popolazzjoni (revision 246128)
63
+ Litwanja (revision 243114)
64
+ Liġijiet tal-Unjoni Ewropea (revision 246298)
65
+ Lussemburgu (revision 244239)
66
+ Lussemburgu (belt) (revision 243587)
67
+ Madejra (revision 243625)
68
+ Malta (revision 247210)
69
+ Montenegro (revision 243930)
70
+ Norveġja (revision 243829)
71
+ Olanda (revision 243989)
72
+ Organizzazzjoni Internazzjonali (revision 246724)
73
+ Pajjiżi l-Baxxi (revision 243989)
74
+ Pajjiżi membri tal-Unjoni Ewropea (revision 243625)
75
+ Pajjiżi ġirien li jdawru l-Unjoni Ewropea (revision 246298)
76
+ Parlament Ewropew (revision 243907)
77
+ Patt ta' Stabilità u Tkabbir (revision 246298)
78
+ Politika agrikola komuni (revision 244363)
79
+ Politika reġjonali tal-Unjoni Ewropea (revision 246298)
80
+ Polonja (revision 244530)
81
+ Portugall (revision 243625)
82
+ Relazzjonijiet ta' terzi pajjiżi ma l-UE (revision 246298)
83
+ Renju Unit (revision 247318)
84
+ Repubblika Federali tal-Ġermanja (revision 244859)
85
+ Repubblika tal-Irlanda (revision 243686)
86
+ Repubblika Ċeka (revision 246832)
87
+ Rumanija (revision 243623)
88
+ Segretarjat tal-Parlament Ewropew (revision 246298)
89
+ Serbja (revision 243728)
90
+ Slovakkja (revision 243831)
91
+ Slovenja (revision 244588)
92
+ Spanja (revision 246856)
93
+ Stati Uniti tal-Amerika (revision 243926)
94
+ Stati membri tal-Unjoni Ewropea (revision 243114)
95
+ Strasburgu (revision 243503)
96
+ Sui generis (revision 247150)
97
+ Suq komuni (revision 246298)
98
+ Svezja (revision 244871)
99
+
100
+ == End of Parsed pages ==
101
+
102
+ - Wikipedia parsing ended at: 2016-09-21 02:07:45.508113
103
+
104
+ 48 characters appeared 474337 times.
105
+
106
+ First 31 characters:
107
+ [ 0] Char a: 12.326257492036252 %
108
+ [ 1] Char i: 12.069899670487438 %
109
+ [ 2] Char t: 8.064941170518008 %
110
+ [ 3] Char l: 7.795301652622502 %
111
+ [ 4] Char e: 6.615971345267184 %
112
+ [ 5] Char n: 6.128132530247482 %
113
+ [ 6] Char r: 5.579577389071483 %
114
+ [ 7] Char u: 4.376424356522894 %
115
+ [ 8] Char o: 3.8337721915009797 %
116
+ [ 9] Char j: 3.7378488289971057 %
117
+ [10] Char m: 3.6084049947611088 %
118
+ [11] Char s: 3.3533120966738834 %
119
+ [12] Char k: 2.588033402412209 %
120
+ [13] Char d: 2.3173397816320462 %
121
+ [14] Char p: 2.0555006250830106 %
122
+ [15] Char b: 2.017131280081461 %
123
+ [16] Char f: 2.004692866042497 %
124
+ [17] Char ħ: 1.6372326004507345 %
125
+ [18] Char w: 1.4801712706366992 %
126
+ [19] Char g: 1.4763765002519307 %
127
+ [20] Char z: 1.3150987588992635 %
128
+ [21] Char ż: 0.9910675321554084 %
129
+ [22] Char h: 0.9750451683086075 %
130
+ [23] Char ġ: 0.7640137708000851 %
131
+ [24] Char ċ: 0.6723068198348432 %
132
+ [25] Char x: 0.5892435125237964 %
133
+ [26] Char v: 0.5668965313690478 %
134
+ [27] Char q: 0.5647883255997318 %
135
+ [28] Char c: 0.2759641352034524 %
136
+ [29] Char à: 0.10730767365817974 %
137
+ [30] Char y: 0.059029761540845424 %
138
+
139
+ The first 31 characters have an accumulated ratio of 0.9994708403519017.
140
+
141
+ 870 sequences found.
142
+
143
+ First 512 (typical positive ratio): 0.9959115850692665
144
+ Next 512 (512-1024): 2.108205769315908e-06
145
+ Rest: -4.423544863740858e-17
146
+
147
+ - Processing end: 2016-09-21 02:07:45.646198
data/ext/uchardet/script/BuildLangModelLogs/LangPolishModel.log ADDED
@@ -0,0 +1,154 @@
1
+ = Logs of language model for Polish (pl) =
2
+
3
+ - Generated by BuildLangModel.py
4
+ - Started: 2016-09-21 17:06:43.735784
5
+ - Maximum depth: 5
6
+ - Max number of pages: 100
7
+
8
+ == Parsed pages ==
9
+
10
+ Krasnyj Krym (revision 46884814)
11
+ 1913 (revision 46708474)
12
+ 1915 (revision 46743905)
13
+ 1917 (revision 46559521)
14
+ 1925 (revision 46809935)
15
+ 1928 (revision 46875978)
16
+ 1929 (revision 46760445)
17
+ 1935 (revision 46487358)
18
+ 1936 (revision 46874348)
19
+ 1939 (revision 46789269)
20
+ 1941 (revision 46856112)
21
+ 1942 (revision 46851808)
22
+ 1943 (revision 46768330)
23
+ 1944 (revision 46866229)
24
+ 1949 (revision 46882598)
25
+ 1953 (revision 46437607)
26
+ 1957 (revision 46591716)
27
+ 1959 (revision 46255886)
28
+ Admirał Butakow (revision 45993412)
29
+ Admirał Spiridow (revision 45993412)
30
+ Aparat torpedowy (revision 46633263)
31
+ Askold (revision 45787848)
32
+ Avro 504 (revision 44668646)
33
+ Ałmaz (1903) (revision 46472283)
34
+ Batumi (revision 46594611)
35
+ Bomba głębinowa (revision 46011227)
36
+ Brest (revision 45771242)
37
+ Burta (revision 45569092)
38
+ Cagliari (revision 46235605)
39
+ Cesariewicz (revision 40031486)
40
+ Czerwona Ukraina (revision 45993524)
41
+ Daty nowego i starego porządku (revision 45622575)
42
+ Drednot (revision 45789788)
43
+ Działo przeciwlotnicze (revision 45160162)
44
+ Flota Bałtycka Marynarki Wojennej Rosji (revision 45700667)
45
+ Gromoboj (revision 44328986)
46
+ Hulk (okręt) (revision 46020688)
47
+ II wojna światowa (revision 46871591)
48
+ I wojna światowa (revision 46869119)
49
+ Imperator Nikołaj I (okręt lotniczy) (revision 45520638)
50
+ Imperium Rosyjskie (revision 46604959)
51
+ Impierator Nikołaj I (1916) (revision 46534166)
52
+ Język rosyjski (revision 46433952)
53
+ Kanonierka (revision 41091952)
54
+ Kanonierki typu Ardagan (revision 46534166)
55
+ Kanonierki typu Bobr (revision 45788694)
56
+ Kanonierki typu Chiwiniec (revision 46534166)
57
+ Kanonierki typu Groziaszczij (revision 46534166)
58
+ Kanonierki typu Mandżur (revision 46534166)
59
+ Karabin maszynowy DSzK (revision 45587452)
60
+ Karabin maszynowy Vickers 12,7 mm (revision 44572918)
61
+ Kocioł parowy (revision 46716473)
62
+ Konstrukcyjna linia wodna (revision 37082620)
63
+ Kontrtorpedowce typu Biesstrasznyj (revision 46534166)
64
+ Kontrtorpedowce typu Brawyj (revision 46534166)
65
+ Kontrtorpedowce typu Grozowoj (revision 46534166)
66
+ Kontrtorpedowce typu Prytkij (revision 46534166)
67
+ Koń mechaniczny (revision 44722357)
68
+ Krab (1915) (revision 42791389)
69
+ Kronsztad (revision 46425497)
70
+ Krążownik lekki (revision 40661490)
71
+ Krążownik liniowy (revision 40601776)
72
+ Krążownik pancernopokładowy (revision 40055901)
73
+ Krążownik pancerny (revision 40324458)
74
+ Krążowniki lekkie typu Swietłana (revision 45993412)
75
+ Krążowniki liniowe typu Borodino (revision 45990866)
76
+ Krążowniki typu Admirał Nachimow (revision 45993521)
77
+ Krążowniki typu Bajan (revision 45991279)
78
+ Krążowniki typu Diana (revision 45991349)
79
+ Krążowniki typu Izumrud (revision 45991349)
80
+ Lend-Lease Act (revision 46877263)
81
+ Marynarka Wojenna Związku Socjalistycznych Republik Radzieckich (revision 45795993)
82
+ Maszyna sterowa (revision 28497888)
83
+ Mecidiye (1903) (revision 43956539)
84
+ Mila morska (revision 45754209)
85
+ Mina morska (revision 45781427)
86
+ Morze Czarne (revision 46729213)
87
+ Nadbudówka (revision 45292731)
88
+ Neapol (revision 46823083)
89
+ Niszczyciel (revision 45799132)
90
+ Niszczyciele rakietowe projektu 61 (revision 46498775)
91
+ Niszczyciele typu Finn (revision 46620140)
92
+ Niszczyciele typu Lejtienant Szestakow (revision 46620140)
93
+ Niszczyciele typu Ochotnik (revision 46620140)
94
+ Niszczyciele typu Ukraina (revision 46620140)
95
+ Noworosyjsk (revision 44721836)
96
+ Odessa (revision 45629804)
97
+ Oerlikon 20 mm (revision 45493862)
98
+ Okres międzywojenny (revision 46668249)
99
+ Okręt-baza wodnosamolotów (revision 45115462)
100
+
101
+ == End of Parsed pages ==
102
+
103
+ - Wikipedia parsing ended at: 2016-09-21 17:21:04.404471
104
+
105
+ 78 characters appeared 1159291 times.
106
+
107
+ First 37 characters:
108
+ [ 0] Char a: 9.685575062689178 %
109
+ [ 1] Char i: 8.815819324052374 %
110
+ [ 2] Char o: 7.920185699707839 %
111
+ [ 3] Char e: 6.871613770830621 %
112
+ [ 4] Char r: 5.8672067668945935 %
113
+ [ 5] Char n: 5.763608964444647 %
114
+ [ 6] Char s: 4.736688199942896 %
115
+ [ 7] Char k: 4.722196583946568 %
116
+ [ 8] Char z: 4.519227700378939 %
117
+ [ 9] Char w: 4.279512219106333 %
118
+ [10] Char t: 4.0191806888865695 %
119
+ [11] Char c: 3.6891513864939864 %
120
+ [12] Char y: 3.565282573572986 %
121
+ [13] Char p: 3.0190004062828053 %
122
+ [14] Char d: 2.851052928039638 %
123
+ [15] Char l: 2.7930002044352973 %
124
+ [16] Char m: 2.7530620008263673 %
125
+ [17] Char u: 2.348504387595522 %
126
+ [18] Char j: 1.881236031332944 %
127
+ [19] Char ł: 1.6885320424293815 %
128
+ [20] Char b: 1.394559260789569 %
129
+ [21] Char g: 1.3928340684090534 %
130
+ [22] Char h: 1.163901039514669 %
131
+ [23] Char ę: 0.8066136975099435 %
132
+ [24] Char ó: 0.5971753425153823 %
133
+ [25] Char ą: 0.563275312238256 %
134
+ [26] Char f: 0.5245447432956868 %
135
+ [27] Char ż: 0.4545019326467643 %
136
+ [28] Char ś: 0.39567287247119143 %
137
+ [29] Char ń: 0.3857530162832283 %
138
+ [30] Char ć: 0.1397405828217419 %
139
+ [31] Char v: 0.12455888987320698 %
140
+ [32] Char ź: 0.10204512930748191 %
141
+ [33] Char x: 0.05468859846233603 %
142
+ [34] Char é: 0.020961087423261287 %
143
+ [35] Char á: 0.01707940456710179 %
144
+ [36] Char q: 0.011386269711401192 %
145
+
146
+ The first 37 characters have an accumulated ratio of 0.9993892818972973.
147
+
148
+ 1321 sequences found.
149
+
150
+ First 512 (typical positive ratio): 0.9894531815946438
151
+ Next 512 (512-1024): 1.7251923805153322e-06
152
+ Rest: 0.0003530230403650733
153
+
154
+ - Processing end: 2016-09-21 17:21:04.878014