RubyGems - language_detector - Versions diffs - 0.1.0 - Mend

language_detector 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (172) hide show

data/README.rdoc +24 -0
data/Rakefile +18 -0
data/VERSION +1 -0
data/lib/language_detector.rb +232 -0
data/lib/model-fm.yml +52504 -0
data/lib/model-tc.yml +53985 -0
data/lib/textcat_ngrams/afrikaans.lm +400 -0
data/lib/textcat_ngrams/albanian.lm +400 -0
data/lib/textcat_ngrams/amharic-utf.lm +400 -0
data/lib/textcat_ngrams/arabic-iso8859_6.lm +400 -0
data/lib/textcat_ngrams/arabic-windows1256.lm +400 -0
data/lib/textcat_ngrams/armenian.lm +400 -0
data/lib/textcat_ngrams/basque.lm +400 -0
data/lib/textcat_ngrams/belarus-windows1251.lm +400 -0
data/lib/textcat_ngrams/bosnian.lm +400 -0
data/lib/textcat_ngrams/breton.lm +400 -0
data/lib/textcat_ngrams/bulgarian-iso8859_5.lm +400 -0
data/lib/textcat_ngrams/catalan.lm +400 -0
data/lib/textcat_ngrams/chinese-big5.lm +400 -0
data/lib/textcat_ngrams/chinese-gb2312.lm +400 -0
data/lib/textcat_ngrams/croatian-ascii.lm +400 -0
data/lib/textcat_ngrams/czech-iso8859_2.lm +400 -0
data/lib/textcat_ngrams/danish.lm +400 -0
data/lib/textcat_ngrams/dutch.lm +400 -0
data/lib/textcat_ngrams/english.lm +400 -0
data/lib/textcat_ngrams/esperanto.lm +400 -0
data/lib/textcat_ngrams/estonian.lm +400 -0
data/lib/textcat_ngrams/finnish.lm +400 -0
data/lib/textcat_ngrams/french.lm +400 -0
data/lib/textcat_ngrams/frisian.lm +400 -0
data/lib/textcat_ngrams/georgian.lm +400 -0
data/lib/textcat_ngrams/german.lm +400 -0
data/lib/textcat_ngrams/greek-iso8859-7.lm +400 -0
data/lib/textcat_ngrams/hebrew-iso8859_8.lm +400 -0
data/lib/textcat_ngrams/hindi.lm +400 -0
data/lib/textcat_ngrams/hungarian.lm +400 -0
data/lib/textcat_ngrams/icelandic.lm +400 -0
data/lib/textcat_ngrams/indonesian.lm +400 -0
data/lib/textcat_ngrams/irish.lm +400 -0
data/lib/textcat_ngrams/italian.lm +400 -0
data/lib/textcat_ngrams/japanese-euc_jp.lm +400 -0
data/lib/textcat_ngrams/japanese-shift_jis.lm +400 -0
data/lib/textcat_ngrams/korean.lm +400 -0
data/lib/textcat_ngrams/latin.lm +400 -0
data/lib/textcat_ngrams/latvian.lm +400 -0
data/lib/textcat_ngrams/lithuanian.lm +400 -0
data/lib/textcat_ngrams/malay.lm +400 -0
data/lib/textcat_ngrams/manx.lm +400 -0
data/lib/textcat_ngrams/marathi.lm +400 -0
data/lib/textcat_ngrams/mingo.lm +400 -0
data/lib/textcat_ngrams/nepali.lm +400 -0
data/lib/textcat_ngrams/norwegian.lm +400 -0
data/lib/textcat_ngrams/persian.lm +400 -0
data/lib/textcat_ngrams/polish.lm +400 -0
data/lib/textcat_ngrams/portuguese.lm +400 -0
data/lib/textcat_ngrams/quechua.lm +400 -0
data/lib/textcat_ngrams/romanian.lm +400 -0
data/lib/textcat_ngrams/rumantsch.lm +400 -0
data/lib/textcat_ngrams/russian-iso8859_5.lm +400 -0
data/lib/textcat_ngrams/russian-koi8_r.lm +400 -0
data/lib/textcat_ngrams/russian-windows1251.lm +400 -0
data/lib/textcat_ngrams/sanskrit.lm +400 -0
data/lib/textcat_ngrams/scots.lm +400 -0
data/lib/textcat_ngrams/scots_gaelic.lm +400 -0
data/lib/textcat_ngrams/serbian-ascii.lm +400 -0
data/lib/textcat_ngrams/slovak-ascii.lm +400 -0
data/lib/textcat_ngrams/slovak-windows1250.lm +400 -0
data/lib/textcat_ngrams/slovenian-ascii.lm +400 -0
data/lib/textcat_ngrams/slovenian-iso8859_2.lm +400 -0
data/lib/textcat_ngrams/spanish.lm +400 -0
data/lib/textcat_ngrams/swahili.lm +400 -0
data/lib/textcat_ngrams/swedish.lm +400 -0
data/lib/textcat_ngrams/tagalog.lm +400 -0
data/lib/textcat_ngrams/tamil.lm +400 -0
data/lib/textcat_ngrams/thai.lm +400 -0
data/lib/textcat_ngrams/turkish.lm +400 -0
data/lib/textcat_ngrams/ukrainian-koi8_u.lm +400 -0
data/lib/textcat_ngrams/vietnamese.lm +400 -0
data/lib/textcat_ngrams/welsh.lm +400 -0
data/lib/textcat_ngrams/yiddish-utf.lm +400 -0
data/lib/training_data/ar-utf8.txt +54 -0
data/lib/training_data/bg-utf8.txt +26 -0
data/lib/training_data/cs-utf8.txt +48 -0
data/lib/training_data/da-utf8.txt +159 -0
data/lib/training_data/de-utf8.txt +569 -0
data/lib/training_data/el-utf8.txt +48 -0
data/lib/training_data/en-utf8.txt +81 -0
data/lib/training_data/es-utf8.txt +185 -0
data/lib/training_data/et-utf8.txt +50 -0
data/lib/training_data/fa-utf8.txt +42 -0
data/lib/training_data/fi-utf8.txt +83 -0
data/lib/training_data/fr-utf8.txt +191 -0
data/lib/training_data/fy-utf8.txt +22 -0
data/lib/training_data/ga-utf8.txt +109 -0
data/lib/training_data/he-utf8.txt +116 -0
data/lib/training_data/hi-utf8.txt +49 -0
data/lib/training_data/hr-utf8.txt +80 -0
data/lib/training_data/hu-utf8.txt +87 -0
data/lib/training_data/io-utf8.txt +41 -0
data/lib/training_data/is-utf8.txt +94 -0
data/lib/training_data/it-utf8.txt +228 -0
data/lib/training_data/ja-utf8.txt +200 -0
data/lib/training_data/ko-utf8.txt +147 -0
data/lib/training_data/nl-utf8.txt +215 -0
data/lib/training_data/no-utf8.txt +281 -0
data/lib/training_data/pl-utf8.txt +120 -0
data/lib/training_data/pt-utf8.txt +214 -0
data/lib/training_data/ro-utf8.txt +66 -0
data/lib/training_data/ru-utf8.txt +310 -0
data/lib/training_data/sl-utf8.txt +263 -0
data/lib/training_data/sv-utf8.txt +174 -0
data/lib/training_data/th-utf8.txt +49 -0
data/lib/training_data/tk-utf8.txt +101 -0
data/lib/training_data/todo/af.txt +114 -0
data/lib/training_data/todo/amharic-utf.txt +95 -0
data/lib/training_data/todo/arabic-windows1256.txt +157 -0
data/lib/training_data/todo/armenian.txt +86 -0
data/lib/training_data/todo/basque.txt +136 -0
data/lib/training_data/todo/belarus-windows1251.txt +97 -0
data/lib/training_data/todo/bosnian.txt +97 -0
data/lib/training_data/todo/breton.txt +159 -0
data/lib/training_data/todo/bulgarian-iso8859_5.txt +115 -0
data/lib/training_data/todo/catalan.txt +93 -0
data/lib/training_data/todo/croatian-ascii.txt +104 -0
data/lib/training_data/todo/esperanto.txt +95 -0
data/lib/training_data/todo/estonian.txt +218 -0
data/lib/training_data/todo/frisian.txt +99 -0
data/lib/training_data/todo/georgian.txt +86 -0
data/lib/training_data/todo/greek-iso8859-7.txt +139 -0
data/lib/training_data/todo/hawaian.txt +108 -0
data/lib/training_data/todo/hebrew-iso8859_8.txt +79 -0
data/lib/training_data/todo/hindi.txt +77 -0
data/lib/training_data/todo/hungarian.txt +102 -0
data/lib/training_data/todo/icelandic.txt +131 -0
data/lib/training_data/todo/indonesian.txt +93 -0
data/lib/training_data/todo/irish.txt +209 -0
data/lib/training_data/todo/latin.txt +120 -0
data/lib/training_data/todo/latvian.txt +126 -0
data/lib/training_data/todo/lithuanian.txt +99 -0
data/lib/training_data/todo/malay.txt +108 -0
data/lib/training_data/todo/manx.txt +78 -0
data/lib/training_data/todo/marathi.txt +100 -0
data/lib/training_data/todo/mf.txt +100 -0
data/lib/training_data/todo/middle_frisian.txt +102 -0
data/lib/training_data/todo/mingo.txt +146 -0
data/lib/training_data/todo/nepali.txt +131 -0
data/lib/training_data/todo/persian.txt +73 -0
data/lib/training_data/todo/quechua.txt +108 -0
data/lib/training_data/todo/romanian.txt +103 -0
data/lib/training_data/todo/rumantsch.txt +110 -0
data/lib/training_data/todo/sanskrit.txt +135 -0
data/lib/training_data/todo/scots.txt +490 -0
data/lib/training_data/todo/scots_gaelic.txt +93 -0
data/lib/training_data/todo/serbian-ascii.txt +121 -0
data/lib/training_data/todo/slovak-ascii.txt +102 -0
data/lib/training_data/todo/slovak-windows1250.txt +115 -0
data/lib/training_data/todo/slovenian-ascii.txt +100 -0
data/lib/training_data/todo/slovenian-iso8859_2.txt +96 -0
data/lib/training_data/todo/sq.txt +110 -0
data/lib/training_data/todo/swahili.txt +120 -0
data/lib/training_data/todo/tagalog.txt +135 -0
data/lib/training_data/todo/tamil.txt +123 -0
data/lib/training_data/todo/turkish.txt +117 -0
data/lib/training_data/todo/ukrainian-koi8_r.txt +214 -0
data/lib/training_data/todo/vietnamese.txt +92 -0
data/lib/training_data/todo/welsh.txt +148 -0
data/lib/training_data/todo/yiddish-utf.txt +83 -0
data/lib/training_data/uk-utf8.txt +75 -0
data/lib/training_data/vi-utf8.txt +47 -0
data/lib/training_data/zh-utf8.txt +228 -0
data/test/language_detector_test.rb +78 -0
metadata +232 -0

data/lib/textcat_ngrams/japanese-euc_jp.lm ADDED

@@ -0,0 +1,400 @@
+�	 78988
+�	 57258
+_	 34456
+�	 22280
+�	 18223
+�	 11290
+�	 11237
+�	 10621
+�	 9502
+�	 8859
+�	 8812
+��	 8266
+�	 8180
+�	 7520
+�	 7325
+�	 6982
+�	 6930
+�	 6929
+�	 6896
+�	 6865
+��	 6858
+�	 6261
+��	 6067
+��	 5926
+���	 5888
+�	 5878
+�	 5806
+�	 5562
+�	 5426
+�	 5314
+��	 5228
+�	 5011
+�	 4836
+��	 4775
+�	 4385
+�_	 4323
+��	 4310
+��	 4305
+��_	 4294
+�	 4253
+��	 4090
+��	 4075
+�	 4032
+�	 4029
+�	 3975
+�	 3947
+�	 3923
+�	 3853
+�	 3845
+�	 3759
+�	 3583
+��	 3485
+��	 3475
+�	 3367
+�	 3366
+�	 3366
+��	 3314
+�	 3313
+�	 3282
+�	 3273
+��	 3255
+�	 3207
+å	 3188
+��	 3182
+��	 3175
+�å	 3170
+��	 3131
+�	 3058
+��	 3035
+��	 3024
+���	 3000
+롣	 2952
+롣_	 2948
+�롣	 2947
+�롣_	 2943
+�	 2905
+�	 2893
+��	 2865
+��	 2848
+�	 2819
+Ȥ	 2791
+��	 2776
+���	 2749
+����	 2730
+����	 2729
+�����	 2729
+�	 2697
+�	 2689
+�	 2545
+��	 2543
+��	 2525
+���	 2473
+�����	 2459
+��	 2399
+.	 2395
+�	 2381
+,	 2380
+��	 2357
+��	 2346
+�	 2340
+�	 2328
+��	 2326
+��	 2316
+�	 2314
+�	 2293
+����	 2283
+���	 2283
+��	 2253
+._	 2251
+��	 2227
+���	 2222
+��	 2200
+_�	 2189
+Ǥ	 2172
+��	 2145
+ӥ	 2137
+��	 2112
+����	 2110
+���	 2110
+�	 2100
+��	 2061
+��	 2027
+�	 2017
+��	 2011
+��	 2007
+�ӥ	 2002
+ˤ	 1995
+���	 1983
+����	 1983
+�	 1947
+�	 1913
+�	 1861
+��	 1847
+��	 1834
+��	 1834
+��	 1826
+�	 1795
+���	 1791
+�	 1782
+�	 1782
+�ˤ	 1763
+�Ȥ	 1760
+��	 1716
+��	 1713
+�	 1709
+�	 1696
+ʤ	 1691
+��	 1688
+_�	 1686
+��	 1679
+���	 1669
+��	 1668
+�	 1666
+��	 1659
+�Ǥ	 1655
+��	 1638
+�	 1637
+��	 1612
+���	 1598
+���	 1584
+����	 1584
+�ʤ	 1570
+��	 1568
+��	 1565
+��	 1562
+��	 1554
+�	 1544
+Ƥ	 1541
+ƥ	 1538
+��	 1526
+��	 1525
+��	 1523
+���	 1523
+�	 1517
+��	 1510
+���	 1509
+-	 1490
+��	 1476
+��	 1465
+��	 1465
+�	 1464
+���	 1454
+��	 1454
+��å	 1453
+�å��	 1453
+�����	 1453
+�å�	 1453
+å��	 1453
+����	 1453
+�å�	 1453
+����	 1453
+���	 1453
+�å	 1453
+å�	 1453
+å���	 1453
+���	 1453
+�����	 1452
+�	 1450
+�	 1445
+�	 1441
+��	 1420
+�	 1419
+�	 1418
+���	 1415
+����	 1413
+����	 1413
+�����	 1413
+���	 1413
+��	 1413
+��	 1411
+ӥ�	 1406
+��ӥ	 1406
+���ӥ	 1406
+�ӥ�	 1406
+����	 1406
+���	 1406
+��	 1403
+�����	 1403
+��ӥ�	 1403
+�ƥ	 1396
+���	 1392
+���	 1383
+�	 1378
+��	 1372
+_�	 1368
+Υ	 1358
+T	 1354
+ϡ	 1341
+��	 1338
+��	 1337
+��	 1325
+�	 1323
+Τ	 1314
+�Υ	 1312
+���	 1311
+����	 1311
+Ԥ	 1308
+)	 1302
+_.	 1297
+�	 1294
+��	 1294
+���	 1292
+��	 1289
+���	 1284
+�Ƥ	 1284
+��	 1273
+��	 1260
+_._	 1244
+�	 1244
+�	 1238
+���	 1211
+��	 1190
+�	 1182
+_�	 1182
+���	 1152
+ȥ	 1151
+��	 1145
+����	 1130
+ǽ	 1129
+ե	 1126
+�	 1120
+�ե	 1113
+�ϡ	 1105
+��	 1103
+�	 1101
+�	 1084
+��	 1080
+��	 1062
+�	 1060
+��	 1051
+��	 1050
+���	 1043
+��	 1033
+��	 1033
+���	 1033
+���	 1032
+����	 1032
+ϡ�	 1030
+�ϡ�	 1030
+Ǥ�	 1022
+�Ǥ�	 1021
+Ǥ��	 1021
+�Ǥ��	 1020
+ǥ	 1012
+�	 1006
+��	 997
+��	 995
+��	 991
+ͣ	 988
+���	 985
+��	 984
+�ͣ	 983
+��	 981
+��	 976
+���	 974
+�Τ	 971
+���	 971
+��	 962
+���	 955
+�	 955
+�	 954
+�ȥ	 953
+�ǥ	 951
+��	 942
+��	 929
+��	 920
+�	 918
+�	 909
+��	 904
+�	 904
+�Ԥ	 903
+����	 900
+���	 900
+�_	 894
+��	 893
+-_	 892
+�,	 887
+�Ȥ	 886
+��,	 886
+��_	 884
+��	 881
+��	 881
+(	 878
+C	 877
+Ǥ���	 877
+I	 877
+��	 875
+��롣	 875
+i	 872
+��	 865
+��	 860
+֤	 857
+�	 856
+��	 855
+��	 855
+��	 846
+��	 841
+ޤ	 834
+���	 834
+e	 825
+���	 825
+��	 819
+����	 818
+���	 818
+��	 808
+��	 802
+�ǥ�	 796
+ǥ�	 796
+_�	 792
+a	 776
+�å�	 771
+å�	 771
+����	 770
+���	 770
+��	 767
+Ѥ	 748
+���	 743
+��	 743
+�ǥ��	 741
+ǥ��	 741
+�_	 738
+���	 732
+N	 729
+��	 728
+��	 728
+��	 727
+��	 724
+�롣	 722
+���	 718
+ˤ�	 716
+ˤ�	 716
+�ˤ�	 715
+�����	 715
+����	 715
+�ˤ�	 715
+ץ	 715
+�ץ	 711
+�����	 710
+����	 710
+�����	 710
+Ĥ	 705
+�	 702
+�	 701
+_��	 699
+���	 696
+��_	 694
+��	 694
+���	 692
+����	 692
+����	 692
+���	 686
+�Ѥ	 685
+�ʤ�	 685
+ʤ�	 685
+��	 684
+Ƥ�	 684
+n	 682
+�	 682
+��	 681
+��	 681

data/lib/textcat_ngrams/japanese-shift_jis.lm ADDED

@@ -0,0 +1,400 @@
+�	 9970
+�	 2204
+_	 1555
+�	 1214
+�	 891
+A	 885
+�	 846
+�	 836
+�A	 831
+�	 818
+�	 815
+��	 790
+�	 739
+�	 729
+�	 618
+�	 607
+�	 589
+�	 568
+�	 566
+�	 555
+�	 552
+�	 528
+��	 527
+��	 500
+�	 489
+�	 481
+�	 461
+�	 460
+�	 453
+�	 449
+��	 449
+�	 445
+�	 440
+B	 419
+�	 411
+�B	 405
+�	 404
+��	 404
+��	 401
+�	 396
+�	 389
+��	 387
+�	 368
+�	 366
+��	 363
+��	 361
+���	 357
+��	 352
+��	 352
+��	 349
+��	 345
+�	 330
+��	 321
+�	 320
+��	 320
+���	 306
+��	 305
+�	 292
+�	 284
+@	 282
+_�	 281
+܂	 275
+��	 268
+v	 263
+�܂	 258
+Ȃ	 256
+�	 251
+�Ȃ	 240
+��	 239
+�	 238
+Ă	 234
+�	 232
+Ƃ	 231
+ł	 226
+�@	 220
+��	 217
+�Ƃ	 214
+�ł	 211
+_�	 210
+�Ă	 205
+��	 204
+��	 203
+���	 199
+�	 198
+̂	 195
+�	 194
+�	 194
+���	 182
+�	 182
+�	 181
+�	 180
+ɂ	 180
+A�	 176
+�	 176
+�	 173
+�ɂ	 172
+�̂	 172
+v�	 169
+�	 169
+��	 167
+�	 167
+�B_	 166
+N	 166
+B_	 166
+O	 165
+�	 165
+�A�	 165
+��	 164
+P	 163
+�	 163
+��	 159
+�	 159
+_�@	 157
+��	 156
+���	 155
+[	 153
+��	 153
+X	 153
+�	 148
+i	 148
+��	 148
+��	 147
+��B	 146
+��	 146
+���B	 145
+j	 142
+���	 142
+�P	 141
+���	 141
+����	 141
+͂	 141
+��	 141
+���	 137
+V	 137
+��	 135
+��	 135
+�	 134
+u	 133
+�[	 132
+��	 131
+l	 129
+�	 129
+s	 128
+�	 128
+�	 125
+��	 125
+S	 125
+g	 124
+��	 123
+��	 123
+�	 123
+���	 122
+��	 122
+�	 120
+��	 120
+��	 120
+�	 120
+\	 119
+́	 119
+�u	 119
+�v	 119
+��	 119
+߂	 118
+W	 118
+�́	 117
+�܂�	 117
+R	 117
+���	 117
+܂�	 117
+�	 114
+�	 114
+�	 113
+�	 113
+{	 113
+�N	 112
+��	 111
+�	 111
+���	 111
+o	 108
+�܂��	 107
+܂��	 107
+Q	 106
+�	 106
+��	 105
+܂��B	 104
+��	 103
+��	 103
+�	 103
+��	 103
+�	 102
+��	 102
+���	 100
+��	 100
+��	 100
+�߂	 99
+����	 98
+���	 98
+�	 97
+��	 96
+�	 96
+��	 96
+I	 95
+[�	 95
+�[�	 95
+�́A	 94
+��	 94
+́A	 94
+��	 93
+�Ă�	 92
+�	 92
+Ă�	 92
+��	 92
+E	 92
+�Q	 91
+�	 90
+w	 90
+�Ă��	 89
+Ă��	 89
+�	 89
+�	 88
+��	 87
+x	 87
+��	 86
+P�	 86
+�	 86
+���	 86
+n	 86
+�	 85
+���	 85
+����	 85
+���	 84
+��	 84
+�	 84
+�͂	 84
+�	 84
+��	 83
+���B	 83
+��B	 83
+�	 83
+��	 83
+�	 82
+�	 82
+�v�	 80
+��	 80
+�P�	 80
+���	 79
+�	 79
+A�	 77
+�	 77
+�	 77
+��	 77
+���Ă	 77
+��Ă	 77
+�	 76
+B�	 75
+�X	 75
+����	 75
+���	 74
+c	 74
+�	 74
+�O	 73
+��	 73
+��	 72
+i�	 72
+���	 71
+��	 71
+�B�	 71
+��	 70
+��	 70
+�	 70
+y	 70
+ǂ	 69
+T	 69
+p	 69
+�	 68
+��	 68
+���	 68
+���B_	 67
+�l	 67
+Z	 67
+��	 67
+��	 67
+��	 67
+�	 67
+��B_	 67
+�	 67
+X�	 66
+�	 66
+���	 66
+�	 66
+��	 66
+A�	 66
+��	 65
+���	 65
+�A�	 65
+U	 65
+��	 64
+�	 64
+��	 63
+���	 62
+C	 62
+����	 62
+��	 62
+�R	 62
+��	 62
+���	 62
+����	 62
+���	 62
+_�	 61
+�܂	 61
+���	 61
+��	 61
+^	 61
+��	 61
+��܂	 61
+�	 61
+l�	 61
+��	 61
+��	 60
+���v	 60
+��	 60
+���	 60
+�v	 60
+��v	 60
+�	 60
+��	 60
+���	 60
+�	 60
+�A�	 60
+�	 59
+��	 59
+�Ȃ�	 59
+Ȃ�	 59
+��	 59
+��	 59
+W�	 59
+��	 58
+�i	 58
+����	 58
+Q�	 58
+�j	 58
+��	 58
+N�	 58
+�{	 57
+���	 57
+��	 57
+�	 57
+���	 57
+̐	 56
+Ɂ	 56
+���A	 56
+e	 56
+�g	 56
+��A	 56
+ʂ	 56
+@�	 56
+	 55
+z	 55
+�I	 55
+Y	 55
+ł�	 55
+��	 55
+�A	 55
+A�	 55
+���	 54
+��	 54
+�X	 54
+���	 54
+t	 54
+�ł�	 54
+�j	 54
+���	 53
+�Ɂ	 53
+��Ƃ	 53
+�	 53
+���Ƃ	 53
+�	 53
+I�	 53
+��	 53
+��	 53
+�A�	 53
+�U	 53
+~	 53
+��	 52
+A�	 52
+x�	 52
+��A	 52
+�̐	 52
+�n	 52
+S�	 51
+�A�	 51
+b	 51
+j�	 51
+�ǂ	 51
+�Q�	 51
+���	 51
+�E	 51
+�	 51
+A�	 51
+��	 51