RubyGems - language_detector - Versions diffs - 0.1.0 - Mend

language_detector 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (172) hide show

data/README.rdoc +24 -0
data/Rakefile +18 -0
data/VERSION +1 -0
data/lib/language_detector.rb +232 -0
data/lib/model-fm.yml +52504 -0
data/lib/model-tc.yml +53985 -0
data/lib/textcat_ngrams/afrikaans.lm +400 -0
data/lib/textcat_ngrams/albanian.lm +400 -0
data/lib/textcat_ngrams/amharic-utf.lm +400 -0
data/lib/textcat_ngrams/arabic-iso8859_6.lm +400 -0
data/lib/textcat_ngrams/arabic-windows1256.lm +400 -0
data/lib/textcat_ngrams/armenian.lm +400 -0
data/lib/textcat_ngrams/basque.lm +400 -0
data/lib/textcat_ngrams/belarus-windows1251.lm +400 -0
data/lib/textcat_ngrams/bosnian.lm +400 -0
data/lib/textcat_ngrams/breton.lm +400 -0
data/lib/textcat_ngrams/bulgarian-iso8859_5.lm +400 -0
data/lib/textcat_ngrams/catalan.lm +400 -0
data/lib/textcat_ngrams/chinese-big5.lm +400 -0
data/lib/textcat_ngrams/chinese-gb2312.lm +400 -0
data/lib/textcat_ngrams/croatian-ascii.lm +400 -0
data/lib/textcat_ngrams/czech-iso8859_2.lm +400 -0
data/lib/textcat_ngrams/danish.lm +400 -0
data/lib/textcat_ngrams/dutch.lm +400 -0
data/lib/textcat_ngrams/english.lm +400 -0
data/lib/textcat_ngrams/esperanto.lm +400 -0
data/lib/textcat_ngrams/estonian.lm +400 -0
data/lib/textcat_ngrams/finnish.lm +400 -0
data/lib/textcat_ngrams/french.lm +400 -0
data/lib/textcat_ngrams/frisian.lm +400 -0
data/lib/textcat_ngrams/georgian.lm +400 -0
data/lib/textcat_ngrams/german.lm +400 -0
data/lib/textcat_ngrams/greek-iso8859-7.lm +400 -0
data/lib/textcat_ngrams/hebrew-iso8859_8.lm +400 -0
data/lib/textcat_ngrams/hindi.lm +400 -0
data/lib/textcat_ngrams/hungarian.lm +400 -0
data/lib/textcat_ngrams/icelandic.lm +400 -0
data/lib/textcat_ngrams/indonesian.lm +400 -0
data/lib/textcat_ngrams/irish.lm +400 -0
data/lib/textcat_ngrams/italian.lm +400 -0
data/lib/textcat_ngrams/japanese-euc_jp.lm +400 -0
data/lib/textcat_ngrams/japanese-shift_jis.lm +400 -0
data/lib/textcat_ngrams/korean.lm +400 -0
data/lib/textcat_ngrams/latin.lm +400 -0
data/lib/textcat_ngrams/latvian.lm +400 -0
data/lib/textcat_ngrams/lithuanian.lm +400 -0
data/lib/textcat_ngrams/malay.lm +400 -0
data/lib/textcat_ngrams/manx.lm +400 -0
data/lib/textcat_ngrams/marathi.lm +400 -0
data/lib/textcat_ngrams/mingo.lm +400 -0
data/lib/textcat_ngrams/nepali.lm +400 -0
data/lib/textcat_ngrams/norwegian.lm +400 -0
data/lib/textcat_ngrams/persian.lm +400 -0
data/lib/textcat_ngrams/polish.lm +400 -0
data/lib/textcat_ngrams/portuguese.lm +400 -0
data/lib/textcat_ngrams/quechua.lm +400 -0
data/lib/textcat_ngrams/romanian.lm +400 -0
data/lib/textcat_ngrams/rumantsch.lm +400 -0
data/lib/textcat_ngrams/russian-iso8859_5.lm +400 -0
data/lib/textcat_ngrams/russian-koi8_r.lm +400 -0
data/lib/textcat_ngrams/russian-windows1251.lm +400 -0
data/lib/textcat_ngrams/sanskrit.lm +400 -0
data/lib/textcat_ngrams/scots.lm +400 -0
data/lib/textcat_ngrams/scots_gaelic.lm +400 -0
data/lib/textcat_ngrams/serbian-ascii.lm +400 -0
data/lib/textcat_ngrams/slovak-ascii.lm +400 -0
data/lib/textcat_ngrams/slovak-windows1250.lm +400 -0
data/lib/textcat_ngrams/slovenian-ascii.lm +400 -0
data/lib/textcat_ngrams/slovenian-iso8859_2.lm +400 -0
data/lib/textcat_ngrams/spanish.lm +400 -0
data/lib/textcat_ngrams/swahili.lm +400 -0
data/lib/textcat_ngrams/swedish.lm +400 -0
data/lib/textcat_ngrams/tagalog.lm +400 -0
data/lib/textcat_ngrams/tamil.lm +400 -0
data/lib/textcat_ngrams/thai.lm +400 -0
data/lib/textcat_ngrams/turkish.lm +400 -0
data/lib/textcat_ngrams/ukrainian-koi8_u.lm +400 -0
data/lib/textcat_ngrams/vietnamese.lm +400 -0
data/lib/textcat_ngrams/welsh.lm +400 -0
data/lib/textcat_ngrams/yiddish-utf.lm +400 -0
data/lib/training_data/ar-utf8.txt +54 -0
data/lib/training_data/bg-utf8.txt +26 -0
data/lib/training_data/cs-utf8.txt +48 -0
data/lib/training_data/da-utf8.txt +159 -0
data/lib/training_data/de-utf8.txt +569 -0
data/lib/training_data/el-utf8.txt +48 -0
data/lib/training_data/en-utf8.txt +81 -0
data/lib/training_data/es-utf8.txt +185 -0
data/lib/training_data/et-utf8.txt +50 -0
data/lib/training_data/fa-utf8.txt +42 -0
data/lib/training_data/fi-utf8.txt +83 -0
data/lib/training_data/fr-utf8.txt +191 -0
data/lib/training_data/fy-utf8.txt +22 -0
data/lib/training_data/ga-utf8.txt +109 -0
data/lib/training_data/he-utf8.txt +116 -0
data/lib/training_data/hi-utf8.txt +49 -0
data/lib/training_data/hr-utf8.txt +80 -0
data/lib/training_data/hu-utf8.txt +87 -0
data/lib/training_data/io-utf8.txt +41 -0
data/lib/training_data/is-utf8.txt +94 -0
data/lib/training_data/it-utf8.txt +228 -0
data/lib/training_data/ja-utf8.txt +200 -0
data/lib/training_data/ko-utf8.txt +147 -0
data/lib/training_data/nl-utf8.txt +215 -0
data/lib/training_data/no-utf8.txt +281 -0
data/lib/training_data/pl-utf8.txt +120 -0
data/lib/training_data/pt-utf8.txt +214 -0
data/lib/training_data/ro-utf8.txt +66 -0
data/lib/training_data/ru-utf8.txt +310 -0
data/lib/training_data/sl-utf8.txt +263 -0
data/lib/training_data/sv-utf8.txt +174 -0
data/lib/training_data/th-utf8.txt +49 -0
data/lib/training_data/tk-utf8.txt +101 -0
data/lib/training_data/todo/af.txt +114 -0
data/lib/training_data/todo/amharic-utf.txt +95 -0
data/lib/training_data/todo/arabic-windows1256.txt +157 -0
data/lib/training_data/todo/armenian.txt +86 -0
data/lib/training_data/todo/basque.txt +136 -0
data/lib/training_data/todo/belarus-windows1251.txt +97 -0
data/lib/training_data/todo/bosnian.txt +97 -0
data/lib/training_data/todo/breton.txt +159 -0
data/lib/training_data/todo/bulgarian-iso8859_5.txt +115 -0
data/lib/training_data/todo/catalan.txt +93 -0
data/lib/training_data/todo/croatian-ascii.txt +104 -0
data/lib/training_data/todo/esperanto.txt +95 -0
data/lib/training_data/todo/estonian.txt +218 -0
data/lib/training_data/todo/frisian.txt +99 -0
data/lib/training_data/todo/georgian.txt +86 -0
data/lib/training_data/todo/greek-iso8859-7.txt +139 -0
data/lib/training_data/todo/hawaian.txt +108 -0
data/lib/training_data/todo/hebrew-iso8859_8.txt +79 -0
data/lib/training_data/todo/hindi.txt +77 -0
data/lib/training_data/todo/hungarian.txt +102 -0
data/lib/training_data/todo/icelandic.txt +131 -0
data/lib/training_data/todo/indonesian.txt +93 -0
data/lib/training_data/todo/irish.txt +209 -0
data/lib/training_data/todo/latin.txt +120 -0
data/lib/training_data/todo/latvian.txt +126 -0
data/lib/training_data/todo/lithuanian.txt +99 -0
data/lib/training_data/todo/malay.txt +108 -0
data/lib/training_data/todo/manx.txt +78 -0
data/lib/training_data/todo/marathi.txt +100 -0
data/lib/training_data/todo/mf.txt +100 -0
data/lib/training_data/todo/middle_frisian.txt +102 -0
data/lib/training_data/todo/mingo.txt +146 -0
data/lib/training_data/todo/nepali.txt +131 -0
data/lib/training_data/todo/persian.txt +73 -0
data/lib/training_data/todo/quechua.txt +108 -0
data/lib/training_data/todo/romanian.txt +103 -0
data/lib/training_data/todo/rumantsch.txt +110 -0
data/lib/training_data/todo/sanskrit.txt +135 -0
data/lib/training_data/todo/scots.txt +490 -0
data/lib/training_data/todo/scots_gaelic.txt +93 -0
data/lib/training_data/todo/serbian-ascii.txt +121 -0
data/lib/training_data/todo/slovak-ascii.txt +102 -0
data/lib/training_data/todo/slovak-windows1250.txt +115 -0
data/lib/training_data/todo/slovenian-ascii.txt +100 -0
data/lib/training_data/todo/slovenian-iso8859_2.txt +96 -0
data/lib/training_data/todo/sq.txt +110 -0
data/lib/training_data/todo/swahili.txt +120 -0
data/lib/training_data/todo/tagalog.txt +135 -0
data/lib/training_data/todo/tamil.txt +123 -0
data/lib/training_data/todo/turkish.txt +117 -0
data/lib/training_data/todo/ukrainian-koi8_r.txt +214 -0
data/lib/training_data/todo/vietnamese.txt +92 -0
data/lib/training_data/todo/welsh.txt +148 -0
data/lib/training_data/todo/yiddish-utf.txt +83 -0
data/lib/training_data/uk-utf8.txt +75 -0
data/lib/training_data/vi-utf8.txt +47 -0
data/lib/training_data/zh-utf8.txt +228 -0
data/test/language_detector_test.rb +78 -0
metadata +232 -0

data/lib/textcat_ngrams/ukrainian-koi8_u.lm ADDED

@@ -0,0 +1,400 @@
+_	 21726
+�	 4832
+�	 4080
+�	 3253
+�	 2775
+�	 2747
+�	 2642
+�	 2481
+�	 2373
+�	 2066
+�	 1929
+�	 1806
+�	 1797
+�	 1760
+�	 1605
+�	 1347
+�	 1345
+�	 1061
+�	 1060
+�_	 971
+�_	 965
+�_	 916
+_�	 902
+�	 881
+_�	 869
+�_	 847
+�	 843
+�_	 810
+�	 760
+_�	 696
+.	 676
+�	 654
+�	 647
+_�	 623
+"	 621
+_�	 606
+��	 582
+_�	 565
+�	 543
+_�	 514
+��	 480
+._	 476
+��	 464
+��	 459
+צ	 457
+�_	 457
+�_	 451
+�	 450
+�_	 449
+_�	 443
+�	 438
+��	 431
+�	 428
+_�	 419
+��	 418
+��	 414
+��	 413
+�	 410
+��	 400
+��	 396
+�_	 394
+��	 377
+��	 375
+��	 373
+��	 358
+��	 356
+�_	 354
+�	 354
+_�	 350
+��	 346
+,	 343
+��	 341
+��	 340
+��	 336
+��	 333
+��	 331
+�	 328
+��	 326
+,_	 323
+��	 322
+��	 321
+��	 319
+��	 318
+��	 317
+�	 314
+_��	 314
+�	 312
+��	 310
+��	 310
+��	 305
+�	 295
+��	 295
+Φ	 294
+_��	 289
+_��	 288
+��	 288
+�	 287
+��	 282
+��	 282
+��_	 280
+��	 274
+�	 273
+��	 268
+_�	 268
+�	 267
+��	 266
+"_	 264
+���	 254
+��	 252
+��	 252
+_"	 251
+��_	 251
+��_	 249
+�_	 248
+�	 246
+��_	 245
+I	 245
+��	 244
+_�	 241
+�_	 239
+_��	 239
+��	 235
+��	 235
+�	 233
+_�	 232
+_�_	 228
+��	 228
+��	 228
+�_	 226
+��	 224
+_��	 221
+��	 219
+_��	 218
+��	 216
+��	 215
+��	 214
+��	 213
+_צ	 210
+��	 209
+_�	 207
+�_	 206
+��	 206
+��_	 204
+��	 203
+_�	 203
+�_	 202
+��	 201
+̦	 200
+�	 199
+���_	 199
+�	 197
+��_	 196
+��	 193
+��	 193
+_��_	 190
+_�	 190
+��	 188
+��	 187
+_�	 186
+��	 186
+��	 185
+��	 184
+��_	 184
+_�	 176
+_�_	 174
+_�	 174
+��	 173
+��	 173
+�	 173
+��	 170
+_��	 169
+��	 168
+��	 167
+�	 166
+��	 166
+��	 163
+�	 163
+Ҧ	 163
+�	 163
+�	 162
+��_	 161
+��	 160
+Φ_	 160
+��	 160
+��_	 159
+צ�	 159
+_�	 158
+Ԧ	 155
+_��	 155
+*	 154
+��	 153
+��	 153
+_*	 152
+��_	 152
+��_	 152
+��	 152
+_��	 151
+�_	 151
+��	 151
+*_	 151
+��	 151
+_*_	 150
+�	 149
+_�_	 149
+_��	 148
+_�	 147
+Ħ	 147
+��	 146
+��	 145
+_��_	 144
+���	 144
+��_	 143
+��	 142
+_��	 141
+��	 141
+��	 140
+�_	 139
+��	 138
+Ц	 137
+���	 137
+��	 136
+��	 136
+_��_	 136
+��	 135
+æ	 135
+_�	 134
+_��	 133
+��	 133
+��_	 133
+��	 132
+��	 131
+��	 130
+��	 129
+_��	 129
+��	 129
+_��	 129
+�_	 128
+��	 128
+��	 128
+ͦ	 128
+_��	 127
+��	 126
+�_	 126
+�צ	 123
+��	 123
+��	 122
+˦	 121
+:	 119
+_�	 118
+��	 118
+��	 116
+Ӧ	 116
+��	 115
+ϧ	 115
+��	 115
+��_	 115
+���	 114
+��_	 113
+_�	 113
+���	 113
+��	 113
+���	 112
+��	 112
+?	 111
+��	 110
+���	 109
+_�	 109
+��	 109
+��	 109
+��	 109
+��_	 108
+��	 108
+_�	 106
+�	 106
+��_	 106
+��	 106
+_צ�	 106
+��	 106
+��	 105
+¦	 105
+���	 105
+���	 104
+��	 104
+��	 104
+���	 104
+��	 103
+��	 103
+_�	 102
+_��	 102
+�_	 102
+��_	 102
+���	 102
+_�	 101
+_��	 101
+���	 101
+�	 100
+_��	 100
+��	 99
+���_	 99
+���	 99
+_��_	 98
+���	 98
+��	 98
+���	 97
+��_	 97
+��	 96
+��	 94
+���_	 94
+_�	 94
+��	 93
+!	 93
+_��	 93
+..	 93
+���	 92
+��	 92
+���	 92
+��	 92
+_���	 91
+��	 91
+���	 91
+_���	 91
+��	 91
+��	 89
+�	 89
+�_	 88
+���_	 88
+��	 88
+��	 86
+��	 86
+��	 86
+����	 86
+��	 85
+�_	 85
+��	 84
+���	 84
+���	 84
+_����	 83
+_Ц	 83
+_��	 82
+��	 82
+��	 82
+�	 81
+�	 81
+_�	 81
+��_	 81
+��	 81
+��	 81
+���	 81
+��_	 80
+_�	 79
+_�	 79
+��	 78
+��	 78
+��	 78
+��	 78
+��	 78
+_���	 77
+��_	 77
+��_	 77
+��	 77
+��	 77
+_�	 77
+_�	 77
+_��	 76
+_��	 76
+��_	 76
+��	 76
+��	 75
+��	 75
+���	 75
+��	 74
+���	 74
+���	 74
+����	 74
+ϧ_	 73
+_���	 73
+���	 72
+".	 72
+."	 72
+����	 72
+��	 72
+��	 71
+��_	 71
+��	 71
+�	 71
+_��	 71
+���	 71
+���	 71
+��_	 70
+_�	 70
+���	 70
+_��	 69
+�Φ	 69
+��	 68
+���	 68
+����	 68
+��	 68
+��_	 68
+����_	 68
+���	 68

data/lib/textcat_ngrams/vietnamese.lm ADDED

@@ -0,0 +1,400 @@
+_	 88044
+n	 17000
+h	 12823
+t	 9071
+i	 8490
+c	 8394
+g	 8035
+ng	 6718
+_t	 6352
+_c	 5234
+a	 5083
+g_	 4883
+ng_	 4882
+_n	 4379
+n_	 4365
+i_	 4365
+u	 4149
+m	 3648
+�	 3635
+nh	 3480
+o	 3451
+�	 3193
+_�	 3168
+r	 3011
+l	 2692
+�	 2659
+,	 2328
+,_	 2295
+c_	 2279
+_l	 2263
+ch	 2226
+v	 2161
+th	 2158
+a_	 2132
+_th	 2113
+y	 2111
+.	 2107
+t_	 2106
+h_	 2074
+_v	 2057
+_h	 1888
+_m	 1834
+_ch	 1813
+s	 1749
+nh_	 1715
+u_	 1679
+�	 1668
+�_	 1649
+tr	 1611
+b	 1589
+_tr	 1581
+_nh	 1541
+_b	 1530
+m_	 1505
+p	 1483
+._	 1455
+k	 1429
+_s	 1415
+o_	 1380
+y_	 1371
+_k	 1367
+_ng	 1348
+�	 1343
+e	 1296
+�	 1208
+�	 1199
+�	 1181
+�	 1139
+�	 1132
+T	 1126
+hi	 1121
+�	 1081
+�	 1058
+�	 1055
+d	 1040
+_g	 1039
+kh	 1034
+_kh	 1027
+_T	 994
+�i	 967
+l�	 943
+_l�	 941
+"	 926
+�	 916
+ܩ	 901
+an	 893
+_d	 888
+�	 877
+�i_	 859
+on	 853
+�	 848
+N	 846
+�	 820
+�_	 813
+�	 801
+�n	 795
+ph	 776
+_p	 772
+_ph	 752
+�	 752
+�	 734
+h�	 728
+ho	 715
+v�	 709
+_v�	 703
+g�	 702
+�	 700
+_r	 696
+H	 693
+_l�_	 691
+l�_	 691
+ܩi	 684
+�n	 679
+�n	 677
+�ng	 676
+ha	 672
+gi	 663
+C	 656
+_gi	 655
+�	 654
+gܩi	 646
+gܩ	 646
+ܩi_	 624
+�ng_	 610
+ng�	 610
+_ng�	 608
+�c	 606
+�	 601
+ngܩ	 600
+ngܩi	 600
+_ngܩ	 598
+gܩi_	 594
+�a	 587
+c�	 586
+_c�	 585
+�t	 585
+c�a	 585
+�n_	 585
+_c�a	 584
+c�	 583
+�a_	 582
+_c�	 581
+c�a_	 581
+�	 581
+_c�a_	 580
+�t_	 578
+_N	 574
+c�_	 574
+�	 573
+_c�_	 572
+i�	 568
+�	 562
+�	 562
+m�	 557
+_C	 553
+_m�	 551
+p_	 540
+�	 540
+m�t	 538
+m�t_	 534
+_m�t	 532
+�n	 528
+_m�t_	 528
+ti	 526
+i�	 525
+�	 517
+�	 512
+�nh	 500
+..	 500
+�	 497
+v�_	 497
+_v�_	 496
+�	 491
+q	 490
+qu	 490
+_H	 487
+_q	 484
+_qu	 484
+ong	 481
+ong_	 471
+h�	 471
+x	 470
+h�	 468
+�	 466
+_"	 460
+ܮ	 456
+ro	 453
+�nh_	 445
+�_	 434
+_x	 434
+�_	 427
+�	 423
+_ti	 423
+in	 422
+�n	 421
+"_	 418
+i�	 415
+�n	 413
+ron	 412
+V	 411
+rong	 410
+�i	 410
+rong_	 409
+�c_	 405
+�	 400
+�n	 399
+h�n	 398
+�i	 395
+ay	 390
+_V	 387
+h�	 382
+h�	 380
+�y	 377
+�t	 376
+uy	 374
+��	 374
+_��	 373
+�i_	 372
+c�	 371
+_c�	 367
+n�	 366
+�_	 366
+h�	 366
+_n�	 365
+ra	 363
+h�ng	 362
+ho_	 359
+�n_	 356
+�n	 356
+�	 355
+�i_	 354
+ai	 352
+hu	 352
+cho	 352
+ܮc	 351
+�c	 351
+�n	 351
+_cho	 349
+tro	 347
+�t	 347
+�o	 347
+_tro	 346
+_tron	 346
+�	 346
+tron	 346
+trong	 346
+M	 345
+kh�	 340
+�u	 338
+_kh�	 338
+cho_	 337
+_cho_	 336
+h�ng_	 336
+ay_	 333
+ch_	 332
+��	 331
+�t_	 331
+(	 331
+_(	 330
+�	 329
+_��	 329
+�ܮ	 328
+_�ܮc	 328
+)	 328
+�ܮc	 328
+_�ܮ	 328
+kh�n	 324
+_�	 324
+_kh�n	 322
+��_	 322
+_��_	 320
+�	 318
+�n	 318
+kh�ng	 316
+�o_	 316
+ܮc_	 316
+�c_	 316
+nh�	 315
+�_	 315
+�ng	 313
+�ng_	 313
+�n	 313
+_nh�	 313
+Th	 312
+h�	 311
+�	 310
+h�n	 310
+h�ng_	 310
+h�ng	 310
+i�	 309
+gh	 309
+�u	 307
+ta	 307
+anh	 307
+�	 307
+ai_	 306
+�ng	 306
+�y_	 304
+�ܮc_	 298
+��	 295
+S	 295
+:	 294
+�t_	 294
+e_	 294
+:_	 290
+�_	 289
+ua	 288
+�n	 288
+�i	 286
+�u_	 285
+_nh�	 285
+nh�	 285
+nh�ng	 285
+nh�n	 285
+_nh�n	 285
+_Th	 285
+�n_	 284
+�n	 283
+�	 281
+_M	 281
+A	 281
+�c	 278
+L	 277
+�	 277
+��n	 276
+_ha	 276
+n,	 275
+�ng	 275
+��ng	 275
+an_	 272
+�i_	 271
+i�u	 269
+s�	 269
+n,_	 268
+�_	 268
+on_	 267
+_s�	 267
+c�c	 265
+�ng_	 265
+_c�c	 264
+anh_	 264
+ngh	 264
+_ta	 263
+hi_	 262
+h�n	 261
+�_	 261
+�u_	 261
+�n	 260
+�	 260
+_ngh	 257
+ia	 255
+�	 252
+...	 252
+�u	 251
+�c	 251
+i,	 249
+i�u_	 248
+nhi	 247
+B	 246
+i,_	 245
+�ng	 244
+�_	 244
+co	 244
+_nhi	 244
+�	 244
+ܧ	 244
+i�n	 243
+D	 243
+Tr	 241
+_S	 240
+h�_	 239
+�n	 237
+h�	 236
+h�	 233
+K	 233
+�ng_	 232
+�i	 232
+��ng_	 232
+�n_	 231
+_co	 231
+�n_	 229
+h�	 229
+�i	 229
+y�	 229
+�n	 229
+�n	 228
+�ng	 228
+_L	 227
+�ng_	 226
+inh	 226
+�i	 225
+m�	 225
+_�i	 225
+ng,	 224
+ang	 224
+P	 224
+�n	 224
+g,	 224
+g,_	 223
+_v�	 223