RubyGems - language_detector - Versions diffs - 0.1.0 - Mend

language_detector 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (172) hide show

data/README.rdoc +24 -0
data/Rakefile +18 -0
data/VERSION +1 -0
data/lib/language_detector.rb +232 -0
data/lib/model-fm.yml +52504 -0
data/lib/model-tc.yml +53985 -0
data/lib/textcat_ngrams/afrikaans.lm +400 -0
data/lib/textcat_ngrams/albanian.lm +400 -0
data/lib/textcat_ngrams/amharic-utf.lm +400 -0
data/lib/textcat_ngrams/arabic-iso8859_6.lm +400 -0
data/lib/textcat_ngrams/arabic-windows1256.lm +400 -0
data/lib/textcat_ngrams/armenian.lm +400 -0
data/lib/textcat_ngrams/basque.lm +400 -0
data/lib/textcat_ngrams/belarus-windows1251.lm +400 -0
data/lib/textcat_ngrams/bosnian.lm +400 -0
data/lib/textcat_ngrams/breton.lm +400 -0
data/lib/textcat_ngrams/bulgarian-iso8859_5.lm +400 -0
data/lib/textcat_ngrams/catalan.lm +400 -0
data/lib/textcat_ngrams/chinese-big5.lm +400 -0
data/lib/textcat_ngrams/chinese-gb2312.lm +400 -0
data/lib/textcat_ngrams/croatian-ascii.lm +400 -0
data/lib/textcat_ngrams/czech-iso8859_2.lm +400 -0
data/lib/textcat_ngrams/danish.lm +400 -0
data/lib/textcat_ngrams/dutch.lm +400 -0
data/lib/textcat_ngrams/english.lm +400 -0
data/lib/textcat_ngrams/esperanto.lm +400 -0
data/lib/textcat_ngrams/estonian.lm +400 -0
data/lib/textcat_ngrams/finnish.lm +400 -0
data/lib/textcat_ngrams/french.lm +400 -0
data/lib/textcat_ngrams/frisian.lm +400 -0
data/lib/textcat_ngrams/georgian.lm +400 -0
data/lib/textcat_ngrams/german.lm +400 -0
data/lib/textcat_ngrams/greek-iso8859-7.lm +400 -0
data/lib/textcat_ngrams/hebrew-iso8859_8.lm +400 -0
data/lib/textcat_ngrams/hindi.lm +400 -0
data/lib/textcat_ngrams/hungarian.lm +400 -0
data/lib/textcat_ngrams/icelandic.lm +400 -0
data/lib/textcat_ngrams/indonesian.lm +400 -0
data/lib/textcat_ngrams/irish.lm +400 -0
data/lib/textcat_ngrams/italian.lm +400 -0
data/lib/textcat_ngrams/japanese-euc_jp.lm +400 -0
data/lib/textcat_ngrams/japanese-shift_jis.lm +400 -0
data/lib/textcat_ngrams/korean.lm +400 -0
data/lib/textcat_ngrams/latin.lm +400 -0
data/lib/textcat_ngrams/latvian.lm +400 -0
data/lib/textcat_ngrams/lithuanian.lm +400 -0
data/lib/textcat_ngrams/malay.lm +400 -0
data/lib/textcat_ngrams/manx.lm +400 -0
data/lib/textcat_ngrams/marathi.lm +400 -0
data/lib/textcat_ngrams/mingo.lm +400 -0
data/lib/textcat_ngrams/nepali.lm +400 -0
data/lib/textcat_ngrams/norwegian.lm +400 -0
data/lib/textcat_ngrams/persian.lm +400 -0
data/lib/textcat_ngrams/polish.lm +400 -0
data/lib/textcat_ngrams/portuguese.lm +400 -0
data/lib/textcat_ngrams/quechua.lm +400 -0
data/lib/textcat_ngrams/romanian.lm +400 -0
data/lib/textcat_ngrams/rumantsch.lm +400 -0
data/lib/textcat_ngrams/russian-iso8859_5.lm +400 -0
data/lib/textcat_ngrams/russian-koi8_r.lm +400 -0
data/lib/textcat_ngrams/russian-windows1251.lm +400 -0
data/lib/textcat_ngrams/sanskrit.lm +400 -0
data/lib/textcat_ngrams/scots.lm +400 -0
data/lib/textcat_ngrams/scots_gaelic.lm +400 -0
data/lib/textcat_ngrams/serbian-ascii.lm +400 -0
data/lib/textcat_ngrams/slovak-ascii.lm +400 -0
data/lib/textcat_ngrams/slovak-windows1250.lm +400 -0
data/lib/textcat_ngrams/slovenian-ascii.lm +400 -0
data/lib/textcat_ngrams/slovenian-iso8859_2.lm +400 -0
data/lib/textcat_ngrams/spanish.lm +400 -0
data/lib/textcat_ngrams/swahili.lm +400 -0
data/lib/textcat_ngrams/swedish.lm +400 -0
data/lib/textcat_ngrams/tagalog.lm +400 -0
data/lib/textcat_ngrams/tamil.lm +400 -0
data/lib/textcat_ngrams/thai.lm +400 -0
data/lib/textcat_ngrams/turkish.lm +400 -0
data/lib/textcat_ngrams/ukrainian-koi8_u.lm +400 -0
data/lib/textcat_ngrams/vietnamese.lm +400 -0
data/lib/textcat_ngrams/welsh.lm +400 -0
data/lib/textcat_ngrams/yiddish-utf.lm +400 -0
data/lib/training_data/ar-utf8.txt +54 -0
data/lib/training_data/bg-utf8.txt +26 -0
data/lib/training_data/cs-utf8.txt +48 -0
data/lib/training_data/da-utf8.txt +159 -0
data/lib/training_data/de-utf8.txt +569 -0
data/lib/training_data/el-utf8.txt +48 -0
data/lib/training_data/en-utf8.txt +81 -0
data/lib/training_data/es-utf8.txt +185 -0
data/lib/training_data/et-utf8.txt +50 -0
data/lib/training_data/fa-utf8.txt +42 -0
data/lib/training_data/fi-utf8.txt +83 -0
data/lib/training_data/fr-utf8.txt +191 -0
data/lib/training_data/fy-utf8.txt +22 -0
data/lib/training_data/ga-utf8.txt +109 -0
data/lib/training_data/he-utf8.txt +116 -0
data/lib/training_data/hi-utf8.txt +49 -0
data/lib/training_data/hr-utf8.txt +80 -0
data/lib/training_data/hu-utf8.txt +87 -0
data/lib/training_data/io-utf8.txt +41 -0
data/lib/training_data/is-utf8.txt +94 -0
data/lib/training_data/it-utf8.txt +228 -0
data/lib/training_data/ja-utf8.txt +200 -0
data/lib/training_data/ko-utf8.txt +147 -0
data/lib/training_data/nl-utf8.txt +215 -0
data/lib/training_data/no-utf8.txt +281 -0
data/lib/training_data/pl-utf8.txt +120 -0
data/lib/training_data/pt-utf8.txt +214 -0
data/lib/training_data/ro-utf8.txt +66 -0
data/lib/training_data/ru-utf8.txt +310 -0
data/lib/training_data/sl-utf8.txt +263 -0
data/lib/training_data/sv-utf8.txt +174 -0
data/lib/training_data/th-utf8.txt +49 -0
data/lib/training_data/tk-utf8.txt +101 -0
data/lib/training_data/todo/af.txt +114 -0
data/lib/training_data/todo/amharic-utf.txt +95 -0
data/lib/training_data/todo/arabic-windows1256.txt +157 -0
data/lib/training_data/todo/armenian.txt +86 -0
data/lib/training_data/todo/basque.txt +136 -0
data/lib/training_data/todo/belarus-windows1251.txt +97 -0
data/lib/training_data/todo/bosnian.txt +97 -0
data/lib/training_data/todo/breton.txt +159 -0
data/lib/training_data/todo/bulgarian-iso8859_5.txt +115 -0
data/lib/training_data/todo/catalan.txt +93 -0
data/lib/training_data/todo/croatian-ascii.txt +104 -0
data/lib/training_data/todo/esperanto.txt +95 -0
data/lib/training_data/todo/estonian.txt +218 -0
data/lib/training_data/todo/frisian.txt +99 -0
data/lib/training_data/todo/georgian.txt +86 -0
data/lib/training_data/todo/greek-iso8859-7.txt +139 -0
data/lib/training_data/todo/hawaian.txt +108 -0
data/lib/training_data/todo/hebrew-iso8859_8.txt +79 -0
data/lib/training_data/todo/hindi.txt +77 -0
data/lib/training_data/todo/hungarian.txt +102 -0
data/lib/training_data/todo/icelandic.txt +131 -0
data/lib/training_data/todo/indonesian.txt +93 -0
data/lib/training_data/todo/irish.txt +209 -0
data/lib/training_data/todo/latin.txt +120 -0
data/lib/training_data/todo/latvian.txt +126 -0
data/lib/training_data/todo/lithuanian.txt +99 -0
data/lib/training_data/todo/malay.txt +108 -0
data/lib/training_data/todo/manx.txt +78 -0
data/lib/training_data/todo/marathi.txt +100 -0
data/lib/training_data/todo/mf.txt +100 -0
data/lib/training_data/todo/middle_frisian.txt +102 -0
data/lib/training_data/todo/mingo.txt +146 -0
data/lib/training_data/todo/nepali.txt +131 -0
data/lib/training_data/todo/persian.txt +73 -0
data/lib/training_data/todo/quechua.txt +108 -0
data/lib/training_data/todo/romanian.txt +103 -0
data/lib/training_data/todo/rumantsch.txt +110 -0
data/lib/training_data/todo/sanskrit.txt +135 -0
data/lib/training_data/todo/scots.txt +490 -0
data/lib/training_data/todo/scots_gaelic.txt +93 -0
data/lib/training_data/todo/serbian-ascii.txt +121 -0
data/lib/training_data/todo/slovak-ascii.txt +102 -0
data/lib/training_data/todo/slovak-windows1250.txt +115 -0
data/lib/training_data/todo/slovenian-ascii.txt +100 -0
data/lib/training_data/todo/slovenian-iso8859_2.txt +96 -0
data/lib/training_data/todo/sq.txt +110 -0
data/lib/training_data/todo/swahili.txt +120 -0
data/lib/training_data/todo/tagalog.txt +135 -0
data/lib/training_data/todo/tamil.txt +123 -0
data/lib/training_data/todo/turkish.txt +117 -0
data/lib/training_data/todo/ukrainian-koi8_r.txt +214 -0
data/lib/training_data/todo/vietnamese.txt +92 -0
data/lib/training_data/todo/welsh.txt +148 -0
data/lib/training_data/todo/yiddish-utf.txt +83 -0
data/lib/training_data/uk-utf8.txt +75 -0
data/lib/training_data/vi-utf8.txt +47 -0
data/lib/training_data/zh-utf8.txt +228 -0
data/test/language_detector_test.rb +78 -0
metadata +232 -0

data/lib/textcat_ngrams/arabic-windows1256.lm ADDED

@@ -0,0 +1,400 @@
+_	 3974
+�	 1252
+�	 958
+�	 626
+�	 534
+�	 498
+��	 496
+�	 482
+_�	 473
+_��	 408
+�	 345
+.	 335
+�	 326
+�	 303
+�	 269
+�	 260
+�_	 258
+�	 244
+_�	 235
+�	 202
+�_	 200
+..	 188
+�	 175
+�	 173
+�	 166
+�	 162
+�	 157
+�	 150
+�	 150
+_�	 145
+�_	 143
+._	 130
+��	 129
+�_	 129
+_�	 124
+�_	 120
+�	 117
+�	 116
+��	 115
+_�	 109
+�	 98
+...	 94
+�	 92
+_�	 92
+_�	 90
+��	 90
+�_	 90
+�_	 88
+��	 87
+���	 86
+��	 84
+_���	 83
+_�	 82
+�	 80
+�_	 79
+�	 77
+.._	 77
+..._	 77
+��	 72
+��	 71
+�_	 69
+��	 69
+�	 69
+��	 69
+�_	 68
+��	 68
+_�	 67
+��	 67
+��	 67
+�	 66
+�.	 66
+��	 65
+�	 64
+��	 63
+��	 63
+�	 63
+�_	 62
+��	 62
+���	 62
+�_	 62
+���	 61
+��	 60
+����	 60
+�	 60
+��_	 60
+_�_	 58
+_��	 58
+_�	 58
+��_	 57
+�..._	 56
+�..	 56
+�...	 56
+-_	 53
+-	 53
+_-	 52
+_-_	 52
+_��	 51
+��_	 50
+��	 50
+���	 50
+_��	 50
+��	 49
+��_	 48
+�	 48
+��	 47
+_�	 47
+�_	 47
+�_	 46
+_���	 45
+��_	 45
+��	 44
+�	 44
+��_	 44
+_��	 43
+_�	 43
+�_	 43
+��_	 43
+_��_	 43
+���_	 42
+����_	 42
+�	 42
+���	 41
+�	 41
+��	 41
+���_	 40
+_�	 40
+���	 40
+_��	 40
+��	 40
+��_	 39
+��	 39
+��	 39
+_���	 38
+��	 38
+��	 37
+��	 37
+_�	 37
+_��_	 36
+�_	 36
+��	 35
+_��	 35
+���	 35
+��	 35
+_��_	 35
+����	 35
+��	 34
+��_	 34
+�_	 34
+�	 34
+���	 34
+�����	 34
+����	 33
+�����	 33
+���	 33
+��	 33
+_�	 32
+��	 32
+��	 31
+���	 31
+��	 31
+��	 31
+��_	 31
+_�	 30
+��	 30
+���	 30
+��	 30
+_����	 30
+��	 30
+��	 30
+��	 30
+�	 30
+_�	 30
+�	 29
+��	 29
+��	 29
+��_	 29
+��	 29
+��	 29
+��	 29
+��	 29
+_��	 29
+�	 29
+��	 29
+_���_	 28
+���	 28
+_���	 28
+_���	 28
+��_	 28
+��	 27
+���_	 27
+_�	 27
+��	 27
+��	 27
+��	 27
+���	 27
+��	 27
+����	 27
+�_	 26
+_.	 26
+��_	 26
+��	 26
+_���	 26
+��	 26
+��_	 26
+��	 26
+���_	 26
+_._	 26
+��_	 25
+��	 25
+��	 25
+����_	 25
+��_	 25
+��	 24
+��	 24
+��	 24
+��	 24
+_��	 24
+��	 23
+���_	 23
+����	 23
+_����	 23
+��	 23
+��	 23
+���	 23
+��	 23
+���	 23
+��	 23
+���	 23
+����	 23
+����	 23
+���	 23
+�����	 23
+����_	 23
+_���	 23
+��	 23
+���	 23
+��	 23
+_���	 23
+���	 23
+���	 23
+����	 23
+_��	 23
+_����	 22
+�����	 22
+�	 22
+��	 22
+��	 22
+��_	 22
+_���	 22
+����	 22
+���_	 22
+����_	 22
+�_	 22
+_�	 21
+_��	 21
+���	 21
+��	 21
+��	 21
+��	 21
+���	 21
+��_	 21
+����	 21
+���	 21
+_����	 20
+��	 20
+��_	 20
+���	 20
+���	 20
+��_	 20
+��	 20
+��	 20
+_�	 20
+��	 20
+��	 20
+��	 19
+��	 19
+��	 19
+�_	 19
+��	 19
+��	 19
+���	 19
+��_	 19
+_��	 19
+��_	 19
+��	 19
+����_	 18
+��	 18
+����	 18
+���_	 18
+����_	 18
+��	 18
+��	 18
+���_	 18
+��_	 18
+��	 18
+���	 18
+��	 18
+_���	 18
+�����	 18
+_��	 18
+_����	 18
+����	 18
+���	 18
+��	 18
+��	 18
+�����	 18
+����	 18
+��	 18
+_��	 17
+���	 17
+���	 17
+��	 17
+��	 17
+�_	 17
+��_	 17
+���	 17
+�_	 17
+����_	 17
+��	 17
+��.	 17
+����	 17
+��	 17
+��	 17
+��	 17
+��	 17
+��	 17
+_��_	 17
+�����	 17
+��_	 17
+��	 16
+��	 16
+_��	 16
+��_	 16
+��_	 16
+�	 16
+_���	 16
+���	 16
+_��	 16
+_��	 16
+��	 16
+��	 16
+��	 16
+��	 16
+��	 16
+_���	 16
+��	 16
+�	 16
+���	 16
+��	 16
+��	 16
+_��	 15
+��	 15
+���_	 15
+����	 15
+���	 15
+_����	 15
+���	 15
+��_	 15
+���	 15
+_��	 15
+_���	 15
+����	 15
+�_	 15
+��	 15
+_����	 15
+��	 15
+���	 15
+��	 15
+_�	 15
+��	 15
+��	 14
+��	 14
+��_	 14
+!	 14
+���_	 14
+����	 14
+_��	 14
+�����	 14
+��	 14
+��_	 14
+���	 14
+��	 14
+����	 14
+_���	 14
+��	 14
+���	 14
+���	 14
+_��_	 14
+�_	 14
+��	 14
+��.	 14
+��	 14
+��	 13
+���_	 13
+���	 13
+_���	 13
+��	 13
+_��	 13
+��	 13
+���	 13

data/lib/textcat_ngrams/armenian.lm ADDED

@@ -0,0 +1,400 @@
+_	 49002
+�	 12626
+�	 9480
+�	 8485
+�	 7623
+�	 7071
+�	 7010
+�	 6370
+�	 6320
+�	 5012
+�	 4656
+�	 4607
+�	 4506
+�	 4210
+�	 3775
+��	 3655
+�	 3369
+�	 3320
+�	 3208
+�	 3016
+��	 2832
+��	 2755
+�	 2552
+�	 2519
+,	 2502
+,_	 2492
+�_	 2177
+��	 2099
+�	 2034
+�	 2007
+�	 1918
+��	 1748
+��	 1703
+�	 1605
+�_	 1587
+�	 1558
+�	 1528
+ϳ	 1502
+:	 1483
+:_	 1482
+�	 1420
+_�	 1403
+�_	 1382
+�_	 1335
+�	 1307
+�	 1294
+ݻ	 1281
+�	 1274
+�	 1264
+��	 1229
+�	 1218
+��	 1184
+�	 1170
+��	 1154
+_�	 1139
+��	 1130
+_�	 1121
+�	 1085
+	 1065
+_�	 1046
+_�	 1036
+��	 1019
+�	 1017
+��	 1001
+���	 993
+��	 987
+�_	 964
+��	 962
+_�	 961
+�	 952
+ٳ	 951
+�_	 945
+�	 941
+�	 938
+ѳ	 934
+�	 929
+ݻ�	 919
+��	 918
+�	 915
+�_	 911
+��	 910
+ݳ	 910
+��_	 882
+��	 863
+��	 846
+�	 834
+��	 827
+��	 824
+�	 818
+��	 817
+��	 804
+���	 804
+_�	 791
+���	 787
+����	 787
+�	 781
+��	 753
+_�	 750
+�_	 748
+_�	 747
+�	 747
+���	 744
+_�	 732
+��	 730
+��	 723
+�	 710
+�_	 710
+��	 709
+_ѳ	 708
+_�	 705
+�	 703
+�	 702
+��	 697
+�	 695
+�_	 684
+_�	 683
+��	 682
+�	 672
+�ϳ	 671
+��	 669
+��	 666
+�	 659
+�	 659
+�	 658
+��	 658
+�	 653
+��	 649
+�	 641
+��	 641
+�	 640
+���	 636
+��	 635
+��	 631
+_�	 626
+��	 623
+۳	 620
+�_	 616
+_�	 615
+��_	 614
+���_	 613
+��	 601
+ϳ�	 597
+��	 596
+��_	 590
+��	 590
+��	 582
+�_	 581
+���	 576
+��	 574
+��	 574
+�	 566
+�	 564
+�	 555
+_�	 532
+��	 527
+��	 527
+��	 526
+�_	 524
+���	 517
+_�	 511
+�	 510
+_�	 507
+��	 500
+��	 500
+����	 497
+�	 496
+�	 488
+�	 485
+��	 480
+_��	 480
+�_	 478
+��_	 477
+��	 476
+���	 475
+����	 475
+�ϳ�	 473
+_�	 469
+�_	 466
+�	 465
+���	 465
+�����	 464
+_�	 463
+����	 459
+�����	 459
+�����	 459
+���	 454
+�	 449
+_��	 449
+�	 444
+_��	 442
+�_	 431
+ٻ	 428
+�_	 427
+_ϳ	 422
+���	 422
+���	 421
+��_	 416
+��	 416
+_��	 415
+��	 414
+_�	 413
+�	 408
+���	 405
+����	 405
+��_	 404
+����	 400
+󑩯	 399
+�󑩯	 399
+󑩯�	 399
+��_	 398
+�_	 395
+۳�	 394
+��_	 387
+-	 385
+���	 383
+���	 382
+��	 380
+�_	 379
+�	 377
+���	 377
+�_	 377
+��	 376
+ݻ��	 374
+��	 371
+���	 371
+��_	 371
+_�	 371
+�	 370
+�	 368
+�	 367
+_�_	 366
+��	 363
+��	 361
+�	 358
+.	 357
+��	 356
+_�	 351
+�	 347
+_��	 345
+�۳	 342
+��	 341
+_�	 341
+_�	 340
+ϳ�_	 339
+��	 338
+_�	 337
+�_	 331
+��_	 330
+��۳	 328
+���۳	 328
+��	 328
+��	 327
+��_	 324
+_��	 324
+��	 324
+_��_	 322
+��	 321
+���_	 321
+_ٻ	 318
+��	 317
+��	 317
+ٳ�	 316
+_�	 314
+��	 314
+�	 309
+_��	 309
+�ϳ�_	 307
+��	 306
+��	 306
+_�	 303
+��	 302
+��	 302
+��	 302
+��	 300
+��	 299
+�۳�	 299
+���	 299
+_��	 299
+_�_	 299
+��	 298
+����	 298
+�	 297
+��_	 294
+_��	 293
+_�	 292
+��	 291
+��	 291
+��۳�	 288
+��	 288
+��	 287
+��_	 286
+_�	 286
+�	 284
+�	 284
+��	 284
+˳	 283
+�	 283
+��_	 281
+��_	 281
+۳�_	 281
+`	 280
+��	 279
+`_	 279
+��	 279
+�ٳ	 278
+��_	 274
+��	 271
+�	 268
+_ٳ	 266
+��	 266
+��	 266
+��	 266
+ݻ��_	 266
+�	 264
+�,	 263
+��	 263
+�_	 262
+�	 262
+�,_	 262
+��	 259
+��	 258
+_�	 258
+ϻ	 257
+���	 257
+�_	 255
+��	 255
+_�	 255
+_��_	 255
+�	 255
+��	 253
+�ݻ	 252
+��_	 252
+��	 250
+��	 248
+�ݻ�	 248
+_��_	 247
+ݹ	 246
+��_	 246
+��	 245
+��	 244
+��	 244
+���	 244
+���	 244
+��	 243
+�۳�_	 243
+󑩁	 242
+�󑩁	 242
+��	 241
+��	 239
+��	 238
+�_	 238
+�ݳ	 237
+��	 237
+��	 237
+��	 237
+_�	 236
+����	 234
+��	 234
+���_	 233
+�_	 233
+ٳ�	 233
+�	 232
+����	 232
+���	 231
+�	 230
+��	 230
+���	 230
+_��	 228
+���	 228
+��	 228
+û	 227
+���	 227
+_��	 226
+ݳ�	 223
+����_	 223
+_-	 222
+��	 222
+��_	 222
+��	 220
+_�	 220
+_��	 218
+ϳ�	 217
+����	 216
+_�	 216
+���	 216
+�	 215
+�	 215
+󑩁�	 214
+��	 214
+_��	 213
+�	 213
+��	 213
+��	 212
+ѳ�	 211
+�	 210
+��	 210
+_��_	 210
+_��	 209
+��	 209
+_��_	 208