RubyGems - language_detector - Versions diffs - 0.1.0 - Mend

language_detector 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (172) hide show

data/README.rdoc +24 -0
data/Rakefile +18 -0
data/VERSION +1 -0
data/lib/language_detector.rb +232 -0
data/lib/model-fm.yml +52504 -0
data/lib/model-tc.yml +53985 -0
data/lib/textcat_ngrams/afrikaans.lm +400 -0
data/lib/textcat_ngrams/albanian.lm +400 -0
data/lib/textcat_ngrams/amharic-utf.lm +400 -0
data/lib/textcat_ngrams/arabic-iso8859_6.lm +400 -0
data/lib/textcat_ngrams/arabic-windows1256.lm +400 -0
data/lib/textcat_ngrams/armenian.lm +400 -0
data/lib/textcat_ngrams/basque.lm +400 -0
data/lib/textcat_ngrams/belarus-windows1251.lm +400 -0
data/lib/textcat_ngrams/bosnian.lm +400 -0
data/lib/textcat_ngrams/breton.lm +400 -0
data/lib/textcat_ngrams/bulgarian-iso8859_5.lm +400 -0
data/lib/textcat_ngrams/catalan.lm +400 -0
data/lib/textcat_ngrams/chinese-big5.lm +400 -0
data/lib/textcat_ngrams/chinese-gb2312.lm +400 -0
data/lib/textcat_ngrams/croatian-ascii.lm +400 -0
data/lib/textcat_ngrams/czech-iso8859_2.lm +400 -0
data/lib/textcat_ngrams/danish.lm +400 -0
data/lib/textcat_ngrams/dutch.lm +400 -0
data/lib/textcat_ngrams/english.lm +400 -0
data/lib/textcat_ngrams/esperanto.lm +400 -0
data/lib/textcat_ngrams/estonian.lm +400 -0
data/lib/textcat_ngrams/finnish.lm +400 -0
data/lib/textcat_ngrams/french.lm +400 -0
data/lib/textcat_ngrams/frisian.lm +400 -0
data/lib/textcat_ngrams/georgian.lm +400 -0
data/lib/textcat_ngrams/german.lm +400 -0
data/lib/textcat_ngrams/greek-iso8859-7.lm +400 -0
data/lib/textcat_ngrams/hebrew-iso8859_8.lm +400 -0
data/lib/textcat_ngrams/hindi.lm +400 -0
data/lib/textcat_ngrams/hungarian.lm +400 -0
data/lib/textcat_ngrams/icelandic.lm +400 -0
data/lib/textcat_ngrams/indonesian.lm +400 -0
data/lib/textcat_ngrams/irish.lm +400 -0
data/lib/textcat_ngrams/italian.lm +400 -0
data/lib/textcat_ngrams/japanese-euc_jp.lm +400 -0
data/lib/textcat_ngrams/japanese-shift_jis.lm +400 -0
data/lib/textcat_ngrams/korean.lm +400 -0
data/lib/textcat_ngrams/latin.lm +400 -0
data/lib/textcat_ngrams/latvian.lm +400 -0
data/lib/textcat_ngrams/lithuanian.lm +400 -0
data/lib/textcat_ngrams/malay.lm +400 -0
data/lib/textcat_ngrams/manx.lm +400 -0
data/lib/textcat_ngrams/marathi.lm +400 -0
data/lib/textcat_ngrams/mingo.lm +400 -0
data/lib/textcat_ngrams/nepali.lm +400 -0
data/lib/textcat_ngrams/norwegian.lm +400 -0
data/lib/textcat_ngrams/persian.lm +400 -0
data/lib/textcat_ngrams/polish.lm +400 -0
data/lib/textcat_ngrams/portuguese.lm +400 -0
data/lib/textcat_ngrams/quechua.lm +400 -0
data/lib/textcat_ngrams/romanian.lm +400 -0
data/lib/textcat_ngrams/rumantsch.lm +400 -0
data/lib/textcat_ngrams/russian-iso8859_5.lm +400 -0
data/lib/textcat_ngrams/russian-koi8_r.lm +400 -0
data/lib/textcat_ngrams/russian-windows1251.lm +400 -0
data/lib/textcat_ngrams/sanskrit.lm +400 -0
data/lib/textcat_ngrams/scots.lm +400 -0
data/lib/textcat_ngrams/scots_gaelic.lm +400 -0
data/lib/textcat_ngrams/serbian-ascii.lm +400 -0
data/lib/textcat_ngrams/slovak-ascii.lm +400 -0
data/lib/textcat_ngrams/slovak-windows1250.lm +400 -0
data/lib/textcat_ngrams/slovenian-ascii.lm +400 -0
data/lib/textcat_ngrams/slovenian-iso8859_2.lm +400 -0
data/lib/textcat_ngrams/spanish.lm +400 -0
data/lib/textcat_ngrams/swahili.lm +400 -0
data/lib/textcat_ngrams/swedish.lm +400 -0
data/lib/textcat_ngrams/tagalog.lm +400 -0
data/lib/textcat_ngrams/tamil.lm +400 -0
data/lib/textcat_ngrams/thai.lm +400 -0
data/lib/textcat_ngrams/turkish.lm +400 -0
data/lib/textcat_ngrams/ukrainian-koi8_u.lm +400 -0
data/lib/textcat_ngrams/vietnamese.lm +400 -0
data/lib/textcat_ngrams/welsh.lm +400 -0
data/lib/textcat_ngrams/yiddish-utf.lm +400 -0
data/lib/training_data/ar-utf8.txt +54 -0
data/lib/training_data/bg-utf8.txt +26 -0
data/lib/training_data/cs-utf8.txt +48 -0
data/lib/training_data/da-utf8.txt +159 -0
data/lib/training_data/de-utf8.txt +569 -0
data/lib/training_data/el-utf8.txt +48 -0
data/lib/training_data/en-utf8.txt +81 -0
data/lib/training_data/es-utf8.txt +185 -0
data/lib/training_data/et-utf8.txt +50 -0
data/lib/training_data/fa-utf8.txt +42 -0
data/lib/training_data/fi-utf8.txt +83 -0
data/lib/training_data/fr-utf8.txt +191 -0
data/lib/training_data/fy-utf8.txt +22 -0
data/lib/training_data/ga-utf8.txt +109 -0
data/lib/training_data/he-utf8.txt +116 -0
data/lib/training_data/hi-utf8.txt +49 -0
data/lib/training_data/hr-utf8.txt +80 -0
data/lib/training_data/hu-utf8.txt +87 -0
data/lib/training_data/io-utf8.txt +41 -0
data/lib/training_data/is-utf8.txt +94 -0
data/lib/training_data/it-utf8.txt +228 -0
data/lib/training_data/ja-utf8.txt +200 -0
data/lib/training_data/ko-utf8.txt +147 -0
data/lib/training_data/nl-utf8.txt +215 -0
data/lib/training_data/no-utf8.txt +281 -0
data/lib/training_data/pl-utf8.txt +120 -0
data/lib/training_data/pt-utf8.txt +214 -0
data/lib/training_data/ro-utf8.txt +66 -0
data/lib/training_data/ru-utf8.txt +310 -0
data/lib/training_data/sl-utf8.txt +263 -0
data/lib/training_data/sv-utf8.txt +174 -0
data/lib/training_data/th-utf8.txt +49 -0
data/lib/training_data/tk-utf8.txt +101 -0
data/lib/training_data/todo/af.txt +114 -0
data/lib/training_data/todo/amharic-utf.txt +95 -0
data/lib/training_data/todo/arabic-windows1256.txt +157 -0
data/lib/training_data/todo/armenian.txt +86 -0
data/lib/training_data/todo/basque.txt +136 -0
data/lib/training_data/todo/belarus-windows1251.txt +97 -0
data/lib/training_data/todo/bosnian.txt +97 -0
data/lib/training_data/todo/breton.txt +159 -0
data/lib/training_data/todo/bulgarian-iso8859_5.txt +115 -0
data/lib/training_data/todo/catalan.txt +93 -0
data/lib/training_data/todo/croatian-ascii.txt +104 -0
data/lib/training_data/todo/esperanto.txt +95 -0
data/lib/training_data/todo/estonian.txt +218 -0
data/lib/training_data/todo/frisian.txt +99 -0
data/lib/training_data/todo/georgian.txt +86 -0
data/lib/training_data/todo/greek-iso8859-7.txt +139 -0
data/lib/training_data/todo/hawaian.txt +108 -0
data/lib/training_data/todo/hebrew-iso8859_8.txt +79 -0
data/lib/training_data/todo/hindi.txt +77 -0
data/lib/training_data/todo/hungarian.txt +102 -0
data/lib/training_data/todo/icelandic.txt +131 -0
data/lib/training_data/todo/indonesian.txt +93 -0
data/lib/training_data/todo/irish.txt +209 -0
data/lib/training_data/todo/latin.txt +120 -0
data/lib/training_data/todo/latvian.txt +126 -0
data/lib/training_data/todo/lithuanian.txt +99 -0
data/lib/training_data/todo/malay.txt +108 -0
data/lib/training_data/todo/manx.txt +78 -0
data/lib/training_data/todo/marathi.txt +100 -0
data/lib/training_data/todo/mf.txt +100 -0
data/lib/training_data/todo/middle_frisian.txt +102 -0
data/lib/training_data/todo/mingo.txt +146 -0
data/lib/training_data/todo/nepali.txt +131 -0
data/lib/training_data/todo/persian.txt +73 -0
data/lib/training_data/todo/quechua.txt +108 -0
data/lib/training_data/todo/romanian.txt +103 -0
data/lib/training_data/todo/rumantsch.txt +110 -0
data/lib/training_data/todo/sanskrit.txt +135 -0
data/lib/training_data/todo/scots.txt +490 -0
data/lib/training_data/todo/scots_gaelic.txt +93 -0
data/lib/training_data/todo/serbian-ascii.txt +121 -0
data/lib/training_data/todo/slovak-ascii.txt +102 -0
data/lib/training_data/todo/slovak-windows1250.txt +115 -0
data/lib/training_data/todo/slovenian-ascii.txt +100 -0
data/lib/training_data/todo/slovenian-iso8859_2.txt +96 -0
data/lib/training_data/todo/sq.txt +110 -0
data/lib/training_data/todo/swahili.txt +120 -0
data/lib/training_data/todo/tagalog.txt +135 -0
data/lib/training_data/todo/tamil.txt +123 -0
data/lib/training_data/todo/turkish.txt +117 -0
data/lib/training_data/todo/ukrainian-koi8_r.txt +214 -0
data/lib/training_data/todo/vietnamese.txt +92 -0
data/lib/training_data/todo/welsh.txt +148 -0
data/lib/training_data/todo/yiddish-utf.txt +83 -0
data/lib/training_data/uk-utf8.txt +75 -0
data/lib/training_data/vi-utf8.txt +47 -0
data/lib/training_data/zh-utf8.txt +228 -0
data/test/language_detector_test.rb +78 -0
metadata +232 -0

data/lib/textcat_ngrams/greek-iso8859-7.lm ADDED

@@ -0,0 +1,400 @@
+_	 89284
+�	 19666
+�	 16086
+�	 15826
+�	 14848
+�	 12766
+�	 12189
+�	 8776
+�	 8653
+�	 8399
+�	 7761
+�	 7529
+�	 6850
+�	 6284
+_�	 5918
+�	 5802
+�	 5371
+�_	 5272
+�	 5236
+�	 4739
+�	 4609
+�	 4489
+��	 4461
+�_	 4111
+��	 4075
+�_	 4033
+�_	 4019
+.	 3916
+_�	 3897
+�_	 3836
+._	 3810
+�_	 3661
+_�	 3414
+_�	 3333
+"	 3247
+_�	 3242
+_��	 3118
+,	 3106
+,_	 3068
+�	 3058
+_�	 2912
+�	 2689
+��	 2593
+�	 2455
+��	 2420
+�	 2411
+��	 2406
+��	 2406
+_�	 2353
+-	 2335
+-_	 2192
+�	 2171
+��	 2162
+��	 2151
+�	 2141
+�_	 2133
+��	 2100
+�	 2053
+�	 1989
+��_	 1974
+��	 1889
+_��	 1882
+��	 1864
+��_	 1802
+��	 1771
+��_	 1727
+��	 1709
+_�	 1647
+_"	 1619
+�_	 1609
+��	 1602
+��_	 1518
+��	 1501
+��	 1501
+���	 1495
+��	 1460
+_���	 1407
+_�	 1405
+_��	 1396
+�	 1364
+��	 1359
+��	 1300
+��	 1298
+�	 1274
+��	 1252
+�_	 1241
+_�	 1231
+�	 1223
+��	 1220
+_��_	 1196
+��	 1187
+��	 1184
+_�	 1183
+_��	 1177
+�	 1158
+_��	 1150
+�_	 1150
+_��	 1124
+��	 1115
+���	 1110
+_���	 1093
+��_	 1082
+��	 1078
+���_	 1061
+_��_	 1060
+_���_	 1053
+��	 1049
+��	 1048
+��	 1036
+��	 1024
+��_	 1018
+��	 1003
+��	 993
+_��	 984
+��_	 974
+��	 973
+_�	 967
+��	 947
+��	 936
+��_	 935
+��	 901
+��	 892
+_��	 890
+��	 884
+���_	 871
+��_	 869
+��	 865
+_���_	 854
+��	 841
+��_	 840
+��	 834
+��	 833
+��	 832
+��	 824
+��	 820
+��	 808
+_��	 804
+��	 796
+�	 796
+��	 783
+��	 777
+��	 772
+��	 766
+��	 762
+�	 759
+��	 745
+��	 742
+��_	 740
+���	 738
+�_	 730
+��_	 728
+�	 717
+�	 714
+_�	 711
+��	 705
+��	 704
+��	 697
+��_	 688
+��	 684
+��_	 683
+_�	 680
+_�	 675
+��	 671
+_�	 670
+���	 663
+���_	 658
+_���	 647
+�_	 645
+��_	 638
+_�	 633
+_�_	 631
+",	 630
+_��	 626
+_�	 624
+��	 624
+��	 623
+",_	 623
+���	 620
+���	 619
+��	 618
+��	 617
+���	 617
+_���	 616
+��	 612
+�_	 611
+��	 607
+��	 604
+��	 595
+_�	 594
+��	 590
+_��_	 586
+��	 580
+���_	 577
+��	 577
+_�	 575
+���_	 572
+��	 570
+��	 570
+_���_	 569
+��	 562
+���	 554
+���	 553
+��	 552
+���_	 551
+��	 547
+��	 547
+��	 544
+_�	 540
+��	 536
+��_	 534
+��	 528
+_�_	 525
+��	 520
+".	 519
+_���	 516
+_�	 515
+"._	 513
+��_	 512
+�.	 510
+���	 508
+���_	 504
+_���	 503
+�._	 501
+_��	 500
+��	 497
+���_	 496
+��	 493
+��	 492
+�	 491
+_���_	 490
+��	 489
+���	 489
+����	 485
+_����	 483
+_���	 483
+��_	 482
+_���_	 481
+��	 480
+��	 480
+_��	 479
+��	 477
+_���_	 475
+��	 473
+_�	 468
+"_	 463
+���	 463
+��	 461
+��	 452
+_��	 452
+��	 451
+��	 451
+_���	 450
+_��	 449
+��	 447
+��	 446
+��	 446
+_��	 445
+_���_	 443
+�.	 441
+�._	 441
+���	 439
+��	 439
+��	 439
+��	 438
+���	 437
+��	 432
+_��	 428
+_��	 427
+��_	 427
+��	 423
+;	 423
+_��	 422
+��	 421
+���	 421
+_�	 418
+���	 415
+��	 411
+��	 408
+���	 405
+�.	 403
+��	 402
+��	 402
+�,	 402
+�,_	 401
+�._	 398
+��	 397
+_��	 397
+��	 396
+_��	 395
+_�	 393
+_��	 392
+��	 391
+��	 391
+��	 390
+���	 389
+��	 389
+���	 388
+��	 388
+_��	 387
+��_	 386
+��	 384
+���	 384
+�	 383
+���	 382
+�_	 376
+���	 375
+��	 372
+_���	 370
+_��_	 367
+_���	 365
+_���	 362
+_��	 361
+��	 361
+���	 359
+��_	 358
+��	 358
+_��	 354
+_���	 354
+���	 352
+�.	 352
+�._	 351
+���	 351
+��	 350
+����	 348
+���	 348
+���_	 348
+��_	 347
+��	 346
+�	 346
+�.	 346
+���_	 346
+�._	 345
+���	 345
+���_	 345
+�,	 344
+�,_	 344
+���_	 344
+��	 343
+��	 343
+�-	 342
+��	 341
+_�.	 340
+���	 339
+���	 339
+_�._	 339
+��	 339
+�-_	 338
+��	 338
+��	 338
+��	 338
+����	 338
+_���	 338
+_���	 338
+_����	 337
+��	 336
+���	 336
+����	 336
+�����	 336
+����	 336
+_��	 334
+�����	 334
+_�	 334
+����	 334
+��_	 333
+��	 332
+���_	 331
+�	 331
+��	 330
+���	 330
+_��	 329
+��_	 328
+��	 328
+�	 326
+_���	 325
+_���_	 325
+��	 324
+��_	 324
+���	 324
+��	 323
+����_	 323
+���	 322
+_�	 321
+�����	 321
+��_	 321
+���_	 321
+_��	 321
+��	 320
+_���_	 320
+���	 320
+��	 318
+;"	 316
+;"_	 316
+�	 315
+��	 314
+��_	 313
+_��	 312
+��	 312
+_��_	 310
+��	 309
+��.	 309

data/lib/textcat_ngrams/hebrew-iso8859_8.lm ADDED

@@ -0,0 +1,400 @@
+_	 81560
+�	 23072
+�	 19215
+�	 15606
+�	 12612
+�	 11293
+�	 11070
+�	 10090
+�	 9648
+�	 9601
+�	 9081
+�_	 7811
+�	 6685
+�	 6326
+_�	 5878
+�	 5542
+�	 5019
+�	 4669
+_�	 4570
+�_	 4267
+�	 3984
+_�	 3812
+�	 3769
+�	 3699
+�_	 3693
+��	 3535
+�_	 3489
+�	 3374
+�_	 3362
+��	 3130
+,	 3067
+_�	 3037
+_,	 3035
+_�	 2936
+�_	 2886
+_��	 2840
+�	 2768
+�	 2626
+�	 2424
+_�	 2395
+_��	 2291
+.	 2256
+�	 2215
+_.	 2205
+��	 2144
+�_	 2003
+_�	 1959
+_�	 1913
+"	 1815
+��	 1797
+��	 1742
+��	 1687
+��	 1665
+�_	 1647
+��	 1620
+�_	 1578
+��	 1525
+��	 1475
+�_	 1469
+��	 1411
+�	 1383
+��	 1353
+��	 1351
+��	 1343
+��	 1343
+��	 1315
+��	 1285
+��	 1236
+��_	 1211
+��	 1206
+��	 1192
+��	 1163
+��	 1155
+��	 1146
+��	 1138
+_��	 1137
+��	 1134
+��	 1087
+�_	 1080
+��	 1074
+��	 1067
+_�	 1062
+��	 1059
+��	 1052
+��	 1047
+-	 1024
+��_	 1008
+��	 973
+��	 967
+��	 954
+��_	 951
+��	 948
+_�	 941
+��	 896
+��	 888
+��	 883
+��	 873
+��	 871
+��	 867
+��	 850
+��	 843
+_-	 830
+�_	 830
+��	 824
+��	 820
+�_	 806
+��	 805
+_��	 799
+��	 784
+��	 773
+_��_	 748
+��	 746
+��	 743
+_�	 726
+_��_	 723
+�	 720
+_��	 719
+-_	 716
+_-_	 713
+��	 709
+��	 706
+��	 699
+��_	 690
+��	 686
+��	 683
+��	 678
+��	 675
+�_	 672
+��	 669
+_�	 668
+_�	 659
+_�	 655
+��	 652
+��	 649
+_��	 640
+��	 635
+��	 635
+��_	 632
+�_	 631
+��	 628
+��	 627
+_��	 624
+��	 624
+��_	 621
+��	 619
+��	 600
+��_	 599
+��	 599
+��	 596
+��	 595
+��	 586
+��	 578
+��	 575
+_��	 569
+��	 564
+_��_	 561
+��	 560
+��	 557
+��_	 557
+���	 554
+��	 548
+��_	 548
+���	 548
+���	 546
+��	 542
+��	 540
+"_	 540
+��	 539
+��_	 538
+��	 538
+���	 535
+��	 534
+��	 533
+��	 532
+����	 530
+�����	 530
+����	 530
+��	 527
+��	 525
+_��	 518
+��	 518
+��_	 511
+��_	 510
+_��	 509
+��	 506
+��	 502
+��_	 500
+��	 498
+��	 497
+��	 497
+��	 494
+_�	 485
+�_	 481
+���	 477
+_��	 477
+��	 472
+��	 470
+,�	 470
+_,�	 470
+�_	 469
+��	 468
+_��	 467
+��	 467
+��_	 466
+_��_	 463
+��	 462
+��	 462
+_��	 461
+�	 455
+��	 454
+���	 454
+��	 453
+��	 453
+��	 452
+���	 451
+_��	 447
+��	 443
+�_	 441
+��	 440
+���	 439
+��_	 439
+'	 435
+��_	 430
+.�	 429
+��	 429
+_.�	 428
+��	 427
+��	 423
+��	 417
+��	 417
+_��	 414
+��	 413
+��	 408
+�_	 405
+��	 400
+��	 400
+��	 399
+,�	 399
+_,�	 399
+��	 397
+��_	 396
+��	 396
+��	 394
+��_	 393
+���	 393
+���	 389
+_��	 385
+���	 384
+��_	 384
+��	 384
+��	 381
+��	 378
+��_	 377
+��	 374
+_�	 374
+��	 374
+��	 373
+��	 373
+��	 371
+��	 370
+��	 370
+_��	 370
+��	 367
+_��	 366
+_���	 364
+��	 363
+�_	 362
+_�	 360
+��	 360
+_��	 360
+��	 359
+,�	 358
+_,�	 357
+��	 356
+���	 353
+���	 353
+��_	 349
+��	 348
+���	 348
+��	 345
+�	 345
+���	 344
+��	 340
+_���	 340
+��	 338
+��	 338
+_���	 336
+���	 335
+_���	 335
+��_	 332
+��	 331
+_��	 329
+_���	 325
+��	 325
+��_	 325
+:	 324
+��	 324
+��_	 324
+��	 322
+��	 321
+��	 320
+�_	 320
+��_	 319
+��_	 319
+_:	 319
+��	 316
+���	 315
+_�	 315
+���	 315
+��	 315
+��	 313
+��_	 312
+��	 312
+_�	 310
+���	 309
+���_	 305
+����_	 305
+.��	 305
+_.��	 304
+��	 303
+���	 303
+_'	 303
+��	 302
+��_	 302
+_���	 301
+��	 298
+��	 294
+��	 293
+��	 293
+���	 292
+��	 292
+����	 292
+��	 292
+���	 290
+��	 289
+��	 289
+��	 289
+��	 287
+_��	 286
+,��	 285
+_,��	 285
+��	 284
+��	 282
+��	 282
+_��	 281
+��	 281
+_���	 281
+��	 280
+���	 278
+��_	 276
+��_	 276
+_��_	 275
+���	 274
+_���	 274
+��	 274
+��_	 271
+��_	 271
+��	 270
+��_	 269
+��	 268
+��	 267
+��	 267
+.�	 266
+_��	 266
+_����	 266
+���	 266
+_.�	 265
+����	 265
+���	 265
+���	 265
+��	 264
+��	 263
+��	 262
+���	 260
+_.�	 260
+.�	 260
+��_	 260
+��_	 259
+���	 258
+����	 258
+�����	 258
+�����	 257
+����	 257
+��_	 256
+��_	 256
+��	 255
+���	 255
+��_	 254
+���	 254
+��	 252
+��	 251
+��_	 251
+_��	 251
+���	 251
+_��	 249
+_��	 249
+_���	 249
+��	 248
+��	 247
+_��	 247