language_detector 0.1.0

data/lib/textcat_ngrams/english.lm

@@ -0,0 +1,400 @@
+_	 20326
+e	 6617
+t	 4843
+o	 3834
+n	 3653
+i	 3602
+a	 3433
+s	 2945
+r	 2921
+h	 2507
+e_	 2000
+d	 1816
+_t	 1785
+c	 1639
+l	 1635
+th	 1535
+he	 1351
+_th	 1333
+u	 1309
+f	 1253
+m	 1175
+p	 1151
+_a	 1145
+the	 1142
+_the	 1060
+s_	 978
+er	 968
+_o	 967
+he_	 928
+d_	 888
+t_	 885
+the_	 844
+_the_	 843
+on	 842
+in	 817
+y	 783
+n_	 773
+b	 761
+re	 754
+,	 734
+,_	 732
+an	 732
+g	 728
+w	 718
+_i	 707
+en	 676
+f_	 599
+y_	 595
+of	 594
+_of	 592
+es	 589
+ti	 587
+v	 580
+_of_	 575
+of_	 575
+nd	 568
+at	 549
+r_	 540
+_w	 534
+it	 522
+ed	 496
+_p	 494
+nt	 485
+_c	 462
+o_	 457
+io	 450
+_an	 439
+te	 432
+or	 425
+_b	 418
+nd_	 407
+to	 406
+st	 402
+is	 401
+_s	 396
+_in	 389
+ion	 385
+and	 385
+de	 384
+ve	 382
+ha	 375
+ar	 366
+_m	 361
+and_	 360
+_and	 360
+_and_	 358
+se	 353
+_to	 347
+me	 346
+to_	 344
+ed_	 339
+.	 330
+be	 329
+_f	 329
+._	 329
+_to_	 320
+co	 317
+ic	 316
+ns	 308
+al	 307
+le	 304
+ou	 304
+ce	 293
+ent	 279
+l_	 278
+_co	 277
+tio	 275
+on_	 274
+_d	 274
+tion	 268
+ri	 266
+_e	 264
+ng	 253
+hi	 251
+er_	 249
+ea	 246
+as	 245
+_be	 242
+pe	 242
+h_	 234
+_r	 232
+ec	 227
+ch	 223
+ro	 222
+ct	 220
+_h	 219
+pr	 217
+in_	 217
+ne	 214
+ll	 214
+rt	 213
+s,_	 210
+s,	 210
+li	 209
+ra	 208
+T	 207
+wh	 204
+a_	 203
+ac	 201
+_wh	 199
+_n	 196
+ts	 196
+di	 196
+es_	 195
+si	 194
+re_	 193
+at_	 192
+nc	 192
+ie	 190
+_a_	 188
+_in_	 185
+ing	 184
+us	 182
+_re	 182
+g_	 179
+ng_	 178
+op	 178
+con	 177
+tha	 175
+_l	 174
+_tha	 174
+ver	 173
+ma	 173
+ion_	 171
+_con	 171
+ci	 170
+ons	 170
+_it	 170
+po	 169
+ere	 168
+is_	 167
+ta	 167
+la	 166
+_pr	 165
+fo	 164
+ho	 164
+ir	 162
+ss	 161
+men	 160
+be_	 160
+un	 159
+ty	 159
+_be_	 158
+ing_	 157
+om	 156
+ot	 156
+hat	 155
+ly	 155
+_g	 155
+em	 153
+_T	 151
+rs	 150
+mo	 148
+ch_	 148
+wi	 147
+we	 147
+ad	 147
+ts_	 145
+res	 143
+_wi	 143
+I	 143
+hat_	 142
+ei	 141
+ly_	 141
+ni	 140
+os	 140
+ca	 139
+ur	 139
+A	 138
+ut	 138
+that	 138
+_that	 137
+ati	 137
+_fo	 137
+st_	 137
+il	 136
+or_	 136
+for	 136
+pa	 136
+ul	 135
+ate	 135
+ter	 134
+it_	 134
+nt_	 133
+that_	 132
+_ha	 129
+al_	 128
+el	 128
+as_	 127
+ll_	 127
+_ma	 125
+no	 124
+ment	 124
+an_	 124
+tion_	 122
+su	 122
+bl	 122
+_de	 122
+nce	 120
+pl	 120
+fe	 119
+tr	 118
+so	 118
+int	 115
+ov	 114
+e,	 114
+e,_	 114
+_u	 113
+ent_	 113
+Th	 113
+her	 113
+j	 112
+atio	 112
+ation	 112
+_Th	 111
+le_	 110
+ai	 110
+_it_	 110
+_on	 110
+_for	 109
+ect	 109
+k	 109
+hic	 108
+est	 108
+der	 107
+tu	 107
+na	 106
+_by_	 106
+by_	 106
+E	 106
+by	 106
+_by	 106
+ve_	 106
+_di	 106
+en_	 104
+vi	 104
+m_	 103
+_whi	 102
+iv	 102
+whi	 102
+ns_	 102
+_A	 101
+ich	 100
+ge	 100
+pro	 99
+ess	 99
+_whic	 99
+ers	 99
+hich	 99
+ce_	 99
+which	 99
+whic	 99
+all	 98
+ove	 98
+_is	 98
+ich_	 97
+ee	 97
+hich_	 97
+n,_	 96
+n,	 96
+im	 95
+ir_	 94
+hei	 94
+ions	 94
+sti	 94
+se_	 94
+per	 93
+The	 93
+_pa	 93
+heir	 93
+id	 93
+eir	 93
+eir_	 93
+ig	 93
+heir_	 93
+_no	 93
+ev	 93
+era	 92
+_int	 92
+ted	 91
+_The	 91
+ies	 91
+art	 91
+thei	 90
+_ar	 90
+_thei	 90
+their	 90
+_pro	 90
+et	 89
+_pe	 88
+_mo	 88
+ther	 88
+x	 87
+gh	 87
+S	 87
+_is_	 87
+ol	 87
+ty_	 87
+_I	 86
+nde	 86
+am	 86
+rn	 86
+nte	 86
+mp	 85
+_su	 84
+_we	 84
+par	 84
+_v	 84
+pu	 82
+his	 82
+ow	 82
+mi	 82
+go	 81
+N	 81
+ue	 81
+ple	 81
+ep	 80
+ab	 80
+;_	 80
+;	 80
+ex	 80
+ain	 80
+over	 80
+_un	 79
+q	 79
+qu	 79
+pp	 79
+ith	 79
+ry	 79
+_as	 79
+ber	 79
+ub	 78
+av	 78
+uc	 78
+s._	 77
+s.	 77
+enc	 77
+are	 77
+iti	 77
+gr	 76
+his_	 76
+ua	 76
+part	 76
+ff	 75
+eve	 75
+O	 75
+rea	 74
+ous	 74
+ia	 74
+The_	 73
+ag	 73
+mb	 73
+_go	 73
+fa	 72
+on,_	 72
+ern	 72
+t,_	 72
+on,	 72
+t,	 72
+_me	 71

data/lib/textcat_ngrams/esperanto.lm

@@ -0,0 +1,400 @@
+_	 57050
+a	 16035
+i	 12706
+e	 12227
+o	 12102
+n	 10393
+s	 8344
+l	 7707
+r	 7492
+t	 7134
+k	 5376
+u	 4558
+j	 3946
+a_	 3875
+m	 3783
+d	 3710
+p	 3693
+la	 2840
+s_	 2769
+e_	 2751
+.	 2706
+_l	 2635
+_k	 2619
+v	 2531
+n_	 2504
+o_	 2444
+i_	 2333
+._	 2278
+on	 2238
+,	 2193
+,_	 2182
+_la	 2100
+en	 2080
+j_	 2050
+as	 2028
+la_	 2012
+ta	 1956
+_la_	 1907
+an	 1882
+_p	 1850
+g	 1831
+_e	 1791
+_d	 1778
+is	 1737
+aj	 1658
+st	 1635
+_s	 1575
+c	 1526
+de	 1517
+oj	 1498
+er	 1476
+ti	 1456
+f	 1443
+_a	 1442
+b	 1427
+ro	 1379
+_m	 1351
+ra	 1341
+nt	 1293
+ka	 1270
+ri	 1258
+al	 1249
+as_	 1248
+aj_	 1213
+to	 1209
+_de	 1203
+_t	 1200
+te	 1179
+_n	 1176
+is_	 1171
+in	 1151
+ko	 1145
+or	 1114
+es	 1083
+re	 1034
+ia	 1029
+li	 1022
+de_	 1016
+_de_	 979
+ar	 974
+_v	 966
+vi	 942
+lo	 932
+x	 928
+io	 917
+ne	 855
+no	 848
+ni	 843
+mi	 835
+ma	 819
+_ka	 816
+el	 815
+pr	 771
+z	 744
+un	 734
+l_	 732
+po	 730
+_f	 725
+�	 724
+est	 691
+na	 687
+ki	 679
+kaj	 676
+si	 665
+u_	 663
+kaj_	 660
+"	 654
+tas	 651
+le	 650
+oj_	 648
+_i	 643
+tr	 642
+_pr	 630
+_es	 628
+jn	 626
+pe	 618
+_kaj	 616
+ig	 616
+_kaj_	 611
+do	 608
+sta	 606
+on_	 602
+ek	 602
+ci	 597
+r_	 595
+�	 594
+_r	 593
+il	 592
+_est	 587
+di	 586
+am	 586
+_mi	 582
+a�	 578
+_vi	 577
+mo	 575
+ant	 565
+_ne	 562
+en_	 561
+o.	 559
+�	 543
+iu	 538
+o,	 529
+ur	 527
+o._	 527
+om	 525
+o,_	 524
+at	 521
+va	 521
+-	 519
+_en	 518
+:	 513
+:_	 512
+_ti	 500
+M	 496
+h	 488
+nd	 484
+me	 484
+_al	 481
+_ko	 479
+ve	 478
+ie	 478
+_ki	 473
+it	 473
+L	 466
+_b	 465
+se	 462
+em	 452
+ol	 450
+nta	 449
+tu	 448
+ik	 444
+ov	 443
+da	 443
+_M	 440
+_po	 439
+tas_	 438
+ne_	 437
+et	 437
+_ma	 436
+_en_	 435
+su	 429
+pl	 426
+_L	 425
+pa	 420
+_o	 417
+vo	 408
+an_	 407
+ro_	 406
+sti	 406
+nu	 399
+kon	 396
+stas	 391
+m_	 391
+ir	 388
+n.	 386
+fa	 386
+jn_	 382
+ku	 382
+os	 376
+ke	 375
+n,	 375
+esta	 374
+n,_	 372
+_su	 362
+ta_	 362
+stas_	 359
+xi	 359
+Mi	 358
+_ne_	 356
+al_	 355
+nk	 353
+so	 353
+n._	 352
+id	 349
+_g	 348
+estas	 347
+ga	 346
+_h	 345
+per	 345
+_Mi	 340
+ok	 339
+K	 339
+mp	 337
+_esta	 337
+s,_	 335
+s,	 335
+_se	 333
+anta	 332
+ul	 326
+ran	 325
+_"	 323
+�_	 322
+te_	 320
+ak	 320
+a�_	 320
+ed	 320
+rt	 319
+ojn	 318
+gi	 318
+_�	 317
+tis	 316
+gx	 316
+mal	 316
+ia_	 315
+ks	 310
+_al_	 310
+mi_	 309
+S	 309
+lu	 309
+ns	 308
+kt	 305
+io_	 302
+ent	 300
+?	 300
+_K	 300
+ec	 300
+el_	 299
+_-	 299
+li_	 299
+E	 298
+�	 298
+_li	 297
+fo	 296
+ter	 296
+_re	 296
+A	 295
+nto	 294
+vi_	 292
+La	 292
+_mal	 290
+nte	 288
+sp	 287
+sa	 287
+_mi_	 279
+ut	 278
+op	 278
+_ke	 277
+bo	 277
+ajn	 276
+un_	 276
+T	 274
+to_	 272
+-_	 272
+bl	 272
+_an	 271
+_La	 271
+�i	 269
+_S	 268
+_pl	 267
+_fa	 266
+ni_	 266
+La_	 265
+_E	 264
+N	 263
+tis_	 263
+_tr	 263
+'	 262
+!	 262
+_-_	 262
+pro	 261
+iu_	 261
+i�	 261
+nc	 260
+_si	 259
+du	 257
+_kon	 256
+ru	 255
+_vi_	 254
+_j	 253
+ce	 251
+ke_	 249
+ap	 248
+us	 247
+be	 247
+im	 247
+B	 246
+_ku	 246
+_La_	 246
+tra	 245
+ad	 245
+uj	 245
+ac	 245
+ita	 243
+pre	 242
+_pro	 242
+co	 241
+rm	 241
+_ni	 238
+_pe	 236
+?_	 234
+on.	 234
+toj	 234
+"_	 234
+j.	 234
+_ke_	 233
+s.	 232
+_A	 231
+av	 230
+ri_	 230
+_el	 229
+por	 229
+`	 224
+ev	 224
+las	 223
+P	 223
+j._	 221
+eni	 220
+_T	 220
+_B	 219
+j,_	 218
+j,	 218
+era	 217
+_in	 216
+on._	 216
+cx	 216
+_N	 215
+ion	 215
+ab	 215
+..	 214
+)	 213
+fi	 213
+or_	 212
+pri	 212
+s._	 212
+_por	 210
+ez	 210
+in_	 210
+am_	 209
+on,_	 209
+ll	 209
+�i	 209
+on,	 209
+_ve	 208
+ris	 208
+esti	 208
+!_	 207
+men	 206
+vas	 205
+iel	 204
+taj	 203
+_c	 201
+aro	 201
+ank	 200
+_pri	 200
+jo	 200
+ja	 200
+ont	 200
+lt	 199
+_P	 199
+igi	 199
+_pa	 197
+oj.	 197
+(	 196
+au	 195
+oro	 195
+ng	 195
+_(	 194
+sto	 194
+ast	 194
+ag	 193