language_detector 0.1.0

data/lib/textcat_ngrams/estonian.lm

@@ -0,0 +1,400 @@
+_	 20738
+a	 7004
+e	 5699
+i	 5321
+s	 4731
+t	 3769
+l	 3448
+u	 3446
+n	 2902
+k	 2584
+d	 2202
+m	 2043
+a_	 1758
+o	 1684
+r	 1429
+g	 1174
+v	 1151
+e_	 1139
+i_	 1136
+_k	 1136
+s_	 1077
+h	 1009
+,	 995
+,_	 995
+_t	 953
+p	 953
+j	 940
+�	 900
+is	 896
+st	 851
+se	 841
+_s	 822
+.	 821
+as	 801
+d_	 801
+le	 800
+ta	 794
+in	 793
+_m	 790
+�	 754
+._	 753
+t_	 746
+ma	 710
+_p	 680
+si	 677
+_v	 660
+es	 636
+al	 626
+us	 619
+el	 602
+_o	 596
+_e	 586
+ja	 580
+_j	 563
+te	 562
+�	 549
+li	 532
+va	 515
+id	 501
+ol	 498
+tu	 497
+da	 490
+_n	 480
+ku	 478
+ud	 459
+nu	 455
+na	 438
+ei	 432
+ks	 418
+mi	 411
+ee	 411
+u_	 407
+ka	 400
+n_	 394
+b	 394
+ga	 386
+_l	 384
+_a	 380
+an	 366
+ja_	 365
+et	 358
+me	 358
+l_	 350
+at	 348
+la	 341
+ad	 340
+st_	 339
+ne	 336
+ll	 333
+_ta	 332
+ra	 330
+_ja	 328
+ik	 323
+en	 318
+ni	 308
+ul	 305
+sa	 302
+_ol	 302
+nd	 299
+_ja_	 299
+nud	 296
+ii	 291
+ko	 286
+_se	 285
+le_	 283
+aa	 281
+is_	 281
+gi	 270
+_te	 269
+ag	 269
+_va	 268
+_ku	 267
+ed	 262
+em	 255
+_mi	 255
+ma_	 247
+ti	 246
+ri	 245
+_h	 242
+gu	 239
+id_	 238
+ast	 237
+it	 236
+ga_	 236
+un	 232
+de	 230
+ud_	 230
+ha	 230
+ak	 228
+ah	 228
+uu	 228
+il	 227
+�i	 226
+as_	 223
+ke	 222
+ar	 220
+a,	 220
+am	 220
+_ko	 220
+a,_	 220
+_ka	 220
+ai	 220
+eg	 216
+sin	 214
+est	 214
+ui	 214
+he	 214
+ks_	 213
+�	 213
+oo	 213
+ju	 207
+�r	 205
+ut	 203
+in_	 203
+oli	 201
+ki	 199
+su	 199
+es_	 199
+lt	 198
+ist	 188
+li_	 186
+ea	 186
+vi	 184
+im	 181
+mu	 181
+se_	 180
+ts	 180
+on	 178
+ise	 178
+ta_	 177
+ek	 176
+_oli	 176
+sel	 173
+nud_	 173
+_�	 172
+a.	 171
+nn	 170
+ema	 169
+ng	 168
+lu	 168
+ge	 167
+_si	 166
+_ei	 165
+_i	 165
+_ei_	 164
+ei_	 164
+_r	 163
+ole	 161
+pa	 160
+lle	 160
+a._	 160
+ust	 159
+du	 156
+er	 156
+v�	 153
+da_	 153
+min	 152
+et_	 151
+d,_	 149
+_M	 149
+ht	 149
+d,	 149
+M	 149
+kui	 148
+_et	 147
+K	 147
+_K	 146
+pe	 145
+gi_	 145
+_v�	 145
+or	 144
+_tu	 142
+lt_	 141
+_ma	 141
+asi	 140
+ve	 139
+us_	 138
+ig	 136
+sin_	 136
+ur	 135
+_ta_	 134
+di	 134
+_et_	 134
+s,_	 132
+tas	 132
+s,	 132
+_kui	 131
+sk	 131
+re	 130
+po	 129
+oli_	 129
+om	 129
+�i	 128
+inu	 128
+_na	 128
+_oli_	 128
+_sa	 128
+aj	 128
+mis	 127
+ui_	 127
+_me	 127
+_pa	 126
+tus	 125
+pi	 125
+te_	 124
+�l	 123
+-	 123
+est_	 122
+_on	 121
+kk	 121
+tt	 120
+aga	 119
+na_	 119
+_T	 119
+T	 119
+b_	 118
+al_	 118
+sta	 118
+_mu	 116
+_ju	 116
+ida	 116
+aks	 116
+gu_	 116
+_ni	 116
+s.	 116
+ad_	 116
+_pe	 114
+eks	 114
+ev	 114
+end	 113
+s._	 113
+use	 111
+�ra	 111
+_po	 111
+_min	 110
+S	 110
+aja	 110
+_la	 110
+ele	 109
+el_	 108
+on_	 108
+ab	 108
+_S	 108
+av	 107
+ing	 107
+kui_	 106
+_on_	 106
+au	 104
+ne_	 104
+ti_	 104
+ell	 103
+ae	 101
+k�	 101
+ed_	 100
+_ke	 99
+ata	 99
+iis	 99
+!	 98
+!_	 98
+sid	 98
+nda	 98
+eh	 98
+lle_	 97
+pu	 97
+��	 97
+v�i	 97
+ine	 96
+t,	 96
+e,_	 96
+ale	 96
+_v�i	 96
+t,_	 96
+e,	 96
+eda	 96
+uk	 95
+ast_	 95
+ld	 95
+?	 94
+_kui_	 94
+_sel	 93
+_k�	 93
+tul	 93
+ega	 93
+lg	 92
+sii	 92
+val	 92
+e.	 92
+_su	 92
+ug	 92
+oh	 92
+k�	 92
+d.	 91
+ee_	 91
+see	 91
+e._	 91
+��	 91
+oma	 91
+_ole	 90
+ses	 90
+stu	 90
+�t	 90
+��	 90
+_om	 89
+me_	 89
+ot	 89
+d._	 89
+_sii	 88
+to	 88
+_en	 87
+atu	 87
+?_	 87
+A	 86
+J	 86
+pea	 86
+j�	 85
+_A	 85
+_see	 85
+ime	 84
+_pi	 84
+_ha	 84
+m�	 84
+n�	 84
+_J	 84
+les	 84
+ste	 84
+kas	 84
+_�	 84
+v�	 83
+E	 83
+p�	 83
+_ve	 83
+_E	 83
+eis	 82
+_j�	 81
+_pea	 81
+_m�	 80
+um	 80
+_k�	 80
+iku	 80
+�d	 80
+all	 79
+eid	 79
+ba	 79
+_v�	 79
+ina	 78
+lj	 78
+sid_	 78
+hu	 78
+tun	 78
+l�	 78
+_oma	 77
+i,_	 77
+i,	 77
+agu	 77
+uh	 77
+lm	 76
+ras	 76
+ss	 76
+k�	 76
+ees	 76

data/lib/textcat_ngrams/finnish.lm

@@ -0,0 +1,400 @@
+_	 19984
+a	 9133
+i	 8384
+t	 7797
+e	 6481
+n	 6431
+s	 5897
+l	 4504
+o	 4163
+u	 4106
+k	 4013
+�	 3354
+n_	 2868
+m	 2569
+a_	 1987
+v	 1905
+r	 1827
+ta	 1580
+en	 1553
+is	 1515
+h	 1508
+y	 1462
+st	 1390
+in	 1375
+p	 1342
+j	 1333
+an	 1139
+si	 1073
+tt	 1030
+te	 1008
+en_	 982
+_k	 980
+it	 974
+ll	 947
+aa	 942
+�_	 902
+va	 878
+el	 855
+_t	 851
+ka	 846
+i_	 835
+.	 832
+se	 818
+li	 806
+t�	 804
+oi	 767
+ai	 744
+._	 739
+tu	 734
+_o	 719
+mi	 715
+al	 703
+on	 684
+d	 681
+_v	 662
+et	 654
+_j	 641
+t_	 635
+ti	 632
+_m	 628
+_s	 620
+ja	 616
+ma	 596
+sa	 595
+la	 582
+ist	 575
+_e	 565
+to	 565
+ks	 557
+in_	 554
+es	 551
+il	 538
+an_	 536
+ki	 527
+,	 525
+ku	 525
+,_	 524
+us	 520
+as	 514
+nt	 512
+ri	 495
+ke	 494
+at	 491
+_p	 485
+le	 484
+ik	 483
+ss	 477
+ut	 469
+�	 469
+sta	 460
+ee	 459
+uu	 458
+ol	 457
+ta_	 451
+ne	 445
+��	 445
+ei	 443
+uo	 436
+ko	 433
+un	 430
+lu	 421
+ii	 420
+e_	 418
+nn	 413
+_h	 412
+ar	 408
+er	 402
+�n	 396
+ja_	 386
+im	 381
+on_	 365
+_va	 363
+aan	 354
+_a	 352
+me	 350
+ak	 345
+ssa	 331
+na	 330
+ie	 329
+pa	 327
+_ja	 326
+ia	 325
+t�_	 322
+_l	 319
+vi	 317
+ise	 316
+tta	 315
+de	 314
+os	 312
+lli	 309
+_ja_	 304
+jo	 295
+v�	 290
+su	 289
+au	 287
+lis	 286
+_on	 285
+s�	 284
+uk	 280
+am	 280
+ot	 280
+ty	 275
+ett	 271
+tt�	 270
+ni	 269
+l�	 267
+ksi	 264
+nk	 264
+ht	 263
+ul	 261
+ell	 261
+sa_	 259
+ha	 257
+sen	 257
+a.	 254
+isi	 253
+ste	 253
+aan_	 252
+_on_	 252
+_ka	 252
+sk	 251
+kk	 246
+itt	 245
+ok	 242
+a._	 239
+all	 239
+yt	 239
+m�	 237
+mu	 237
+av	 237
+_y	 236
+lla	 233
+taa	 231
+ais	 231
+een	 230
+K	 230
+lt	 228
+s_	 227
+ast	 227
+iv	 226
+ssa_	 225
+ra	 225
+-	 223
+kse	 223
+oit	 220
+om	 220
+T	 219
+_ku	 218
+�n_	 216
+aa_	 214
+at_	 214
+tel	 211
+ui	 210
+si_	 208
+rk	 207
+sta_	 207
+_jo	 203
+k�	 202
+_K	 201
+est	 200
+em	 200
+he	 199
+_n	 199
+vo	 198
+_ta	 196
+eh	 196
+_ol	 196
+S	 196
+nta	 196
+_ko	 194
+je	 194
+st�	 194
+�r	 193
+ust	 191
+mis	 191
+ns	 190
+pu	 189
+nen	 188
+�t	 188
+toi	 188
+iin	 187
+ten	 187
+min	 186
+ista	 185
+hd	 184
+a,	 184
+a,_	 184
+sen_	 183
+E	 182
+lle	 181
+vat	 179
+ill	 177
+no	 176
+p�	 176
+lm	 176
+llis	 175
+n.	 175
+io	 172
+ine	 171
+n._	 170
+pi	 169
+uks	 168
+ava	 168
+��n	 166
+nen_	 165
+ah	 165
+_mu	 164
+tus	 163
+mm	 162
+_to	 162
+ek	 160
+int	 159
+_r	 159
+lin	 158
+oim	 158
+_T	 158
+A	 158
+imi	 157
+t�	 157
+la_	 157
+j�	 157
+aj	 156
+yh	 155
+o_	 154
+lo	 154
+oli	 153
+een_	 153
+le_	 153
+_si	 153
+g	 152
+aik	 151
+vat_	 150
+L	 149
+ur	 149
+ti_	 149
+sia	 148
+ite	 147
+inen	 147
+ain	 146
+sti	 146
+lla_	 146
+ys	 145
+_mi	 145
+val	 144
+stu	 144
+�m	 144
+alli	 143
+pe	 143
+utt	 142
+et_	 141
+_tu	 141
+eri	 140
+_E	 140
+:	 140
+nki	 139
+ir	 139
+ll�	 138
+up	 138
+�i	 137
+ama	 137
+_ha	 135
+id	 135
+_se	 135
+po	 134
+inen_	 134
+tte	 133
+nna	 133
+ten_	 132
+or	 132
+ts	 131
+n�	 131
+yk	 131
+�s	 131
+_S	 130
+ses	 130
+ve	 130
+ess	 129
+�l	 129
+ita	 129
+lai	 129
+H	 129
+van	 127
+�k	 127
+kin	 127
+N	 127
+_te	 126
+den	 126
+tee	 126
+P	 126
+kaa	 126
+iin_	 125
+kun	 125
+ois	 125
+sit	 125
+oh	 124
+V	 124
+y�	 124
+�v	 124
+tav	 124
+voi	 124
+ia_	 123
+I	 123
+oll	 123
+maa	 122
+ih	 122
+oj	 122
+rj	 121
+ro	 121
+ikk	 120
+so	 120
+oo	 120
+oimi	 120
+do	 120
+pp	 119
+M	 119
+_ei	 118
+toim	 118
+op	 118
+uut	 118
+tet	 118
+_i	 118
+_ma	 117
+vai	 117
+l�_	 116
+u_	 116
+sy	 116
+kau	 116
+utta	 116
+un_	 115
+eu	 115
+ss�	 115
+tti	 115
+_sa	 115
+mp	 114
+eis	 114
+ka_	 112
+ett�	 112
+taa_	 111
+_et	 111
+hu	 111
+itu	 111
+suu	 111
+den_	 111
+ksen	 110
+ap	 110
+_ke	 110
+uv	 110
+tam	 110
+yv	 109
+aup	 109
+st�_	 109
+asta	 109
+�y	 109
+kan	 108
+nu	 108
+ukse	 108
+_toi	 107
+ien	 107
+hi	 107
+iss	 107