language_detector 0.1.0

data/lib/textcat_ngrams/tagalog.lm

@@ -0,0 +1,400 @@
+_	 10664
+a	 6092
+n	 3421
+i	 2332
+g	 2149
+ng	 1488
+an	 1332
+g_	 1308
+ng_	 1243
+t	 1155
+a_	 1138
+o	 1137
+l	 1105
+s	 1067
+k	 1035
+_n	 957
+m	 909
+y	 801
+ang	 787
+u	 767
+ang_	 683
+p	 682
+na	 670
+_a	 634
+la	 596
+.	 572
+in	 561
+r	 559
+sa	 545
+_s	 502
+._	 487
+ma	 476
+ka	 473
+_na	 464
+b	 462
+_m	 458
+al	 448
+d	 445
+pa	 414
+n_	 412
+at	 394
+ak	 393
+h	 378
+ya	 358
+_k	 356
+ag	 350
+ni	 344
+_p	 339
+_sa	 335
+o_	 328
+ta	 308
+_an	 307
+iy	 300
+sa_	 296
+_ang	 294
+_ang_	 293
+t_	 290
+ay	 286
+ga	 283
+on	 280
+it	 275
+_sa_	 265
+ala	 258
+_ng	 256
+am	 252
+_i	 248
+_pa	 248
+i_	 243
+na_	 243
+an_	 240
+e	 239
+_ma	 237
+_ka	 235
+iya	 231
+y_	 229
+il	 228
+li	 228
+w	 226
+_ni	 225
+_na_	 224
+_ng_	 220
+as	 214
+ba	 207
+si	 206
+"	 195
+ti	 195
+ha	 186
+,	 179
+ar	 178
+ing	 173
+ra	 173
+A	 172
+ki	 168
+ong	 167
+_b	 167
+ap	 166
+,_	 162
+ong_	 161
+ko	 159
+ay_	 159
+un	 157
+ul	 155
+yo	 154
+to	 152
+_l	 150
+ah	 148
+is	 147
+hi	 147
+_t	 146
+lan	 145
+ama	 142
+niy	 139
+at_	 138
+_niy	 138
+aka	 136
+wa	 136
+niya	 135
+_niya	 135
+ab	 134
+-	 134
+di	 133
+_si	 132
+"_	 131
+aw	 129
+_d	 126
+_A	 126
+yan	 123
+ya_	 122
+ata	 120
+a.	 120
+gi	 120
+P	 120
+ing_	 118
+um	 115
+o.	 113
+aki	 113
+ri	 113
+ik	 112
+nd	 112
+ila	 111
+mo	 110
+da	 110
+a._	 110
+in_	 109
+la_	 107
+ali	 106
+S	 106
+man	 105
+ig	 105
+iya_	 105
+s_	 104
+_ak	 104
+_at	 103
+_h	 102
+yon	 102
+asa	 101
+ina	 101
+_P	 101
+n.	 99
+N	 98
+aa	 98
+ga_	 97
+_mo	 97
+_ba	 97
+_"	 95
+ito	 94
+bi	 94
+yang	 94
+n._	 93
+pag	 92
+lang	 92
+yang_	 92
+_la	 92
+o._	 90
+K	 90
+_at_	 90
+tu	 88
+_S	 88
+ara	 87
+nga	 87
+ro	 85
+apa	 83
+rr	 82
+lam	 82
+lo	 81
+nan	 81
+_N	 80
+aman	 79
+aha	 78
+mg	 78
+mga	 78
+mga_	 78
+_mga_	 78
+_mg	 78
+_mga	 78
+_K	 78
+siy	 77
+kan	 76
+it_	 76
+san	 76
+d_	 75
+ad	 75
+di_	 74
+tin	 74
+'	 74
+ati	 73
+siya	 73
+kin	 72
+M	 72
+lang_	 71
+mo_	 70
+_mo_	 70
+ako	 70
+uma	 70
+_pag	 69
+pi	 69
+l_	 68
+_siy	 68
+rrr	 68
+_siya	 67
+ula	 67
+_M	 66
+Pa	 66
+iyo	 66
+mi	 66
+bu	 66
+mu	 65
+no	 65
+pu	 65
+nag	 65
+ung	 65
+Na	 65
+ot	 64
+_Na	 64
+niya_	 64
+iyan	 64
+ku	 64
+k_	 63
+go	 62
+awa	 62
+ip	 61
+_Pa	 61
+lu	 61
+_di	 60
+pan	 60
+_ta	 60
+ini	 60
+isa	 60
+nt	 60
+iyang	 60
+_iyo	 59
+_iy	 59
+on_	 59
+tan	 59
+mang	 59
+aba	 59
+gan	 59
+ut	 58
+I	 58
+hin	 58
+nak	 58
+an.	 57
+akin	 57
+_r	 57
+han	 57
+Ka	 57
+_ay	 57
+_ako	 56
+may	 56
+iyon	 56
+rrrr	 55
+Sa	 55
+aga	 55
+to_	 55
+nit	 55
+_ko	 54
+er	 54
+ib	 54
+ari	 54
+ana	 54
+ili	 54
+an._	 54
+ahi	 54
+au	 54
+ala_	 54
+gk	 53
+pa_	 53
+_is	 53
+rin	 53
+ilan	 52
+_kan	 52
+_Ka	 52
+_it	 52
+_Sa	 51
+king	 51
+ko_	 51
+_nak	 51
+gin	 51
+_ay_	 50
+bo	 50
+_iyon	 50
+amang	 50
+'y	 49
+os	 49
+mang_	 49
+_pa_	 49
+kat	 49
+a,	 49
+An	 48
+Ma	 48
+ny	 48
+mag	 48
+_ku	 48
+_ito	 48
+_ha	 47
+yong	 47
+?	 47
+aking	 47
+T	 47
+ni_	 47
+yong_	 47
+_An	 47
+king_	 47
+_akin	 46
+sang	 46
+_nag	 46
+kas	 46
+_aki	 46
+_ni_	 46
+ayo	 45
+kit	 45
+'y_	 45
+mat	 45
+_Ma	 45
+lal	 45
+ot_	 45
+nya	 44
+ban	 44
+ndi	 44
+oo	 44
+_u	 44
+ngi	 44
+_hi	 44
+sang_	 44
+B	 43
+su	 43
+may_	 43
+rrrrr	 43
+p_	 43
+ita	 43
+wal	 43
+ika	 43
+abi	 43
+aan	 43
+_may	 43
+lama	 42
+naka	 42
+mal	 42
+_I	 42
+_ri	 42
+alan	 42
+any	 42
+im	 42
+_pu	 42
+ai	 42
+wala	 41
+anya	 41
+a,_	 41
+_B	 41
+ndi_	 41
+as_	 41
+pat	 41
+po	 41
+nang	 41
+_mag	 41
+laman	 40
+lala	 40
+kal	 40
+g-	 40
+..	 40
+ir	 40
+!	 40
+uk	 40
+gu	 39
+ito_	 39
+ro_	 39
+_g	 39
+_da	 39
+_isa	 39
+_lam	 39
+ilang	 39
+kanya	 39
+w_	 39
+kany	 39
+agk	 38
+pal	 38
+ka_	 38
+_naka	 38
+siya_	 38
+isan	 38

data/lib/textcat_ngrams/tamil.lm

@@ -0,0 +1,400 @@
+_	 11468
+�	 2697
+�	 2533
+�	 1960
+�	 1786
+�	 1249
+�	 1103
+�	 990
+�	 976
+�	 849
+�	 847
+�	 831
+�	 821
+�	 794
+�	 758
+�	 750
+�	 721
+�	 715
+�	 713
+.	 702
+�_	 671
+�	 655
+�	 648
+�	 625
+._	 611
+�	 601
+�	 571
+�	 557
+��	 556
+�	 524
+_�	 520
+�	 518
+_�	 492
+�	 489
+_�	 488
+�	 484
+_�	 482
+�	 461
+�	 442
+�	 418
+�	 416
+�	 403
+�_	 398
+��	 373
+_�	 368
+�	 360
+�	 359
+_�	 349
+��	 348
+��	 342
+�	 337
+_�	 331
+�	 331
+�_	 323
+_�	 321
+��	 320
+_�	 310
+տ	 308
+�	 299
+��	 294
+�	 292
+��	 292
+��	 290
+է	 288
+��	 284
+�_	 279
+�	 271
+��	 266
+��	 265
+_�	 265
+��	 241
+_�	 234
+��	 232
+=	 231
+�	 229
+��	 227
+==	 221
+��	 218
+�_	 218
+է_	 213
+===	 211
+�_	 210
+====	 201
+�_	 199
+�_	 198
+��	 197
+=====	 191
+,	 188
+ؽ	 188
+��	 187
+�	 186
+_�	 184
+,_	 184
+�	 183
+ԯ	 181
+�_	 179
+Ԩ	 178
+�	 172
+��_	 171
+�	 169
+��	 168
+���	 165
+��	 165
+�_	 165
+�	 164
+�	 161
+�_	 160
+ͨ	 160
+_ͨ	 157
+�	 156
+_�	 156
+��	 152
+Դ	 152
+ٻ	 151
+�	 150
+_�	 149
+�	 149
+�	 146
+��	 146
+ؽ�	 145
+��	 143
+�	 142
+׶	 142
+��	 142
+��	 140
+�_	 139
+ԧ	 138
+_��	 138
+��	 138
+��	 137
+ը	 137
+�_	 136
+��	 133
+_׶	 132
+��	 128
+���	 128
+��	 127
+��	 127
+_�	 127
+�	 127
+_ؽ	 127
+��	 125
+��	 125
+��	 124
+��	 122
+��	 121
+ޣ	 120
+��	 119
+��	 119
+׽	 119
+��	 119
+_�	 118
+�_	 118
+�.	 117
+���	 116
+�տ	 115
+��	 115
+_��	 113
+�.	 113
+׻	 113
+�._	 112
+��_	 112
+��	 111
+ա	 111
+��	 109
+״	 109
+�	 109
+�._	 108
+_��	 107
+մ	 106
+�_	 105
+��	 104
+'	 104
+��	 103
+�_	 102
+��	 102
+ش	 101
+��	 100
+��	 100
+_��	 99
+_׻	 99
+��	 98
+���	 98
+�	 98
+_׽	 97
+�_	 97
+Ի	 97
+_���	 97
+Ԩ_	 97
+�_	 97
+�	 96
+��	 96
+״�	 96
+�Ԩ	 95
+�ޣ	 94
+տ_	 94
+ػ	 94
+�	 94
+��	 93
+�	 93
+_ؽ�	 93
+��	 93
+ը_	 92
+��_	 91
+��	 91
+_��	 90
+��	 90
+��	 90
+��	 90
+_��	 90
+���	 90
+ٹ	 89
+_��	 89
+_��_	 88
+_��	 88
+��_	 88
+�_	 86
+硴	 86
+��	 86
+��	 85
+��	 85
+���	 84
+��	 84
+��	 83
+ԥ	 83
+�	 83
+_�	 83
+��	 82
+�_	 81
+Դ_	 81
+�.	 80
+_��	 80
+�է	 80
+�	 80
+��	 80
+�_	 79
+���	 79
+_��	 79
+��	 79
+ԣ	 79
+��	 78
+�_	 78
+�._	 77
+_��	 77
+���	 77
+_��	 77
+�	 77
+��	 76
+ޣ_	 75
+��_	 75
+پ	 75
+¯	 75
+��	 74
+��	 73
+��	 72
+�	 72
+��_	 71
+ԧ_	 71
+��	 71
+��	 71
+��	 70
+�է	 70
+���	 70
+��	 69
+ٻ_	 69
+��	 69
+��	 69
+�_	 68
+��	 68
+ؾ	 67
+_ش	 67
+բ	 66
+��	 66
+�Ԩ_	 66
+�	 66
+_ǻ	 66
+ǻ	 66
+Թ	 66
+�	 66
+�է_	 65
+��	 65
+_��	 64
+�.	 64
+��	 63
+ե	 63
+ԡ	 63
+_״	 63
+_�	 63
+ٴ	 62
+���	 62
+տ�	 62
+�._	 61
+����	 61
+_��	 61
+_׻�	 61
+׻�	 61
+���	 61
+��	 60
+��տ	 60
+_�	 60
+_����	 60
+Ծ	 60
+��	 59
+׶�	 59
+�ޣ_	 59
+��	 59
+_��	 58
+��_	 58
+��	 58
+��	 58
+��	 58
+���տ	 58
+��	 58
+��	 57
+�ԯ	 57
+��	 57
+��	 57
+_��	 57
+�	 56
+��	 56
+_׶�	 55
+?	 55
+��	 54
+��	 54
+��	 54
+���	 54
+_״�	 54
+����	 53
+�տ_	 53
+_�	 53
+���	 53
+����	 53
+硴�	 52
+��	 52
+��_	 52
+��	 51
+�_	 51
+'_	 51
+_��	 51
+��	 51
+ͨ�	 51
+��	 51
+�	 51
+�	 51
+_ͨ�	 51
+�_	 51
+տ�	 51
+��	 51
+_ɧ	 50
+ɧ	 50
+��	 50
+�է_	 50
+׽�	 50
+���	 50
+��	 49
+�_	 49
+��	 49
+���	 49
+��	 49
+��_	 49
+���	 48
+�ԧ	 48
+�	 48
+ա�	 48
+դ	 48
+��	 48
+��	 48
+�տ	 48
+_�硴	 47
+�_	 47
+�	 47
+ͨ�_	 47
+׶ԧ	 47
+�硴	 47
+�_	 47
+_ͨ�_	 47
+?_	 46
+ؿ	 46
+Կ	 46
+_ػ	 46
+��	 46
+_�	 46
+���_	 46
+�	 45
+���_	 45
+�ٻ	 45
+��	 45
+��	 45
+_׽�	 44
+��	 44
+ԣ_	 44
+���	 44
+��	 44
+�	 44
+��	 44
+���	 44
+�ԣ	 44
+�.	 44
+_׶ԧ	 43
+��	 43