language_detector 0.1.0

data/lib/textcat_ngrams/korean.lm

@@ -0,0 +1,400 @@
+_	 11636
+�	 2659
+�	 1629
+�	 1578
+�	 1458
+�	 1397
+�	 1299
+�	 1118
+�	 1005
+�	 928
+�	 849
+�	 840
+_�	 809
+�	 808
+�	 780
+�	 773
+�	 764
+�	 674
+�	 640
+�	 591
+�	 580
+��	 538
+�	 526
+�	 519
+�	 518
+_�	 518
+�	 506
+�	 483
+�	 464
+�	 463
+�	 454
+��	 451
+�	 404
+�	 395
+�	 384
+_�	 382
+�	 377
+.	 374
+�	 373
+�	 358
+._	 347
+_�	 344
+��	 338
+�_	 333
+�_	 329
+��	 327
+_�	 319
+��_	 317
+��.	 313
+�.	 313
+�	 308
+�	 308
+��._	 301
+�._	 301
+�_	 299
+_�	 291
+�	 288
+�	 282
+��	 279
+�	 278
+�	 274
+��_	 271
+�_	 269
+,	 266
+�	 266
+��	 261
+��	 258
+,_	 258
+��	 250
+�	 248
+_�	 246
+�	 245
+��	 244
+�_	 243
+_�	 243
+_�	 242
+�	 238
+�	 235
+��	 234
+��_	 234
+��	 234
+_�	 233
+��	 228
+��	 227
+��	 222
+��	 222
+_�	 215
+��	 212
+_�	 210
+�_	 208
+_�	 206
+�_	 206
+�_	 205
+_�	 205
+�_	 204
+�	 198
+��	 198
+��_	 194
+��_	 193
+�	 193
+��_	 193
+��_	 192
+��	 192
+_��	 190
+�_	 189
+��_	 189
+_�	 189
+��	 188
+��	 188
+��_	 188
+_�	 186
+�	 177
+�	 175
+�	 171
+��	 161
+�	 154
+_��	 153
+�	 152
+��	 151
+��	 150
+��	 147
+�	 147
+��	 145
+��	 143
+�	 142
+��	 142
+_�	 141
+��	 140
+�	 139
+�	 138
+�	 138
+�	 137
+��	 136
+�	 134
+��	 134
+_��	 133
+�	 132
+�_	 127
+��	 125
+��_	 123
+��	 122
+��_	 121
+��	 121
+��	 118
+�	 117
+�	 116
+�_	 115
+��	 115
+�_	 115
+�	 114
+ִ	 113
+��	 112
+��	 111
+�	 109
+��	 109
+�	 108
+�	 108
+_�	 108
+��	 107
+��_	 107
+�	 107
+�	 106
+�ִ	 105
+ȸ	 103
+��	 101
+�_	 101
+�	 100
+_�ִ	 100
+õ	 100
+���	 100
+"	 98
+����	 97
+��	 97
+���	 97
+��	 95
+_��	 94
+��	 94
+��	 94
+_��	 94
+�	 94
+��	 94
+��	 92
+��	 92
+_�	 91
+�	 91
+ϰ	 90
+��	 90
+���_	 90
+�	 89
+�	 89
+�_	 89
+�_	 89
+����_	 89
+��	 89
+�	 88
+���	 88
+���	 88
+����	 88
+�	 87
+��_	 87
+��	 85
+_�	 85
+�_	 84
+��	 84
+���	 84
+��	 84
+��	 83
+��	 83
+_��	 82
+�ϰ	 82
+��	 81
+��	 80
+����_	 79
+���_	 79
+��_	 78
+�	 78
+��	 78
+�_	 78
+��	 78
+'	 77
+��	 77
+�	 77
+�	 77
+��	 77
+�_	 76
+��_	 76
+�	 76
+�_	 75
+(	 74
+)	 74
+��	 74
+��	 74
+��	 73
+_��	 72
+_��	 72
+_��	 72
+-	 72
+��	 72
+ִ�	 72
+��	 71
+�ִ�	 71
+��	 70
+��	 70
+��	 70
+��	 69
+��	 69
+��_	 69
+_�ִ�	 69
+_��	 69
+�_	 69
+ߴ	 69
+��	 68
+��	 68
+_�	 68
+_��	 68
+�_	 67
+ü	 67
+��	 66
+_��_	 66
+�ߴ	 66
+��	 65
+�	 65
+_�	 65
+_��	 65
+�ߴ�	 64
+ߴ�	 64
+_��	 63
+��	 62
+�	 62
+ϴ	 62
+��	 62
+��_	 62
+��	 61
+��	 61
+ϰ�	 61
+̴	 60
+��	 60
+�ϰ�	 60
+�	 59
+_��_	 59
+_��	 59
+��	 59
+��	 58
+�_	 58
+_��	 58
+�_	 58
+��	 57
+ߴ�.	 57
+�	 57
+�	 57
+�ߴ�.	 57
+��	 57
+��	 57
+ִ�.	 56
+�_	 56
+�ִ�.	 56
+��	 56
+ߴ�._	 56
+��	 56
+��	 55
+��	 55
+��	 55
+Ʈ	 55
+��	 55
+�̴	 55
+ִ�._	 54
+_��	 54
+��_	 53
+��	 53
+�	 53
+��	 53
+_��	 53
+��	 53
+��	 53
+��_	 53
+��	 53
+��	 52
+_��	 52
+�_	 52
+��	 52
+_��	 51
+���	 51
+��	 51
+��	 51
+ȭ	 51
+�	 50
+_��	 50
+���	 50
+ġ	 50
+ϰ�_	 49
+�ϰ�_	 49
+_��	 49
+�	 49
+�	 49
+_"	 48
+��	 48
+��	 48
+�̴�	 48
+��	 48
+_���	 48
+̴�	 48
+��	 47
+̴�.	 47
+_��	 47
+�̴�.	 47
+��_	 47
+_��	 47
+��	 47
+��	 47
+�_	 46
+��	 46
+�_	 46
+�_	 46
+�	 46
+�ϴ	 46
+_��	 45
+_��	 45
+_��	 45
+��	 45
+��	 45
+��	 45
+̴�._	 45
+��	 44
+��	 44
+��	 44
+��_	 43
+(_	 43
+��	 43
+��	 43
+�_	 43
+��	 42
+��	 42
+�_	 42
+��_	 42
+_��_	 42
+��_	 42
+��	 42
+��	 41
+��	 41
+���	 41
+��	 41
+��	 40
+�õ	 40
+��	 40
+�	 40
+��	 40
+_��	 40
+��	 40
+��	 40
+_��	 40
+��_	 39
+_��	 39
+��	 39
+��	 39
+�_	 39
+_��	 39
+��	 38
+���	 38
+��	 38
+�_	 38

data/lib/textcat_ngrams/latin.lm

@@ -0,0 +1,400 @@
+_	 20136
+e	 6892
+i	 5604
+a	 5443
+u	 4581
+t	 4552
+s	 4354
+r	 3923
+n	 3375
+m	 3063
+o	 2921
+c	 2224
+l	 1805
+e_	 1625
+s_	 1503
+p	 1424
+d	 1397
+,	 1285
+,_	 1276
+er	 1077
+qu	 1028
+q	 1028
+a_	 1019
+t_	 1018
+is	 942
+_a	 921
+re	 902
+m_	 891
+v	 858
+b	 821
+um	 808
+_s	 773
+us	 772
+en	 766
+nt	 733
+in	 729
+ue	 727
+te	 720
+g	 718
+_i	 710
+_p	 679
+it	 676
+_c	 669
+et	 653
+que	 652
+_e	 643
+at	 643
+ue_	 616
+ra	 614
+que_	 611
+f	 601
+or	 598
+ri	 576
+ti	 572
+ta	 559
+tu	 552
+an	 551
+ae	 527
+_m	 513
+am	 501
+_t	 493
+us_	 488
+is_	 487
+es	 479
+em	 479
+_f	 451
+um_	 443
+_v	 442
+ia	 442
+li	 438
+_d	 436
+.	 432
+i_	 430
+et_	 429
+ni	 412
+ne	 409
+h	 406
+de	 404
+ur	 396
+._	 392
+ar	 388
+os	 388
+mi	 382
+pe	 382
+la	 376
+st	 371
+s,	 368
+di	 367
+_et	 366
+s,_	 365
+_in	 363
+on	 360
+o_	 359
+_n	 351
+_et_	 351
+as	 346
+im	 336
+na	 327
+se	 320
+ma	 315
+cu	 307
+vi	 306
+si	 303
+ro	 303
+r_	 302
+su	 299
+un	 295
+_l	 291
+to	 291
+ec	 290
+ci	 288
+co	 287
+_r	 287
+ere	 286
+ce	 284
+tr	 280
+re_	 278
+ent	 275
+x	 275
+ct	 274
+ve	 271
+ru	 259
+ul	 256
+me	 255
+ui	 255
+c_	 252
+_o	 250
+ic	 249
+ns	 247
+_qu	 242
+_q	 242
+no	 241
+ant	 235
+am_	 235
+_co	 233
+sa	 231
+ca	 230
+t,	 226
+mu	 225
+t,_	 225
+_re	 223
+el	 222
+ib	 222
+id	 218
+om	 212
+_te	 211
+al	 209
+le	 209
+it_	 208
+mo	 208
+ol	 206
+_u	 203
+;	 199
+_h	 199
+ac	 198
+;_	 198
+bu	 197
+nu	 196
+ua	 195
+n_	 195
+ll	 194
+tis	 191
+A	 189
+rt	 188
+ge	 188
+nd	 187
+au	 187
+lu	 186
+iu	 185
+squ	 185
+per	 185
+sq	 185
+ter	 185
+pa	 183
+_A	 183
+em_	 183
+ia_	 180
+ed	 179
+_pe	 178
+m,	 176
+sque	 175
+_su	 175
+ae_	 175
+m,_	 175
+pr	 175
+bi	 175
+bus	 174
+_vi	 174
+os_	 173
+ta_	 172
+mqu	 171
+mq	 171
+ss	 170
+sque_	 169
+ibu	 167
+ad	 166
+ibus	 165
+I	 164
+nte	 163
+ra_	 163
+mque	 162
+_de	 162
+po	 161
+_se	 160
+ere_	 160
+nc	 160
+qua	 159
+T	 159
+lo	 157
+oc	 156
+mque_	 156
+_T	 155
+_pa	 155
+_pr	 155
+tem	 154
+bus_	 152
+nti	 149
+rum	 149
+er_	 149
+ab	 148
+ir	 148
+da	 147
+_ve	 146
+ibus_	 146
+ex	 146
+ut	 145
+pi	 145
+tur	 145
+_ca	 143
+_me	 142
+es_	 142
+gi	 142
+te_	 141
+_I	 141
+vo	 141
+do	 141
+_si	 140
+tus	 139
+il	 137
+_ar	 136
+du	 133
+nt_	 133
+uc	 133
+fa	 132
+as_	 132
+rr	 131
+ba	 130
+_ad	 128
+ne_	 127
+_ma	 127
+ens	 127
+gn	 126
+s.	 126
+y	 126
+min	 125
+ris	 124
+in_	 123
+tum	 123
+P	 123
+_g	 123
+mp	 123
+e,	 122
+io	 122
+_P	 122
+ea	 122
+hi	 122
+e,_	 121
+era	 120
+sc	 120
+_la	 120
+qui	 120
+unt	 120
+fe	 119
+_in_	 118
+_no	 118
+ore	 118
+iam	 118
+va	 117
+tis_	 117
+s._	 117
+at_	 117
+eri	 116
+d_	 116
+con	 115
+fu	 115
+pu	 114
+cum	 114
+ub	 114
+ng	 114
+ine	 113
+_au	 113
+:	 113
+_di	 112
+ag	 111
+_con	 111
+ect	 111
+i,	 111
+equ	 111
+i,_	 111
+be	 111
+eq	 111
+_po	 110
+so	 110
+:_	 110
+nis	 109
+ha	 109
+uo	 109
+_fa	 108
+na_	 107
+ip	 107
+is,	 107
+_cu	 106
+cr	 106
+ate	 105
+is,_	 105
+ig	 105
+tor	 105
+rat	 104
+_qua	 103
+eg	 103
+a,	 103
+a,_	 102
+tra	 102
+_mo	 101
+sp	 101
+mis	 100
+itu	 100
+D	 99
+ali	 99
+eb	 99
+eni	 99
+_sa	 98
+ie	 98
+imu	 98
+_ex	 97
+_D	 96
+res	 95
+est	 94
+tri	 94
+ene	 94
+_mi	 94
+str	 94
+enti	 93
+t.	 92
+av	 92
+_per	 91
+ur_	 91
+ora	 91
+lt	 91
+umqu	 90
+_vo	 90
+umq	 90
+up	 89
+t._	 88
+quo	 88
+_ne	 88
+gen	 88
+rum_	 87
+tqu	 87
+tq	 87
+_fu	 86
+ep	 86
+ma_	 86
+umque	 86
+it,_	 85
+ine_	 85
+it,	 85
+men	 85
+mus	 84
+ort	 83
+ven	 83
+ina	 83
+us,_	 83
+us,	 83
+tque	 82
+_ge	 82
+per_	 82
+mor	 82
+inc	 82
+are	 81
+tus_	 81
+_an	 81
+rim	 81
+tque_	 81
+ot	 81
+ani	 80
+H	 80
+_tu	 80
+ho	 80
+tem_	 80
+u_	 80
+ser	 79
+um,_	 79
+um,	 79
+S	 79
+ten	 79
+ver	 79
+sti	 79
+ntu	 78
+fer	 78