language_detector 0.1.0

data/lib/textcat_ngrams/hindi.lm

@@ -0,0 +1,400 @@
+_	 75620
+�	 19109
+U	 16333
+�	 11131
+�	 11107
+�	 10241
+��	 7855
+�	 6993
+�U	 6598
+�	 6322
+�	 6151
+�	 5887
+_�	 5800
+�	 5370
+�	 5168
+U_	 5118
+�_	 4935
+�	 4508
+�	 4467
+�	 4292
+�_	 3962
+_��	 3937
+�_	 3904
+�	 3718
+�_	 3609
+�	 3563
+�U	 3514
+_�	 3394
+�	 3363
+�	 2962
+�	 2907
+�U_	 2851
+_�	 2668
+_�	 2632
+_�	 2575
+�	 2527
+�	 2513
+�_	 2512
+�	 2451
+U�	 2336
+�	 2314
+_�	 2280
+_�	 2260
+�	 2187
+�	 2082
+�	 2078
+��	 1988
+�	 1913
+���	 1886
+��	 1881
+�	 1877
+��	 1860
+�	 1842
+�_	 1830
+��_	 1746
+�	 1714
+��	 1703
+�_	 1695
+��_	 1694
+��	 1631
+�	 1610
+_��	 1579
+_��	 1567
+�_	 1556
+��U	 1536
+��	 1528
+��	 1525
+U�	 1515
+_��_	 1488
+�_	 1411
+��_	 1407
+�	 1390
+_��	 1357
+��	 1357
+�	 1337
+�_	 1319
+_�	 1315
+��	 1300
+�	 1279
+�	 1270
+_�	 1258
+_���	 1209
+緤	 1203
+��	 1195
+_�	 1187
+�	 1181
+�	 1166
+��	 1163
+��	 1162
+�	 1157
+�	 1155
+_��	 1150
+��_	 1140
+���	 1127
+��_	 1092
+_��	 1086
+_��_	 1077
+�U	 1075
+_�U	 1071
+���_	 1065
+��U	 1062
+_��U	 1062
+��	 1053
+��	 1052
+��	 1038
+��	 1035
+_�	 1009
+��U	 972
+��	 970
+_緤	 967
+_���	 959
+�U�	 936
+�U�	 930
+_���_	 929
+���	 928
+�	 926
+_��	 904
+�U	 904
+�	 904
+�	 890
+�	 887
+�U	 885
+���U	 866
+_�	 865
+_�U	 862
+_�	 854
+��_	 850
+_�	 847
+_�	 837
+S	 834
+U�	 831
+_�	 830
+U�_	 823
+_�	 798
+��	 788
+�_	 783
+_�U	 773
+U�_	 770
+��	 753
+��	 752
+�	 749
+��_	 732
+��	 732
+�_	 704
+_�	 704
+��	 704
+_��	 697
+��	 695
+�U�	 693
+��_	 690
+�_	 688
+��U_	 685
+,	 684
+_�	 682
+���_	 679
+,_	 674
+_���	 670
+�	 665
+�U	 664
+��	 648
+_��	 647
+U�	 644
+�_	 643
+_�	 641
+緤_	 639
+�_	 639
+��	 633
+_��_	 633
+��	 623
+U�	 621
+��_	 615
+_���U	 611
+U�_	 599
+�U_	 595
+��	 594
+U�	 589
+�U�	 580
+_��U�	 580
+��U�	 580
+_�	 576
+_��	 573
+�U�_	 572
+��U�_	 572
+��	 569
+_�	 569
+_��	 569
+��	 561
+_緤_	 557
+�	 556
+��	 552
+��	 550
+��U	 549
+��	 544
+��	 538
+_���_	 535
+�_	 532
+��	 525
+��	 523
+���	 519
+��_	 502
+�	 501
+�	 498
+_��_	 495
+�	 494
+_��	 493
+��	 493
+��_	 490
+_��U	 485
+��	 484
+�	 481
+�	 478
+��	 469
+.	 459
+��U_	 458
+��	 449
+��	 449
+_��	 445
+��	 444
+��U_	 443
+��	 442
+_��	 440
+_��U_	 439
+��	 437
+��_	 435
+�	 434
+_�	 433
+�U�	 431
+�	 431
+_���	 430
+��	 427
+��	 425
+�_	 420
+��_	 419
+�U	 415
+�	 412
+��	 412
+��	 410
+�U	 405
+�U	 399
+��	 399
+���U_	 394
+U�	 394
+��_	 393
+��	 393
+��_	 392
+���_	 392
+��	 389
+�U�	 385
+��	 385
+Ǹ	 383
+��_	 382
+��	 381
+��	 379
+��	 375
+�_	 374
+�U�	 374
+�U�	 374
+��	 373
+��_	 372
+_���_	 371
+_��	 369
+��	 364
+��U	 364
+�U_	 362
+���	 358
+Ȥ	 357
+U��	 354
+�	 353
+�	 352
+�_	 350
+U�	 349
+���	 346
+��	 346
+����	 346
+��	 345
+��	 344
+��U	 342
+����	 342
+_��	 341
+_��	 341
+_��	 340
+�U_	 337
+_��U_	 337
+�	 337
+��U_	 337
+��	 336
+�U�_	 334
+��	 334
+�	 334
+_��_	 332
+��	 327
+���U	 327
+��	 326
+_�	 325
+U��	 323
+����U	 323
+U�	 322
+�U�	 321
+_��	 320
+ǸU	 319
+_��	 319
+��	 319
+��U_	 319
+_�U�	 318
+��	 317
+�_	 316
+_�	 316
+�	 313
+��	 310
+_�U�	 309
+��	 309
+�	 308
+��U	 308
+_��	 308
+_��U	 307
+_����	 304
+_���	 304
+���	 304
+_���U	 304
+_��	 304
+���U	 304
+���U_	 303
+_��	 303
+�U�_	 300
+U�_	 300
+_S	 300
+��	 300
+U�	 298
+�U�_	 296
+��	 296
+��	 294
+�U_	 294
+��_	 293
+��	 292
+��	 291
+_��	 290
+��_	 290
+��	 290
+��_	 289
+U�U	 288
+��U	 288
+U�	 288
+_��U	 286
+_�	 286
+_��U�	 285
+_��	 285
+��U�	 285
+淤	 284
+_��	 284
+��U�	 283
+U�_	 282
+��	 280
+_��	 280
+�U	 280
+��	 279
+_��	 278
+_��U	 278
+��U	 278
+�_	 277
+_��	 275
+���	 274
+��_	 273
+��	 272
+���	 272
+��	 272
+��_	 272
+_�U�	 272
+��_	 272
+-	 271
+��U	 271
+���U	 271
+�_	 271
+_���	 270
+�U�_	 270
+_���U	 269
+�U�U	 268
+��	 267
+��	 265
+��	 263
+��	 263
+���_	 262
+����_	 262
+���	 262
+��U	 258
+_��	 258
+��_	 258
+_���	 257
+�U	 256
+v	 253
+��	 252
+_�U�	 252
+��	 249
+��	 247
+��	 246
+_�	 246
+�U�	 246
+��	 245
+._	 245
+U�	 244
+�U	 243

data/lib/textcat_ngrams/hungarian.lm

@@ -0,0 +1,400 @@
+_	 19186
+e	 5753
+a	 4627
+t	 4522
+s	 3480
+l	 3437
+n	 3137
+k	 3036
+i	 2527
+r	 2437
+z	 2399
+o	 2303
+�	 2039
+�	 1995
+g	 1978
+m	 1695
+y	 1338
+_a	 1256
+b	 1186
+d	 1148
+a_	 1108
+v	 1057
+t_	 901
+sz	 889
+el	 832
+,	 819
+,_	 818
+h	 792
+k_	 769
+.	 767
+et	 743
+gy	 711
+s_	 705
+_m	 702
+_a_	 695
+en	 671
+�	 662
+n_	 646
+_k	 645
+j	 623
+._	 613
+i_	 606
+eg	 601
+p	 586
+_e	 580
+u	 579
+le	 576
+�	 542
+er	 495
+f	 485
+ek	 477
+te	 477
+�s	 473
+_s	 471
+al	 464
+ta	 458
+�	 453
+_h	 444
+_t	 442
+an	 426
+ze	 425
+me	 406
+at	 405
+l_	 401
+es	 395
+�	 387
+y_	 381
+z_	 375
+tt	 374
+ke	 372
+_v	 369
+�s	 368
+ak	 367
+_�	 365
+ny	 363
+t�	 359
+c	 358
+re	 350
+to	 347
+A	 343
+e_	 340
+�	 332
+ne	 330
+os	 326
+�l	 320
+_f	 320
+az	 317
+zt	 317
+�r	 317
+_n	 315
+ko	 312
+_A	 303
+_sz	 302
+is	 301
+ve	 299
+gy_	 297
+�t	 293
+_b	 293
+ra	 291
+or	 289
+ol	 284
+_i	 281
+em	 279
+_l	 274
+la	 264
+ez	 262
+be	 260
+lt	 260
+ok	 260
+ye	 256
+_me	 252
+on	 251
+en_	 247
+ar	 245
+_az	 245
+in	 243
+�n	 242
+se	 242
+�g	 238
+egy	 237
+ha	 237
+r_	 237
+t�	 237
+�r	 235
+sze	 233
+�n	 226
+ly	 224
+g_	 221
+"	 221
+ll	 219
+iz	 214
+de	 214
+ek_	 213
+mi	 212
+rt	 211
+ba	 209
+ho	 209
+A_	 209
+�s_	 209
+az_	 205
+va	 204
+ag	 203
+ka	 202
+na	 202
+og	 201
+ik	 201
+nt	 200
+_A_	 199
+�	 199
+-	 198
+tt_	 198
+_az_	 197
+ni	 195
+cs	 194
+ki	 190
+k�	 187
+�t	 187
+_eg	 187
+nd	 186
+fe	 185
+_�s	 185
+l�	 182
+bi	 181
+nk	 180
+_le	 179
+an_	 179
+_�s_	 177
+t�s	 175
+k�	 174
+meg	 173
+_egy	 172
+ma	 171
+as	 170
+sa	 170
+s�	 169
+ge	 167
+ot	 167
+za	 167
+E	 166
+m_	 165
+_meg	 165
+_el	 165
+bb	 164
+ro	 164
+z�	 163
+he	 162
+m�	 161
+s�	 160
+_r	 160
+s�t	 160
+tos	 159
+ti	 159
+st	 158
+_j	 158
+�l	 157
+it	 156
+_ho	 156
+�l	 156
+_ha	 155
+v�	 154
+am	 152
+oz	 152
+ele	 151
+ya	 151
+zto	 150
+�	 149
+biz	 147
+so	 147
+et_	 145
+izto	 144
+izt	 144
+ap	 141
+"_	 141
+ed	 141
+ss	 140
+bizt	 140
+�k	 140
+bizto	 140
+iztos	 139
+�t	 139
+ztos	 139
+os�t	 138
+zet	 138
+os�	 138
+m�	 137
+_is	 137
+t,	 136
+tos�t	 136
+tos�	 136
+t,_	 136
+�_	 135
+agy	 135
+ztos�	 135
+li	 134
+om	 134
+_fe	 134
+ere	 133
+�g	 133
+t.	 132
+nek	 131
+vi	 129
+_d	 129
+zo	 128
+k,_	 128
+k,	 128
+_k�	 127
+_p	 127
+M	 126
+let	 126
+ak_	 125
+j�	 125
+ett	 125
+�	 124
+si	 124
+�sz	 123
+_E	 123
+�p	 123
+v�	 123
+r�	 123
+t._	 123
+is_	 123
+S	 123
+�t�s	 122
+�t�	 122
+kor	 121
+ai	 121
+fel	 120
+da	 120
+_mi	 120
+pe	 119
+ogy	 118
+ban	 118
+ad	 117
+ga	 116
+_va	 116
+ott	 114
+_ne	 114
+_ki	 113
+�r	 113
+z�	 112
+ben	 112
+_te	 111
+zi	 111
+s�	 110
+s�g	 109
+do	 109
+t�	 108
+em_	 108
+_"	 108
+_ta	 108
+_M	 107
+ogy_	 107
+_�	 107
+k.	 106
+�l	 105
+_ke	 105
+_g	 104
+:	 103
+gye	 102
+�k	 102
+hog	 102
+ri	 102
+mo	 101
+ok_	 101
+:_	 101
+hogy	 101
+il	 101
+el_	 100
+zer	 100
+ete	 99
+nn	 99
+nak	 98
+je	 98
+s�t�	 98
+sz�	 98
+yo	 98
+os�t�	 98
+s�t�s	 98
+_k�	 98
+_hog	 98
+l�	 97
+_S	 97
+_hogy	 97
+ig	 97
+_-	 96
+hogy_	 96
+ban_	 96
+ese	 95
+_bi	 94
+fo	 94
+ja	 94
+ul	 94
+�_	 94
+k._	 94
+_c	 93
+k�	 91
+es_	 91
+n�	 91
+�ny	 91
+g�	 91
+�s_	 91
+egy_	 90
+�b	 90
+rd	 89
+I	 89
+�ny	 89
+_biz	 89
+_fel	 88
+�v	 88
+ala	 88
+szer	 88
+po	 88
+_ma	 88
+leg	 88
+t�	 88
+ket	 87
+un	 87
+di	 87
+ai_	 87
+nek_	 87
+rm	 86
+tal	 86
+�v	 85
+_is_	 85
+nem	 85
+ti_	 84
+�z	 84
+szt	 84
+ut	 83
+ter	 83
+d�	 83
+kk	 83
+or_	 83
+b_	 82
+n�	 82
+os_	 82
+re_	 82
+rs	 82
+_bizt	 82
+min	 82
+ben_	 81
+ra_	 81
+ik_	 81
+go	 80
+len	 80
+lm	 80
+�t	 80
+ely	 80
+aj	 80
+�l	 80
+_sze	 80
+_be	 79
+ev	 79
+r�	 79
+ssz	 79
+nt_	 79
+gya	 79
+K	 79
+si_	 79
+sza	 78