language_detector 0.1.0

data/lib/textcat_ngrams/persian.lm

@@ -0,0 +1,400 @@
+_	 12318
+?	 5938
+�	 2613
+?_	 1815
+�	 1745
+�	 1569
+�	 1546
+�	 1422
+-	 1333
+�	 1155
+�	 991
+�	 841
+_�	 778
+�	 761
+�	 693
+�	 664
+_�	 663
+�	 660
+�?	 631
+�_	 624
+_?	 601
+?�	 598
+�	 589
+_�	 565
+??	 564
+�	 513
+�_	 486
+_�	 473
+_�	 469
+�?	 459
+�	 428
+�_	 427
+_�	 398
+�_	 392
+�	 390
+�	 384
+�?	 379
+�?	 361
+_�	 337
+_-	 334
+>	 327
+�	 324
+_�?	 319
+�?	 313
+_�_	 308
+�	 305
+��	 303
+�_	 297
+?�	 290
+�?	 285
+��	 272
+-_	 271
+S	 266
+_�?	 265
+�	 259
+-�	 255
+�	 255
+??_	 255
+�?	 254
+�?	 250
+Y	 246
+��_	 245
+�	 243
+_�?	 238
+?�	 234
+_�	 229
+�	 229
+_S	 228
+��	 216
+��	 215
+?�	 215
+?�	 214
+_��	 214
+�	 208
+�?_	 207
+��	 203
+?-	 202
+_�	 199
+�?_	 199
+��	 195
+�_	 190
+�?_	 190
+_??	 189
+�?	 188
+.	 188
+_��_	 185
+_�	 183
+?�	 181
+_��	 180
+�?	 177
+*	 176
+�_	 176
+_�?_	 174
+?�	 173
+��	 172
+�_	 170
+��	 164
+�	 161
+�	 158
+-�?	 156
+�_	 155
+�?_	 155
+_�?	 152
+��	 151
+��	 150
+_�?	 146
+�?_	 144
+��	 143
+��	 141
+?�_	 140
+*_	 138
+��	 137
+?�	 136
+�	 133
+��	 132
+_��	 131
+-�	 131
+�?	 130
+?�	 129
+_.	 129
+��	 129
+�_	 128
+��	 124
+��	 121
+�-	 120
+��_	 120
+_��	 120
+_??_	 119
+�?	 119
+-�?_	 119
+�	 118
+-?	 118
+>_	 116
+��?	 114
+�_	 111
+��	 109
+_�?_	 109
+_��	 109
+_��_	 108
+_��?	 107
+�?�	 107
+�	 106
+_��	 106
+.�	 104
+?�	 103
+_?�	 103
+��	 103
+�?	 103
+��	 103
+�?	 102
+��	 100
+?�	 100
+?�?	 98
+�_	 98
+�?�	 96
+��	 95
+��?_	 95
+_�	 94
+�	 94
+Y_	 94
+_�?�	 93
+_�?�	 93
+��	 92
+��	 91
+_�?	 91
+_��?_	 91
+?�_	 90
+�?_	 90
+��	 90
+�?_	 90
+�-	 90
+��	 89
+:	 89
+��_	 88
+��?	 88
+��	 88
+_��	 87
+-�	 86
+��	 85
+_��	 83
+,	 82
+��_	 82
+��	 82
+?�_	 80
+�	 80
+?��	 79
+_��?	 79
+��	 77
+???	 77
+��	 76
+_-?	 75
+��	 75
+?�	 75
+?�	 75
+��	 74
+_��	 73
+�_	 73
+���	 73
+�?	 73
+���	 72
+����	 71
+��	 71
+?>	 71
+�?	 70
+_?�_	 70
+_.�	 70
+��	 69
+�	 69
+_�	 68
+���	 68
+����_	 67
+_�-	 67
+���	 67
+���_	 67
+��	 67
+��_	 67
+��	 67
+��_	 66
+?-_	 65
+��	 65
+�?�	 65
+���	 64
+��?	 64
+�_	 64
+?�	 63
+?-�	 63
+�-	 63
+_�?�	 62
+�?_	 62
+��_	 62
+_-�	 61
+��	 61
+�����	 60
+����	 60
+���	 60
+?�	 60
+���	 60
+_��	 60
+����	 60
+_���	 60
+-�?	 60
+�_	 60
+��	 60
+��?	 60
+��	 59
+�����	 59
+�?	 59
+_�?	 59
+����	 59
+��_	 59
+�??	 58
+�?�	 58
+_��	 58
+.-	 58
+?�	 58
+��	 57
+�	 57
+���	 57
+?�?_	 57
+�?�	 57
+�-	 57
+��_	 56
+�?�	 56
+��_	 56
+?��?	 56
+�?	 56
+�	 56
+�?�	 56
+��_	 55
+�?	 55
+�?�	 55
+_����	 55
+-�?�	 55
+�_	 55
+�?��	 55
+��_	 55
+S�	 55
+��	 54
+_S�	 53
+���_	 53
+_��	 53
+�	 53
+��	 53
+����_	 53
+��_	 52
+��?	 52
+��	 52
+��?	 52
+�>	 52
+-�	 52
+��?	 52
+��	 52
+�?	 52
+_�	 52
+�?	 52
+��	 52
+??�	 52
+��	 51
+��	 51
+-�	 51
+_��?	 51
+�?	 50
+_��	 50
+���_	 50
+.-�	 50
+�?�	 50
+�	 50
+_�?	 50
+��	 50
+_�?�	 49
+.��	 49
+��	 49
+�?�?	 49
+�-	 48
+_��	 48
+_�?��	 48
+��?_	 48
+�_	 48
+��	 48
+�?_	 48
+��_	 47
+�	 47
+�	 47
+.-�?	 47
+_��	 47
+>�	 47
+��	 47
+�?�	 47
+�	 47
+?��?_	 47
+?��	 47
+��	 46
+_.-	 46
+_�?�?	 46
+��	 46
+��	 46
+-�	 46
+_��	 46
+��	 46
+��	 45
+��	 45
+_��	 45
+�?-	 45
+��	 45
+_�	 45
+f	 45
+�-	 44
+_�-	 44
+��	 44
+_��?	 44
+�-�	 44
+-�	 44
+�	 44
+�?��?	 44
+��	 44
+.-�?_	 44
+��_	 44
+��	 44
+_-�?	 44
+�?�	 43
+�?	 43
+�	 43
+?�_	 43
+_����	 43
+��	 43
+?�	 43
+?�_	 43
+_���	 43
+�_	 43
+��	 43
+�??	 42
+Y?	 42
+�_	 42
+��	 42
+��	 42
+_��	 42
+��?	 42
+��	 42
+/	 42
+_�	 42
+???_	 42
+_���	 42
+_��	 42
+s	 42
+���	 42
+�?-�	 41
+��	 41
+��?	 41
+?*	 41
+S��	 41
+_�?�	 41
+��	 41
+_.-�	 41
+��	 41
+��	 41
+��	 41
+S��	 41
+��	 41
+�?�?_	 40
+���	 40

data/lib/textcat_ngrams/polish.lm

@@ -0,0 +1,400 @@
+_	 31480
+a	 7945
+i	 7766
+e	 7462
+o	 6838
+z	 5104
+n	 5077
+r	 4178
+w	 4139
+s	 3596
+c	 3580
+y	 3569
+t	 3381
+d	 3027
+k	 2819
+p	 2639
+m	 2494
+ie	 2484
+u	 2016
+l	 1947
+j	 1932
+ni	 1930
+e_	 1746
+_p	 1712
+�	 1649
+a_	 1496
+o_	 1431
+,	 1368
+,_	 1366
+b	 1296
+_w	 1257
+g	 1249
+i_	 1140
+.	 1118
+_s	 1045
+ze	 1041
+._	 1035
+_n	 997
+nie	 958
+cz	 956
+rz	 952
+h	 950
+_z	 933
+�	 918
+ow	 911
+ie_	 902
+y_	 899
+na	 885
+ch	 871
+po	 864
+pr	 863
+wi	 851
+st	 847
+�	 835
+�	 832
+an	 814
+�	 800
+ra	 778
+zy	 766
+ia	 726
+za	 718
+_t	 699
+wa	 692
+ro	 692
+_d	 683
+_pr	 675
+�	 661
+w_	 658
+sz	 652
+_po	 649
+_o	 628
+m_	 613
+li	 613
+dz	 611
+ki	 611
+en	 609
+mi	 593
+ta	 571
+ci	 564
+ej	 562
+nie_	 558
+_m	 553
+_k	 549
+ar	 543
+go	 541
+_i	 540
+em	 532
+od	 525
+yc	 520
+a�	 511
+rze	 508
+do	 504
+eg	 503
+ko	 502
+ac	 482
+to	 478
+_na	 467
+_ni	 467
+h_	 466
+ch_	 466
+�	 465
+i�	 461
+_b	 458
+on	 458
+u_	 458
+zi	 454
+ka	 450
+er	 448
+sk	 447
+si	 447
+wy	 444
+te	 437
+ak	 434
+�_	 431
+_j	 429
+je	 429
+z_	 427
+ny	 422
+aw	 422
+ne	 420
+�w	 418
+_w_	 415
+_c	 414
+ego	 412
+prz	 412
+_r	 410
+al	 407
+�a	 405
+"	 405
+re	 405
+es	 401
+_nie	 396
+dzi	 394
+ty	 389
+j_	 380
+ic	 380
+_prz	 379
+ad	 367
+ej_	 364
+le	 359
+�_	 358
+ed	 354
+ych	 346
+_za	 346
+_do	 344
+zn	 344
+go_	 344
+ani	 343
+_i_	 342
+no	 339
+or	 337
+�o	 336
+tr	 334
+P	 330
+os	 329
+am	 329
+da	 328
+ec	 327
+ol	 325
+�_	 325
+by	 322
+ego_	 321
+at	 321
+�e	 319
+ym	 317
+wie	 314
+�_	 312
+-	 310
+na_	 309
+_si	 303
+W	 301
+as	 301
+wo	 300
+pa	 299
+si�	 295
+nia	 293
+owa	 292
+o�	 286
+_P	 283
+el	 282
+_si�	 279
+ja	 278
+rzy	 276
+prze	 276
+_wy	 275
+i�_	 269
+in	 267
+_-	 267
+de	 267
+kie	 267
+dn	 264
+ob	 262
+_u	 261
+ych_	 260
+-_	 258
+ez	 257
+_-_	 256
+�c	 255
+ws	 255
+�ci	 254
+em_	 253
+_si�_	 253
+si�_	 253
+_nie_	 253
+kt	 252
+ski	 252
+we	 251
+_g	 251
+_W	 250
+t_	 249
+_prze	 249
+_je	 248
+aj	 247
+_a	 247
+�e_	 246
+_�	 244
+ia_	 243
+eni	 241
+om	 240
+la	 240
+k_	 235
+mo	 235
+f	 234
+pi	 232
+is	 231
+cze	 231
+_z_	 230
+�	 228
+nt	 227
+ce	 224
+sta	 221
+ry	 220
+ma	 219
+cj	 219
+zie	 218
+ek	 216
+oc	 213
+dy	 212
+owi	 208
+sp	 208
+K	 208
+t�	 205
+_"	 204
+ud	 203
+S	 202
+ier	 202
+pra	 202
+czn	 201
+ys	 200
+nia_	 199
+j�	 198
+_�e	 196
+oz	 194
+N	 192
+zo	 191
+dzie	 190
+ku	 190
+�w_	 189
+czy	 189
+_mi	 188
+_�e_	 187
+mie	 186
+do_	 186
+iej	 185
+ym_	 185
+_do_	 184
+cy	 184
+_pa	 183
+y�	 182
+_na_	 181
+zy_	 181
+ew	 180
+_l	 180
+_cz	 178
+_by	 178
+ru	 177
+to_	 175
+�c	 175
+_wi	 175
+ln	 174
+_K	 171
+ok	 170
+ot	 170
+raw	 169
+nych	 168
+nyc	 168
+az	 168
+ik	 167
+bi	 167
+i,_	 166
+i,	 166
+_te	 165
+tu	 163
+wan	 163
+et	 162
+J	 162
+�r	 161
+cie	 160
+_od	 160
+ba	 160
+oj	 159
+o�c	 158
+i�	 158
+_ro	 158
+og	 158
+mu	 158
+A	 158
+o�ci	 158
+d_	 157
+tw	 156
+io	 155
+gr	 155
+_J	 155
+_N	 155
+_to	 155
+us	 155
+�_	 154
+a�	 154
+ach	 154
+yd	 153
+_kt	 153
+_ko	 152
+_st	 152
+pow	 151
+yt	 150
+kr	 149
+_S	 149
+a�_	 149
+ak_	 149
+li_	 148
+T	 148
+ur	 148
+sa	 148
+op	 147
+yw	 147
+ent	 147
+bo	 146
+_sp	 145
+O	 145
+a,_	 144
+a,	 144
+iem	 144
+kt�	 143
+praw	 143
+wn	 142
+t�r	 142
+i�	 141
+�y	 141
+zu	 141
+dni	 140
+im	 139
+icz	 139
+r�	 139
+�e	 138
+kt�r	 137
+my	 137
+est	 136
+awi	 135
+przy	 134
+nych_	 134
+e,_	 132
+e,	 132
+ki_	 132
+pro	 131
+_kt�	 131
+_kt�r	 131
+_przy	 130
+_ja	 129
+szy	 127
+ia�	 127
+Po	 127
+ania	 127
+M	 126
+ze_	 125
+ne_	 125
+�s	 124
+_ty	 124
+sze	 124
+a�	 124
+nn	 124
+pod	 124
+a�_	 122
+B	 122
+za_	 122
+cza	 122
+�o_	 122
+Z	 122
+_to_	 121
+iu	 120
+zc	 119
+esz	 119
+skie	 118
+i.	 118
+_ka	 117
+so	 117
+trz	 117
+o�	 116
+ieg	 116
+nik	 116
+ga	 116