RubyGems - scylla - Versions diffs - 0.9.3 → 1.0.0 - Mend

scylla 0.9.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (79) hide show

data/lib/scylla/lms/arabic.lm +399 -399
data/lib/scylla/lms/bulgarian.lm +400 -400
data/lib/scylla/lms/catalan.lm +323 -323
data/lib/scylla/lms/chinese.lm +389 -389
data/lib/scylla/lms/czech.lm +377 -377
data/lib/scylla/lms/danish.lm +383 -383
data/lib/scylla/lms/dutch.lm +398 -398
data/lib/scylla/lms/english.lm +355 -355
data/lib/scylla/lms/finnish.lm +381 -381
data/lib/scylla/lms/french.lm +379 -379
data/lib/scylla/lms/german.lm +382 -382
data/lib/scylla/lms/greek.lm +400 -400
data/lib/scylla/lms/hebrew.lm +400 -400
data/lib/scylla/lms/hindi.lm +400 -400
data/lib/scylla/lms/icelandic.lm +219 -219
data/lib/scylla/lms/indonesian.lm +364 -364
data/lib/scylla/lms/italian.lm +381 -381
data/lib/scylla/lms/japanese.lm +400 -400
data/lib/scylla/lms/kannada.lm +392 -392
data/lib/scylla/lms/korean.lm +389 -389
data/lib/scylla/lms/marathi.lm +364 -364
data/lib/scylla/lms/norwegian.lm +325 -325
data/lib/scylla/lms/persian.lm +397 -397
data/lib/scylla/lms/polish.lm +380 -380
data/lib/scylla/lms/portuguese.lm +375 -375
data/lib/scylla/lms/romanian.lm +318 -318
data/lib/scylla/lms/russian.lm +398 -398
data/lib/scylla/lms/slovak.lm +358 -358
data/lib/scylla/lms/slovenian.lm +256 -256
data/lib/scylla/lms/spanish.lm +353 -353
data/lib/scylla/lms/swedish.lm +400 -400
data/lib/scylla/lms/tagalog.lm +245 -245
data/lib/scylla/lms/thai.lm +400 -400
data/lib/scylla/lms/turkish.lm +379 -379
data/lib/scylla/lms/vietnamese.lm +373 -373
data/lib/scylla/lms/welsh.lm +293 -293
data/test/classifier_test.rb +5 -3
data/test/fixtures/lms/arabic.lm +400 -0
data/test/fixtures/lms/bulgarian.lm +400 -0
data/test/fixtures/lms/catalan.lm +400 -0
data/test/fixtures/lms/chinese.lm +400 -0
data/test/fixtures/lms/czech.lm +400 -0
data/test/fixtures/lms/danish.lm +399 -399
data/test/fixtures/lms/dutch.lm +400 -0
data/test/fixtures/lms/english.lm +400 -400
data/test/fixtures/lms/finnish.lm +400 -0
data/test/fixtures/lms/french.lm +397 -397
data/test/fixtures/lms/german.lm +400 -400
data/test/fixtures/lms/greek.lm +400 -0
data/test/fixtures/lms/hebrew.lm +400 -0
data/test/fixtures/lms/hindi.lm +400 -400
data/test/fixtures/lms/icelandic.lm +400 -0
data/test/fixtures/lms/indonesian.lm +400 -0
data/test/fixtures/lms/italian.lm +400 -400
data/test/fixtures/lms/japanese.lm +400 -400
data/test/fixtures/lms/kannada.lm +400 -0
data/test/fixtures/lms/korean.lm +400 -0
data/test/fixtures/lms/marathi.lm +400 -0
data/test/fixtures/lms/norwegian.lm +399 -399
data/test/fixtures/lms/persian.lm +400 -0
data/test/fixtures/lms/polish.lm +400 -0
data/test/fixtures/lms/portuguese.lm +400 -0
data/test/fixtures/lms/romanian.lm +400 -0
data/test/fixtures/lms/russian.lm +400 -0
data/test/fixtures/lms/slovak.lm +400 -0
data/test/fixtures/lms/slovenian.lm +400 -0
data/test/fixtures/lms/spanish.lm +400 -400
data/test/fixtures/lms/swedish.lm +400 -0
data/test/fixtures/lms/tagalog.lm +400 -0
data/test/fixtures/lms/thai.lm +400 -0
data/test/fixtures/lms/turkish.lm +400 -0
data/test/fixtures/lms/vietnamese.lm +400 -0
data/test/fixtures/lms/welsh.lm +400 -0
data/test/fixtures/test_languages/japanese +149 -67
data/test/generator_test.rb +1 -43
data/test/language_test.rb +5 -1
data/test/loader_test.rb +1 -1
data/test/scylla_test.rb +4 -4
metadata +105 -63

data/test/fixtures/lms/dutch.lm ADDED Viewed

@@ -0,0 +1,400 @@
+_	22244
+e	11500
+n	6122
+d	4278
+a	4090
+r	3921
+i	3793
+t	3567
+o	3449
+n_	3082
+e_	2627
+en	2502
+de	2434
+l	2396
+s	2267
+en_	1947
+er	1753
+_d	1731
+g	1730
+v	1472
+an	1401
+t_	1376
+de_	1366
+h	1342
+_de	1235
+k	1146
+_e	1135
+m	1078
+_v	1000
+u	930
+te	925
+nd	915
+w	908
+in	903
+ee	877
+ge	825
+b	820
+r_	805
+j	755
+p	754
+et	750
+_h	748
+s_	743
+he	732
+c	725
+d_	709
+or	709
+el	677
+et_	651
+st	648
+ij	646
+aa	646
+oo	645
+_i	632
+_o	610
+z	601
+ie	600
+la	578
+an_	577
+_he	565
+va	558
+_en	533
+_w	526
+_b	502
+van	501
+der	495
+_g	490
+het	486
+_va	484
+_n	483
+and	481
+ed	476
+on	461
+me	447
+_in	446
+lan	443
+rd	440
+_m	433
+oor	426
+nde	422
+ch	421
+re	420
+al	418
+er_	412
+_a	410
+le	407
+ng	406
+_t	400
+_z	399
+be	397
+ar	393
+ve	373
+li	372
+ne	371
+we	370
+g_	369
+is	368
+_s	368
+in_	366
+den	361
+f	356
+ede	355
+ro	353
+ke	351
+ri	349
+_ee	336
+een	336
+vo	335
+_ge	323
+ti	323
+it	320
+_be	319
+at	317
+l_	297
+rl	296
+di	290
+_l	282
+nd_	278
+na	277
+ste	275
+te_	273
+es	266
+aar	265
+k_	264
+op	254
+erl	249
+ui	248
+ver	248
+ing	248
+ra	246
+_me	237
+_vo	237
+erd	234
+zi	233
+jk	231
+ijk	231
+_ne	230
+nt	226
+wa	225
+ei	222
+rla	222
+ie_	220
+ten	219
+ter	219
+or_	218
+ned	215
+_k	215
+gr	212
+da	211
+_p	210
+se	210
+rs	208
+gen	207
+_we	205
+oe	199
+ot	196
+sc	196
+eg	195
+_ve	195
+ond	194
+sch	193
+is_	191
+id	188
+ng_	187
+_zi	183
+ig	182
+ta	182
+ni	181
+eer	178
+_wa	176
+_on	176
+_r	174
+rde	174
+_op	173
+ze	170
+bi	169
+ur	168
+ol	168
+ord	168
+ers	167
+ere	167
+p_	167
+ev	167
+voo	167
+lij	165
+_is	164
+ds	164
+to	160
+do	160
+cht	159
+_di	159
+ht	159
+om	158
+aan	158
+ls	154
+_te	152
+ma	152
+el_	152
+pe	152
+wo	150
+_la	149
+est	148
+ns	145
+wer	144
+rt	144
+uit	143
+ls_	142
+doo	142
+_al	142
+ek	141
+am	141
+ijn	140
+jn	140
+ts	140
+eu	139
+lo	138
+_da	137
+mi	137
+_do	135
+met	135
+gro	135
+nge	134
+rij	134
+eli	132
+no	132
+_gr	132
+zij	131
+op_	131
+ren	130
+ken	130
+se_	129
+ic	127
+zo	127
+og	127
+jn_	126
+nds	126
+m_	126
+als	126
+end	125
+a_	125
+rd_	123
+_na	123
+ar_	122
+ha	122
+die	122
+tr	122
+tie	121
+eel	121
+as	121
+men	120
+_st	120
+em	120
+ang	117
+_aa	116
+le_	116
+ke_	115
+it_	115
+_u	110
+ho	109
+ld	109
+_oo	108
+mee	108
+vi	107
+eid	107
+wor	106
+_to	106
+ind	106
+tu	105
+eb	105
+ege	105
+wi	105
+al_	105
+_wo	104
+f_	101
+ele	100
+_zo	100
+sta	99
+ec	98
+_ui	98
+ok	95
+rm	95
+st_	94
+jke	94
+ll	93
+j_	93
+rk	93
+ich	93
+_ma	92
+ij_	92
+ko	91
+kt	90
+at_	90
+jk_	90
+ak	90
+nk	90
+waa	89
+_c	89
+ot_	88
+pa	88
+ge_	87
+ag	86
+uw	86
+il	86
+_f	86
+len	85
+erk	85
+bij	84
+geb	84
+eld	84
+re_	83
+del	83
+_er	82
+eri	82
+ov	82
+af	82
+pr	81
+ez	81
+ela	81
+gi	80
+aat	80
+hi	79
+ch_	79
+h_	79
+ati	79
+po	78
+gel	78
+id_	77
+ac	77
+_bi	77
+_re	76
+ent	76
+ens	76
+bel	75
+_mi	75
+ok_	75
+rg	75
+rs_	75
+dt	75
+vol	74
+ale	74
+dt_	74
+lle	74
+si	73
+isc	73
+ook	73
+uu	72
+_no	72
+uur	71
+nte	71
+ite	71
+tot	71
+taa	71
+du	70
+ep	70
+jd	70
+ijd	70
+_ze	69
+_j	69
+io	69
+un	69
+_ko	68
+ad	68
+eke	68
+ied	68
+che	68
+ran	68
+od	68
+ru	67
+ort	67
+dee	67
+maa	67
+so	67
+per	66
+ist	66
+ige	66
+tw	66
+ote	64
+as_	64
+tij	64
+ier	64
+ove	63
+sp	63
+gev	63
+ed_	63
+ew	63
+ht_	62
+of	62
+tuu	62
+euw	62
+zic	62
+roo	62
+ame	62
+_ho	61
+kel	61
+rb	61
+ach	61
+orm	61
+rot	61
+ja	60
+rdt	60

data/test/fixtures/lms/english.lm CHANGED Viewed

@@ -1,400 +1,400 @@
-_	2598
-e	779
-i	547
-t	538
-a	512
-o	496
-s	459
-n	427
-r	397
-u	339
-l	290
-m	241
-e_	229
-d	228
-c	213
-h	192
-p	184
-s_	171
-t_	150
-_t	144
-re	135
-_a	132
-in	128
-m_	119
-_i	114
-f	104
-or	103
-b	102
-th	97
-n_	93
-g	90
-d_	90
-_o	88
-_e	84
-er	83
-_th	83
-he	81
-y	80
-at	79
-an	79
-_s	78
-on	74
-te	74
-es	72
-_p	70
-r_	69
-is	69
-se	66
-o_	66
-v	65
-it	64
-w	63
-the	63
-um	63
-en	62
-_the	61
-_c	59
-ti	59
-_b	58
-nd	58
-_d	58
-_l	57
-a_	53
-um_	52
-ur	52
-as	52
-y_	52
-lo	51
-nt	51
-su	50
-re_	49
-qu	49
-q	49
-he_	48
-_w	48
-us	46
-the_	46
-le	46
-em	45
-f_	45
-of	44
-si	43
-_m	43
-ta	43
-ore	42
-k	42
-_of	42
-_in	42
-ed	42
-to	42
-li	42
-et	41
-in_	41
-_r	41
-lor	40
-ng	40
-of_	39
-is_	39
-ea	39
-_h	39
-and	39
-_of_	39
-st	38
-la	38
-ed_	38
-_f	38
-ol	38
-de	37
-es_	36
-_an	36
-io	35
-ri	35
-_n	35
-ar	34
-ve	34
-co	33
-ing	33
-ra	33
-ou	33
-ce	33
-g_	32
-ec	32
-ng_	32
-l_	32
-nd_	32
-al	32
-em_	32
-ro	32
-to_	31
-ut	31
-ip	31
-ing_	31
-_to	31
-ha	31
-no	31
-ne	30
-ni	30
-el	30
-_u	30
-ci	29
-pa	29
-di	29
-lore	29
-it_	29
-om	29
-_v	29
-ic	28
-_re	28
-ma	28
-ns	28
-and_	28
-ct	28
-am	28
-ai	28
-on_	27
-_to_	27
-_and	27
-rem	27
-ion	27
-do	27
-ll	27
-_lo	27
-_a_	26
-_qu	26
-_co	26
-_q	26
-sum	26
-ia	26
-sum_	25
-_pa	25
-ips	25
-il	25
-ru	25
-ps	25
-tio	25
-et_	25
-x	25
-pl	24
-ati	24
-me	24
-_in_	24
-ui	24
-un	24
-rem_	23
-tr	23
-hi	23
-or_	23
-_ips	23
-h_	23
-_ip	23
-ut_	23
-ex	23
-psum	23
-i_	23
-at_	23
-se_	23
-ss	23
-ipsu	23
-tion	23
-psu	23
-orem	23
-_do	23
-ac	22
-id	22
-us_	22
-im	22
-mo	22
-ure	22
-nc	21
-nt_	21
-_ma	21
-ur_	21
-ons	20
-_lor	20
-qui	20
-ge	20
-_si	20
-_no	20
-tu	20
-pt	19
-er_	19
-bu	19
-ain	18
-pe	18
-ho	18
-ect	18
-_et	18
-ca	18
-_de	18
-ab	18
-as_	18
-be	18
-ple	18
-du	17
-_it	17
-_is	17
-rum_	17
-_wh	17
-rs	17
-_be	17
-sa	17
-_qui	17
-up	17
-eas	17
-vo	17
-ch	17
-wh	17
-pr	17
-am_	17
-rum	17
-ure_	17
-bl	17
-_ex	17
-ag	16
-ep	16
-tur	16
-na	16
-ua	16
-po	16
-bo	16
-ere	16
-lu	16
-ul	16
-_et_	16
-con	16
-cu	16
-olo	16
-ate	16
-_con	16
-lea	16
-ent	16
-au	16
-_dol	15
-leas	15
-ter	15
-_pl	15
-olor	15
-os	15
-dolo	15
-_is_	15
-dol	15
-tat	15
-ver	15
-_se	15
-en_	14
-_la	14
-te_	14
-so	14
-_ple	14
-_g	14
-int	14
-qua	14
-_ha	14
-wi	14
-sur	14
-plea	14
-ke	14
-ue	14
-_it_	13
-vol	13
-_by	13
-ain_	13
-ir	13
-_ve	13
-sure	13
-by	13
-res	13
-_al	13
-her	13
-fi	13
-olu	13
-le_	13
-_te	13
-ion_	13
-atio	13
-od	13
-_mo	13
-upt	13
-by_	13
-_by_	13
-olup	12
-_pr	12
-sin	12
-pta	12
-unt	12
-lup	12
-fr	12
-_vo	12
-lupt	12
-asu	12
-da	12
-est	12
-easu	12
-cc	12
-nte	12
-age	12
-upta	12
-_vol	12
-volu	12
-_li	12
-ore_	12
-ly	12
-_wi	12
-mp	12
-ae	12
-ly_	12
-_fr	12
-asur	12
-oo	12
-tur_	11
-nce	11
-k_	11
-pain	11
-fo	11
-_on	11
-oru	11
-ib	11
-oi	11
-pai	11
-_us	11
-equ	11
-_y	11
-eq	11
-_pai	11
-yo	11
-here	11
-tem	11
-orum	11
-cons	11
-tin	11
-ini	10
-use	10
-_es	10
-st_	10
-lat	10
-pu	10
-atu	10
-ll_	10
-ass	10
-ome	10
-ns_	10
-ess	10
-que	10
-rat	10
-_tr	10
-ill	10
-ble	10
-rom	10
-_he	10
-_or	10
-res_	10
-ere_	10
-rd	10
-ev	10
-ce_	10
-_su	10
-_so	10
-ibu	10
-om_	10
-for	10
+_	19026
+e	5658
+t	4138
+n	3839
+a	3693
+i	3421
+o	3229
+s	2963
+r	2878
+h	2469
+e_	2091
+d	1994
+l	1971
+_t	1697
+th	1590
+c	1418
+s_	1392
+he	1376
+_th	1342
+_a	1278
+d_	1264
+the	1260
+u	1163
+he_	1078
+g	1067
+in	1053
+m	1032
+f	978
+an	974
+n_	894
+p	830
+_i	809
+nd	776
+w	738
+re	710
+_o	653
+nd_	645
+y	643
+er	640
+t_	638
+en	628
+and	620
+on	605
+b	605
+_s	601
+ng	589
+y_	535
+_w	519
+ar	518
+es	508
+ed	502
+is	496
+_an	482
+_in	480
+_c	478
+ed_	475
+_e	464
+or	459
+ti	449
+st	443
+it	440
+of	429
+la	425
+f_	425
+_of	422
+h_	421
+te	418
+as	416
+_b	410
+of_	409
+nt	407
+v	398
+r_	392
+at	391
+ri	382
+_f	375
+al	368
+in_	368
+_p	358
+ro	326
+ing	310
+as_	307
+re_	301
+to	296
+_m	288
+co	287
+de	283
+o_	282
+g_	282
+ng_	277
+io	267
+l_	266
+es_	263
+le	263
+om	262
+ou	258
+li	255
+_l	253
+se	251
+_r	251
+ve	248
+me	244
+_h	244
+ea	242
+_en	242
+ch	242
+on_	241
+ur	234
+ion	230
+is_	229
+a_	229
+ce	223
+gl	221
+ic	221
+lan	220
+_co	217
+ent	217
+er_	217
+ngl	217
+eng	215
+_to	213
+_d	207
+to_	206
+hi	203
+st_	202
+_ar	199
+ha	198
+ra	197
+si	197
+k	195
+_is	193
+un	193
+al_	183
+nc	181
+m_	180
+_n	174
+_a_	174
+ns	174
+fo	173
+tr	169
+wa	162
+_re	162
+pe	161
+ma	159
+ll	159
+_fo	158
+ne	155
+th_	154
+ly	154
+her	153
+rt	153
+ly_	152
+po	151
+are	151
+el	151
+tio	150
+be	150
+na	150
+ca	147
+gla	147
+ec	147
+ge	144
+_as	144
+ho	144
+sh	144
+ol	144
+mo	143
+ere	143
+no	142
+ie	141
+_wa	140
+ni	140
+_u	137
+ch_	136
+lo	134
+ta	133
+we	132
+est	131
+il	129
+_be	128
+ts	126
+ted	125
+om_	125
+wh	124
+pr	124
+tu	123
+di	122
+ot	122
+pa	122
+ct	121
+ate	120
+os	119
+ati	119
+ts_	119
+op	119
+rs	119
+rom	118
+_wh	117
+wi	117
+fr	116
+_ha	115
+ai	115
+an_	115
+for	115
+or_	114
+_fr	113
+am	113
+ter	112
+us	111
+ia	111
+nt_	110
+su	110
+ry	110
+ish	109
+ow	109
+_mo	107
+_pr	107
+_by	105
+by_	105
+_s_	105
+by	105
+iv	104
+_wi	104
+ry_	104
+con	102
+so	102
+_la	101
+em	100
+le_	100
+_g	100
+was	99
+ut	99
+ad	99
+et	98
+_it	97
+cl	97
+ns_	96
+ul	96
+me_	95
+mi	95
+wo	95
+ac	95
+ee	95
+inc	94
+ig	94
+do	94
+ons	94
+oun	94
+ir	93
+sh_	93
+ci	93
+ver	93
+tur	92
+uc	92
+fro	92
+_ma	91
+ce_	90
+ies	90
+_pa	89
+ith	87
+_on	87
+du	86
+_we	86
+wit	85
+lis	85
+tin	84
+rl	84
+iti	84
+nce	84
+at_	84
+_de	84
+mp	83
+en_	83
+ev	82
+id	82
+ld	82
+all	82
+lu	82
+_k	82
+_su	80
+pl	80
+fi	79
+sp	78
+_se	78
+par	77
+_no	77
+rs_	76
+im	76
+ov	76
+rea	76
+te_	76
+gh	75
+vi	75
+_un	75
+ss	75
+man	74
+x	74
+art	74
+ite	73
+gi	72
+rit	71
+ain	71
+od	71
+it_	71
+ag	71
+und	70
+whi	70
+br	70
+ve_	69
+res	69
+ntr	68
+rn	68
+ist	68
+gli	67
+_wo	65
+_st	65
+cen	65
+_so	64
+der	64
+ive	64
+ba	63
+int	63
+lar	63
+_si	63
+ost	63
+_po	63
+eg	62
+ki	62
+ud	62
+ty	62
+ove	62
+men	62
+nti	62
+ers	61
+_ce	61
+cou	61
+pro	60
+ld_	60
+rin	60
+has	60
+_br	60
+rm	60
+rd	59
+ame	59
+ern	59
+_al	59
+ort	59
+lat	58
+gr	58
+_pe	58
+bo	58
+ug	58
+ine	58
+ica	57
+wor	57
+rou	57
+ect	56
+uri	56
+kin	56
+clu	56
+sc	56
+oth	56
+ue	56
+av	55
+_le	55
+_lo	55
+uch	55
+eve	54
+se_	54
+bri	54
+cal	54
+com	53
+rie	53
+uni	53
+ex	53
+wer	53
+his	53
+ear	53
+eat	53
+cr	52
+c_	52
+ori	52
+_at	51
+ona	51
+nal	51
+tra	51
+sed	51
+ls	51
+rc	51
+ncl	51
+ide	51
+rg	50
+hic	50
+ga	50
+lud	50
+unt	50
+bl	50
+suc	50
+_ki	50
+de_	49