RubyGems - scylla - Versions diffs - 0.9.3 → 1.0.0 - Mend

scylla 0.9.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (79) hide show

data/lib/scylla/lms/arabic.lm +399 -399
data/lib/scylla/lms/bulgarian.lm +400 -400
data/lib/scylla/lms/catalan.lm +323 -323
data/lib/scylla/lms/chinese.lm +389 -389
data/lib/scylla/lms/czech.lm +377 -377
data/lib/scylla/lms/danish.lm +383 -383
data/lib/scylla/lms/dutch.lm +398 -398
data/lib/scylla/lms/english.lm +355 -355
data/lib/scylla/lms/finnish.lm +381 -381
data/lib/scylla/lms/french.lm +379 -379
data/lib/scylla/lms/german.lm +382 -382
data/lib/scylla/lms/greek.lm +400 -400
data/lib/scylla/lms/hebrew.lm +400 -400
data/lib/scylla/lms/hindi.lm +400 -400
data/lib/scylla/lms/icelandic.lm +219 -219
data/lib/scylla/lms/indonesian.lm +364 -364
data/lib/scylla/lms/italian.lm +381 -381
data/lib/scylla/lms/japanese.lm +400 -400
data/lib/scylla/lms/kannada.lm +392 -392
data/lib/scylla/lms/korean.lm +389 -389
data/lib/scylla/lms/marathi.lm +364 -364
data/lib/scylla/lms/norwegian.lm +325 -325
data/lib/scylla/lms/persian.lm +397 -397
data/lib/scylla/lms/polish.lm +380 -380
data/lib/scylla/lms/portuguese.lm +375 -375
data/lib/scylla/lms/romanian.lm +318 -318
data/lib/scylla/lms/russian.lm +398 -398
data/lib/scylla/lms/slovak.lm +358 -358
data/lib/scylla/lms/slovenian.lm +256 -256
data/lib/scylla/lms/spanish.lm +353 -353
data/lib/scylla/lms/swedish.lm +400 -400
data/lib/scylla/lms/tagalog.lm +245 -245
data/lib/scylla/lms/thai.lm +400 -400
data/lib/scylla/lms/turkish.lm +379 -379
data/lib/scylla/lms/vietnamese.lm +373 -373
data/lib/scylla/lms/welsh.lm +293 -293
data/test/classifier_test.rb +5 -3
data/test/fixtures/lms/arabic.lm +400 -0
data/test/fixtures/lms/bulgarian.lm +400 -0
data/test/fixtures/lms/catalan.lm +400 -0
data/test/fixtures/lms/chinese.lm +400 -0
data/test/fixtures/lms/czech.lm +400 -0
data/test/fixtures/lms/danish.lm +399 -399
data/test/fixtures/lms/dutch.lm +400 -0
data/test/fixtures/lms/english.lm +400 -400
data/test/fixtures/lms/finnish.lm +400 -0
data/test/fixtures/lms/french.lm +397 -397
data/test/fixtures/lms/german.lm +400 -400
data/test/fixtures/lms/greek.lm +400 -0
data/test/fixtures/lms/hebrew.lm +400 -0
data/test/fixtures/lms/hindi.lm +400 -400
data/test/fixtures/lms/icelandic.lm +400 -0
data/test/fixtures/lms/indonesian.lm +400 -0
data/test/fixtures/lms/italian.lm +400 -400
data/test/fixtures/lms/japanese.lm +400 -400
data/test/fixtures/lms/kannada.lm +400 -0
data/test/fixtures/lms/korean.lm +400 -0
data/test/fixtures/lms/marathi.lm +400 -0
data/test/fixtures/lms/norwegian.lm +399 -399
data/test/fixtures/lms/persian.lm +400 -0
data/test/fixtures/lms/polish.lm +400 -0
data/test/fixtures/lms/portuguese.lm +400 -0
data/test/fixtures/lms/romanian.lm +400 -0
data/test/fixtures/lms/russian.lm +400 -0
data/test/fixtures/lms/slovak.lm +400 -0
data/test/fixtures/lms/slovenian.lm +400 -0
data/test/fixtures/lms/spanish.lm +400 -400
data/test/fixtures/lms/swedish.lm +400 -0
data/test/fixtures/lms/tagalog.lm +400 -0
data/test/fixtures/lms/thai.lm +400 -0
data/test/fixtures/lms/turkish.lm +400 -0
data/test/fixtures/lms/vietnamese.lm +400 -0
data/test/fixtures/lms/welsh.lm +400 -0
data/test/fixtures/test_languages/japanese +149 -67
data/test/generator_test.rb +1 -43
data/test/language_test.rb +5 -1
data/test/loader_test.rb +1 -1
data/test/scylla_test.rb +4 -4
metadata +105 -63

data/test/fixtures/lms/finnish.lm ADDED Viewed

@@ -0,0 +1,400 @@
+_	17794
+a	8212
+i	7011
+t	5906
+s	5775
+n	5574
+e	5070
+u	4250
+l	4246
+o	3954
+k	3187
+n_	2826
+a_	2652
+m	2465
+ä	2237
+v	1765
+r	1465
+en	1456
+j	1420
+is	1368
+ta	1233
+in	1154
+_s	1143
+st	1103
+y	1051
+en_	1004
+p	998
+h	956
+al	934
+_k	907
+i_	880
+an	874
+si	871
+li	849
+_j	847
+uo	829
+ll	794
+sa	773
+ja	766
+se	753
+aa	748
+va	745
+_m	730
+_v	708
+ä_	704
+t_	702
+tt	690
+ma	686
+it	683
+ai	682
+la	675
+su	672
+_o	666
+ti	647
+on	641
+ka	640
+te	640
+ja_	598
+ta_	590
+in_	590
+tu	575
+_t	575
+_l	560
+el	544
+_su	536
+_ja	535
+d	533
+om	531
+es	530
+oi	521
+at	502
+_a	494
+me	489
+et	480
+ss	474
+as	473
+ist	470
+ku	469
+uu	464
+ko	460
+mi	454
+lu	451
+suo	440
+an_	440
+_p	438
+uom	426
+nt	421
+tä	418
+un	418
+il	418
+ii	409
+on_	408
+us	402
+sta	402
+ik	399
+ut	397
+ki	395
+sa_	392
+_e	390
+ne	390
+to	389
+ks	386
+ke	374
+os	370
+ol	370
+le	369
+ssa	365
+nn	361
+ei	352
+na	346
+ee	345
+ri	339
+jo	323
+ul	320
+_va	315
+ise	314
+ö	314
+er	313
+vu	303
+_n	298
+_on	295
+ty	295
+im	292
+lli	289
+ia	288
+de	285
+lis	283
+s_	274
+ome	271
+at_	271
+ää	264
+ak	263
+_h	263
+lä	261
+mu	257
+ot	254
+la_	249
+lla	248
+aan	246
+sä	245
+ie	241
+_ka	238
+e_	236
+sen	235
+vi	235
+all	234
+ais	230
+_al	226
+ni	225
+ur	224
+_ma	221
+maa	220
+uv	220
+ksi	217
+lt	215
+_ku	214
+si_	213
+est	212
+_jo	209
+tä_	208
+pi	203
+än	202
+vä	201
+sk	201
+vat	201
+ra	199
+_mu	198
+uk	198
+_r	196
+tta	196
+ns	195
+een	195
+pa	194
+ok	193
+ht	191
+men	189
+ui	189
+ar	188
+ia_	185
+itt	184
+val	183
+lo	182
+au	181
+ti_	179
+aa_	179
+_y	178
+mm	177
+kk	176
+ha	175
+no	175
+tti	175
+isi	173
+ala	171
+ill	171
+iin	170
+suu	168
+o_	164
+am	163
+kä	163
+lai	162
+_ko	161
+_sa	160
+na_	160
+av	159
+den	159
+oli	158
+mä	156
+g	156
+pu	156
+nen	154
+vuo	153
+ine	152
+_ta	150
+iv	150
+et_	149
+sti	148
+ell	148
+ts	147
+nk	147
+_lu	146
+_tu	145
+aj	145
+so	145
+_vu	144
+ast	144
+lk	144
+he	143
+eu	142
+ess	142
+_se	141
+oma	140
+kan	137
+ste	137
+rk	137
+ro	136
+un_	136
+io	135
+ve	134
+_ol	134
+lle	134
+nna	132
+jä	132
+_ke	132
+äl	131
+eis	130
+stä	129
+ois	129
+ät	129
+alu	128
+yö	127
+ek	127
+ain	126
+ten	126
+ou	126
+lm	126
+u_	125
+ett	124
+vo	124
+or	123
+utt	123
+ru	122
+ir	122
+taa	122
+ent	122
+äi	121
+kun	121
+mis	119
+ue	118
+_ 	118
+ 	118
+tel	117
+_i	117
+op	117
+em	117
+yt	116
+sin	115
+kse	115
+le_	115
+oh	114
+än_	114
+od	114
+_me	112
+ust	111
+oit	111
+li_	111
+aik	111
+_u	110
+lue	110
+nta	110
+uot	110
+po	109
+lin	107
+mal	106
+id	106
+ov	106
+nä	105
+hi	105
+_si	105
+yh	105
+alt	105
+sto	105
+ses	105
+_ha	104
+lii	104
+ali	104
+luv	104
+ien	103
+oll	103
+yk	102
+_la	102
+je	102
+mat	101
+lta	101
+iss	101
+är	101
+ans	100
+omi	100
+ka_	99
+int	98
+ud	98
+lä_	97
+kuu	97
+ttä	96
+ap	96
+pe	96
+sä_	96
+ita	96
+f	96
+uvu	95
+toi	95
+unn	95
+uks	95
+äm	94
+ttu	93
+uur	93
+ost	93
+kaa	92
+nsa	92
+tii	91
+eh	91
+tai	91
+ova	90
+_to	90
+tet	90
+ass	90
+mes	90
+_ _	89
+ _	89
+ika	89
+_li	89
+min	88
+ava	88
+my	88
+joi	88
+utu	87
+tei	87
+stu	87
+kes	86
+uri	86
+_ov	85
+ään	85
+_vä	85
+re	84
+yl	84
+oin	84
+eli	83
+pä	83
+mi_	83
+uon	83
+sii	83
+uus	83
+ös	82
+mp	82
+ull	81
+äh	81
+ij	81
+ide	81
+tal	81
+llä	81
+uva	80
+äk	80
+osi	80
+_my	80
+nu	80
+imm	80
+ude	79
+tav	79
+saa	78
+sia	78
+_po	78
+us_	78
+ikk	78
+ih	78
+kau	78
+myö	77
+ssä	77
+voi	77
+ama	77
+onn	77
+ys	77

data/test/fixtures/lms/french.lm CHANGED Viewed

@@ -1,400 +1,400 @@
-_	31950
-e	11946
-s	6507
-a	6436
-i	6268
-n	6183
-t	5689
-r	5610
-l	5016
-e_	4933
-u	4308
-o	4232
-s_	3398
-d	3279
-�	3243
-c	2751
-p	2562
-_d	2485
-m	2301
-_l	2197
-es	2147
-é	2115
-�	2115
-t_	1917
-es_	1693
-le	1640
-de	1633
-en	1546
-_e	1437
-on	1430
-_p	1417
-_de	1396
-an	1276
-nt	1259
-n_	1199
-re	1180
-f	1101
-ti	1029
-la	1012
-de_	1012
-is	984
-_s	954
-_c	938
-g	928
-_a	913
-ra	909
-_de_	905
-a_	887
-v	842
-le_	833
-_le	798
-_la	790
-_f	747
-la_	745
-ai	743
-_la_	742
-te	729
-ar	723
-ent	715
-nt_	714
-q	699
-�	698
-b	695
-qu	688
-�	688
-�	687
-_m	687
-in	678
-se	674
-et	663
-ce	663
-me	660
-it	659
-��	652
-�	652
-’	652
-ur	643
-ne	634
-er	619
-ue	613
-r_	608
-u_	605
-io	601
-ns	598
-pa	592
-et_	581
-ion	573
-_�	571
-co	554
-les	550
-st	550
-au	548
-ie	545
-_r	541
-_et	537
-ri	537
-_et_	534
-les_	530
-at	530
-li	528
-ran	527
-re_	524
-nc	520
-tr	513
-ou	512
-al	500
-ro	492
-r�	484
-h	483
-fr	475
-on_	475
-ta	472
-_pa	468
-_en	467
-eu	461
-_fr	453
-que	453
-fra	434
-tio	431
-tion	429
-_fra	426
-x	425
-em	425
-ce_	422
-fran	422
-_le_	420
-en_	419
-_t	417
-po	415
-un	415
-or	414
-pr	408
-si	407
-ne_	406
-rt	401
-ent_	394
-ns_	393
-par	390
-ll	386
-_co	383
-des	381
-oi	378
-ré	378
-anc	375
-des_	367
-nce	366
-ue_	363
-is_	359
-�	359
-è	359
-ma	358
-ir	355
-l�	354
-_l�	354
-l�	354
-_l�	354
-l’	354
-_des	351
-_en_	349
-du	345
-_les	342
-ion_	340
-el	339
-_par	336
-é_	333
-�_	333
-i_	331
-us	326
-_n	326
-ui	325
-n�	322
-ati	321
-nd	320
-nce_	320
-ve	316
-que_	315
-so	314
-t�	313
-_u	312
-ance	311
-om	310
-iq	296
-iqu	296
-il	295
-�e	294
-_pr	294
-ée	294
-mi	293
-men	291
-_un	291
-_au	289
-__	288
-�_	288
-est	286
-na	286
-ique	286
+_	28586
+e	10327
+s	5870
+a	5422
+n	5335
+t	5043
+i	4889
+r	4631
+l	4323
+e_	3992
+u	3708
+o	3568
+s_	3244
+d	2939
+p	2738
+_d	2283
+c	2226
+_l	2164
+es	2030
+m	1956
+t_	1929
+é	1766
+es_	1606
+de	1536
+_p	1456
+le	1453
+en	1414
+_e	1403
+_de	1317
+on	1231
+nt	1188
+an	1085
+n_	1067
+re	986
+la	933
+f	927
+de_	914
+_s	853
+a_	848
+_le	827
+ti	825
+_c	818
+_a	806
+is	800
+_la	764
+la_	741
+le_	731
+ra	729
+nt_	726
+g	724
+’	679
+v	679
+ent	667
+et	650
+_f	637
+ar	615
+ce	600
+u_	597
+pa	590
+te	587
+q	582
+et_	575
+r_	569
+it	569
+qu	568
+me	567
+_m	566
+ai	556
+in	553
+ns	549
+ur	545
+_et	544
+se	541
+ne	522
+les	497
+er	493
+st	490
+au	487
+ue	487
+b	479
+io	472
+_pa	471
+co	471
+_en	469
+ion	452
+nc	448
+tr	445
+ran	441
+ri	441
+at	437
+ou	431
+ie	430
+_r	424
+re_	417
+eu	405
+par	404
+on_	403
+des	400
+en_	399
+fr	390
+li	387
+ta	386
+al	386
+ns_	385
+po	385
+l’	375
+_l’	373
+_t	373
+em	373
+ce_	371
+pr	371
+ro	370
+un	369
+_fr	367
+h	362
+tio	356
+_co	356
+que	349
+fra	349
+or	347
+ne_	344
+so	337
+rt	333
+oi	328
+à_	326
+à	326
+du	318
+ré	317
+nce	315
+is_	315
+_à	313
+_à_	313
+anc	309
+x	308
+est	299
+si	299
+_au	298
+_n	298
+el	291
+us	290
+om	288
+_u	286
+p_	285
 pl	284
-di	283
-_�_	279
-té	278
-ci	277
-te_	275
-ranc	272
-_du	271
-ment	270
-ut	269
-ss	268
-és	263
-�s	263
-ais	262
-lu	262
-eur	261
-mo	261
-du_	261
-con	259
-atio	258
-_du_	257
-y	256
-no	256
-pe	255
-_so	253
-ol	252
-rs	252
-l_	251
-d�	251
-lle	250
-_i	248
-_qu	245
-_g	245
-_q	245
-ch	241
-dé	241
-res	241
-ont	240
-ic	240
-ire	240
-se_	239
-ons	238
-lo	236
-st_	234
-me_	233
-ant	233
-sa	233
-_po	233
-est_	231
-i�	231
-vi	229
-nn	226
-ni	223
-ct	223
-_o	219
-_es	218
-_é	216
-_d�	215
-ux	215
-au_	214
-x_	214
-art	213
-ur_	213
-eme	208
-to	208
-d�	207
-d�	207
-_dé	207
-d’	207
-emen	205
-_re	205
-_est	204
-pu	204
-_ma	203
-�t	202
-ét	202
-_d�	201
-_d�	201
-_v	201
-ec	200
-ale	197
-da	196
-_mo	196
-_se	196
-_pl	196
-tre	193
-op	191
-ul	189
-bl	188
-mp	187
-ux_	187
-_con	187
-ac	182
-_r�	182
-rs_	182
-tu	181
-ér	180
-�r	180
-rti	179
-ts	179
-ie_	178
-’a	177
-�a	177
-��a	177
-gr	176
-ise	174
-_b	173
-ans	173
-une	172
-us_	172
-_au_	172
-iè	172
-�	170
-ç	170
-as	170
-ia	170
-ar_	170
-_ré	170
-ts_	168
-son	168
-cl	167
-ont_	167
-an�	167
-ran�	166
-su	166
+ll	284
+ue_	284
+nd	282
+é_	281
+ma	281
+ui	280
+è	275
+_pr	269
+_un	268
+ée	264
+ir	263
+ati	262
+men	261
+lu	259
+_du	259
+_so	258
+ont	257
+du_	254
+st_	250
+ve	249
+mi	249
+rs	246
+_q	244
+_qu	244
+és	240
+ci	239
+i_	239
+té	236
+di	235
+ut	232
+_es	230
+mo	229
+ss	229
+_o	227
+ant	223
+_é	220
+pe	218
+eur	217
+ais	214
+au_	212
+con	211
+lle	210
+res	209
+_po	208
+l_	205
+ct	204
+il	201
+iqu	200
+iq	200
+na	199
+te_	198
+ep	198
+ux	197
+_pl	197
+d’	197
+son	197
+nn	196
+da	195
+y	195
+ol	195
+rs_	194
+x_	192
+_d’	192
+ar_	191
+ons	191
+lo	191
+_i	190
+me_	189
+_se	187
+to	187
+no	186
+ur_	185
+ch	182
+eme	180
+ec	178
+tre	178
+ux_	177
+_re	176
+us_	175
+vi	175
+ans	174
+_mo	172
+pp	172
+pu	172
+ic	172
+mp	171
+dé	170
+lus	170
+art	170
+_ma	167
+ire	166
 un_	166
-im	166
-lus	166
-nç	165
-té_	165
-par_	164
-res_	163
-ire_	163
-anç	161
-_plu	160
-plu	160
-ill	159
-m�	158
-une_	158
-ell	158
-ca	157
-�a	156
-ça	156
-nça	156
-fi	156
-_ce	156
-er_	156
-arti	155
-ans_	155
-elle	154
-j	154
-plus	154
-né	154
+ale	166
+_v	165
+plu	165
+une	164
+op	162
+ét	161
+gr	161
+_g	159
+_ce	158
+dan	158
+se_	157
+ul	157
+ac	154
+sa	154
+j	153
 mm	153
-�ais	152
-�ai	152
-çai	152
-ge	152
-rr	151
-és_	150
-�s_	150
-iv	150
-l�	149
-ag	148
-dan	147
-it_	146
-ions	146
-ien	144
-ons_	144
-ée_	143
-our	143
-ain	143
-�e_	143
-ot	142
-_son	142
-ep	141
-éc	141
-_no	141
-�c	141
-pr�	140
-��e	139
-’e	139
-�e	139
-lus_	137
-ais_	137
-air	137
-_su	136
-ine	135
-tre_	135
-ant_	135
-_tr	135
-tra	135
-ntr	133
-gi	133
-_si	133
-aire	133
-tai	133
-d_	132
-ter	132
-lit	132
-aux	131
-part	130
-ap	130
-lle_	130
+’a	151
+tu	151
+né	151
+fi	149
+im	148
+ts	147
+és_	143
+as	143
+ts_	142
+sp	142
+_dé	141
+it_	141
+_da	138
+su	138
+rti	137
+té_	136
+bl	136
+er_	135
+ge	135
+ell	134
+ise	134
+_ré	134
+ér	133
+our	133
+ç	133
+ia	133
+_no	131
+ag	130
+ntr	128
+ien	127
+nç	127
+ée_	125
+nça	124
+ça	124
+’e	124
+anç	123
+ni	122
+pp_	122
+iv	122
+çai	121
+ort	121
+com	119
+rr	117
+aux	116
+ond	116
+ui_	115
+ins	113
+tra	112
+_tr	112
+ain	111
+_pe	110
+’_	110
+ées	109
+iè	108
+ie_	108
+qui	108
+ses	107
+_su	107
+ca	107
+mb	107
+ine	106
+_b	105
+do	105
+_di	104
+éc	104
+ig	104
+d_	104
+br	104
+fa	103
+mme	103
+ect	103
+nne	102
+uv	101
+ap	101
+ot	101
+sé	101
+rés	100
+ép	100
+fo	100
+cu	99
+uis	99
+mai	98
+_j	98
+lit	98
+_te	97
+ter	96
+urs	96
+ris	96
+id	95
+ill	95
+oc	95
+cti	94
+air	94
+ité	93
+bre	93
+ues	93
+ier	93
+gi	93
+l’_	92
+iti	92
+por	91
+nte	90
+mon	90
+pro	90
+tan	90
+hi	90
+_in	90
+vo	89
+oir	89
+ouv	89
+av	89
+va	88
+nom	88
+_si	88
+pré	88
+ist	87
+tie	87
+_do	87
+ys	86
+onn	85
+_an	85
+mé	84
+ois	84
+_fa	84
+ers	84
+ure	84
+ég	83
+rop	83
+nts	83
+ga	83
+tes	83
+in_	82
+nde	82
+and	81
+omm	81
+_to	81
+ay	81
+tat	81
+mbr	81
+pri	81
+rm	80
+pui	80
+pou	80
+_ou	80
+eux	79
+ial	79
+gra	79
+ass	79