scylla 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -2
- data/Gemfile.lock +16 -1
- data/lib/scylla/classifier.rb +1 -1
- data/lib/scylla/generator.rb +16 -4
- data/lib/scylla/lms/afrikaans.lm +232 -232
- data/lib/scylla/lms/arabic.lm +175 -175
- data/lib/scylla/lms/bulgarian.lm +225 -225
- data/lib/scylla/lms/catalan.lm +309 -309
- data/lib/scylla/lms/danish.lm +167 -167
- data/lib/scylla/lms/english.lm +398 -398
- data/lib/scylla/lms/finnish.lm +237 -237
- data/lib/scylla/lms/french.lm +148 -148
- data/lib/scylla/lms/german.lm +258 -258
- data/lib/scylla/lms/greek.lm +236 -236
- data/lib/scylla/lms/hebrew.lm +154 -154
- data/lib/scylla/lms/hindi.lm +139 -139
- data/lib/scylla/lms/icelandic.lm +239 -239
- data/lib/scylla/lms/indonesian.lm +244 -244
- data/lib/scylla/lms/italian.lm +248 -248
- data/lib/scylla/lms/japanese.lm +90 -90
- data/lib/scylla/lms/korean.lm +306 -306
- data/lib/scylla/lms/norwegian.lm +193 -193
- data/lib/scylla/lms/polish.lm +241 -241
- data/lib/scylla/lms/portuguese.lm +232 -232
- data/lib/scylla/lms/romanian.lm +246 -246
- data/lib/scylla/lms/slovak.lm +242 -242
- data/lib/scylla/lms/slovenian.lm +229 -229
- data/lib/scylla/lms/spanish.lm +164 -164
- data/lib/scylla/lms/swedish.lm +157 -157
- data/lib/scylla/lms/tagalog.lm +247 -247
- data/lib/scylla/lms/thai.lm +252 -252
- data/lib/scylla/lms/turkish.lm +285 -285
- data/lib/scylla/lms/vietnamese.lm +250 -250
- data/lib/scylla/lms/welsh.lm +248 -248
- data/lib/scylla/resources.rb +1 -9
- data/lib/scylla.rb +4 -0
- data/scylla.gemspec +2 -120
- data/source_texts/english.txt +62 -27
- data/test/classifier_test.rb +1 -3
- data/test/fixtures/lms/danish.lm +173 -173
- data/test/fixtures/lms/english.lm +220 -220
- data/test/fixtures/lms/french.lm +175 -175
- data/test/fixtures/lms/german.lm +254 -254
- data/test/fixtures/lms/hindi.lm +139 -139
- data/test/fixtures/lms/italian.lm +236 -236
- data/test/fixtures/lms/japanese.lm +88 -88
- data/test/fixtures/lms/norwegian.lm +182 -182
- data/test/fixtures/lms/spanish.lm +164 -164
- data/test/fixtures/test_languages/spanish +0 -1
- data/test/generator_test.rb +13 -0
- data/test/helper.rb +2 -0
- metadata +18 -25
- data/.document +0 -5
- data/lib/scylla/lms/13375P33K.lm +0 -400
- data/scylla-0.1.0.gem +0 -0
- data/source_texts/13375P33K.txt +0 -199
- data/test/fixtures/lms/13375p33k.lm +0 -400
- data/test/fixtures/source_texts/13375P33K.txt +0 -199
data/lib/scylla/lms/spanish.lm
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
_
|
|
1
|
+
_ 34431
|
|
2
2
|
a 10584
|
|
3
3
|
e 10526
|
|
4
4
|
o 7129
|
|
@@ -10,128 +10,128 @@ l 5112
|
|
|
10
10
|
d 4622
|
|
11
11
|
t 3867
|
|
12
12
|
c 3674
|
|
13
|
-
a_
|
|
14
|
-
e_
|
|
13
|
+
a_ 3001
|
|
14
|
+
e_ 2946
|
|
15
15
|
u 2914
|
|
16
16
|
� 2572
|
|
17
|
-
s_
|
|
17
|
+
s_ 2533
|
|
18
18
|
de 2468
|
|
19
|
-
_d
|
|
19
|
+
_d 2335
|
|
20
20
|
p 2303
|
|
21
21
|
m 2195
|
|
22
|
-
_de
|
|
23
|
-
o_
|
|
24
|
-
_e
|
|
25
|
-
n_
|
|
22
|
+
_de 2093
|
|
23
|
+
o_ 1885
|
|
24
|
+
_e 1787
|
|
25
|
+
n_ 1774
|
|
26
26
|
en 1746
|
|
27
27
|
de_ 1679
|
|
28
28
|
_de_ 1591
|
|
29
29
|
la 1459
|
|
30
30
|
es 1454
|
|
31
|
-
_l
|
|
31
|
+
_l 1380
|
|
32
32
|
, 1276
|
|
33
|
-
,_
|
|
34
|
-
l_
|
|
33
|
+
,_ 1261
|
|
34
|
+
l_ 1259
|
|
35
35
|
os 1221
|
|
36
36
|
er 1194
|
|
37
37
|
on 1122
|
|
38
38
|
as 1103
|
|
39
39
|
ci 1095
|
|
40
|
-
_p
|
|
40
|
+
_p 1077
|
|
41
|
+
_c 1054
|
|
41
42
|
el 1049
|
|
42
|
-
_c 1047
|
|
43
43
|
an 1032
|
|
44
44
|
ra 1018
|
|
45
45
|
al 997
|
|
46
|
+
os_ 941
|
|
46
47
|
g 941
|
|
47
|
-
_la
|
|
48
|
-
os_ 930
|
|
48
|
+
_la 935
|
|
49
49
|
nt 926
|
|
50
50
|
te 922
|
|
51
51
|
co 901
|
|
52
|
+
_a 898
|
|
52
53
|
b 896
|
|
53
|
-
_a 893
|
|
54
54
|
re 888
|
|
55
55
|
ta 858
|
|
56
56
|
ad 827
|
|
57
57
|
ri 825
|
|
58
|
+
la_ 816
|
|
58
59
|
ar 816
|
|
59
60
|
or 812
|
|
60
|
-
|
|
61
|
+
_s 800
|
|
61
62
|
el_ 796
|
|
62
|
-
_s 795
|
|
63
63
|
. 766
|
|
64
64
|
pa 757
|
|
65
65
|
� 756
|
|
66
66
|
ó 755
|
|
67
67
|
do 751
|
|
68
68
|
st 737
|
|
69
|
+
as_ 726
|
|
69
70
|
ro 725
|
|
70
71
|
y 717
|
|
71
|
-
|
|
72
|
-
_la_ 712
|
|
72
|
+
_la_ 714
|
|
73
73
|
na 691
|
|
74
74
|
ue 688
|
|
75
75
|
v 663
|
|
76
76
|
in 658
|
|
77
|
-
_en
|
|
77
|
+
_en 651
|
|
78
78
|
ca 643
|
|
79
|
+
es_ 636
|
|
79
80
|
ic 635
|
|
80
81
|
en_ 631
|
|
81
82
|
da 621
|
|
82
83
|
ia 620
|
|
83
|
-
es_ 620
|
|
84
84
|
E 615
|
|
85
85
|
to 609
|
|
86
86
|
lo 605
|
|
87
|
-
_m
|
|
87
|
+
_m 604
|
|
88
88
|
f 596
|
|
89
|
-
_co
|
|
89
|
+
_co 595
|
|
90
90
|
y_ 562
|
|
91
|
+
_el 557
|
|
91
92
|
� 557
|
|
92
93
|
í 557
|
|
93
|
-
|
|
94
|
+
_en_ 535
|
|
94
95
|
ti 535
|
|
95
96
|
no 532
|
|
96
|
-
|
|
97
|
-
_y 523
|
|
98
|
-
un 513
|
|
97
|
+
_y 525
|
|
99
98
|
ent 513
|
|
99
|
+
un 513
|
|
100
|
+
r_ 510
|
|
101
|
+
_el_ 509
|
|
100
102
|
le 507
|
|
101
|
-
_el_ 505
|
|
102
|
-
r_ 499
|
|
103
103
|
io 495
|
|
104
|
-
_y_
|
|
104
|
+
_y_ 494
|
|
105
105
|
ón 488
|
|
106
106
|
�n 488
|
|
107
107
|
i� 487
|
|
108
108
|
po 475
|
|
109
109
|
a� 472
|
|
110
110
|
se 468
|
|
111
|
+
_E 464
|
|
111
112
|
sp 460
|
|
112
|
-
_E 459
|
|
113
113
|
q 456
|
|
114
114
|
qu 455
|
|
115
|
-
� 453
|
|
116
115
|
ñ 453
|
|
116
|
+
� 453
|
|
117
117
|
C 451
|
|
118
118
|
tr 450
|
|
119
|
-
_t
|
|
119
|
+
_t 449
|
|
120
120
|
ma 440
|
|
121
|
-
._
|
|
121
|
+
._ 438
|
|
122
122
|
id 432
|
|
123
123
|
ac 431
|
|
124
124
|
ió 428
|
|
125
125
|
is 427
|
|
126
126
|
ni 426
|
|
127
127
|
om 425
|
|
128
|
-
ne 414
|
|
129
128
|
nte 414
|
|
129
|
+
ne 414
|
|
130
130
|
con 410
|
|
131
|
-
do_
|
|
131
|
+
do_ 408
|
|
132
132
|
nc 406
|
|
133
|
+
_r 404
|
|
133
134
|
nd 404
|
|
134
|
-
_r 403
|
|
135
135
|
li 403
|
|
136
136
|
ie 401
|
|
137
137
|
si 399
|
|
@@ -141,260 +141,260 @@ añ 395
|
|
|
141
141
|
á 391
|
|
142
142
|
h 383
|
|
143
143
|
pr 382
|
|
144
|
+
�n_ 379
|
|
145
|
+
ón_ 379
|
|
146
|
+
_C 376
|
|
144
147
|
spa 375
|
|
145
|
-
�n_ 375
|
|
146
|
-
ón_ 375
|
|
147
148
|
que 374
|
|
148
149
|
ión 373
|
|
149
|
-
ue_
|
|
150
|
+
ue_ 371
|
|
150
151
|
pa� 363
|
|
151
|
-
|
|
152
|
+
_con 341
|
|
153
|
+
ión_ 341
|
|
152
154
|
ec 341
|
|
153
155
|
mi 340
|
|
154
|
-
|
|
155
|
-
_con 338
|
|
156
|
-
_re 336
|
|
157
|
-
rt 336
|
|
156
|
+
_re 337
|
|
158
157
|
di 336
|
|
158
|
+
rt 336
|
|
159
159
|
pe 333
|
|
160
|
-
|
|
160
|
+
al_ 332
|
|
161
|
+
te_ 332
|
|
162
|
+
_pr 330
|
|
161
163
|
A 327
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
_qu
|
|
165
|
-
_q 319
|
|
164
|
+
que_ 325
|
|
165
|
+
_q 320
|
|
166
|
+
_qu 320
|
|
166
167
|
it 317
|
|
167
168
|
pañ 315
|
|
168
169
|
Es 313
|
|
169
|
-
|
|
170
|
+
_es 313
|
|
171
|
+
_se 311
|
|
170
172
|
mo 310
|
|
171
|
-
spa� 309
|
|
172
173
|
_lo 309
|
|
173
174
|
spañ 309
|
|
174
|
-
|
|
175
|
-
|
|
175
|
+
spa� 309
|
|
176
|
+
_que 309
|
|
177
|
+
_po 308
|
|
176
178
|
los 305
|
|
177
|
-
|
|
178
|
-
|
|
179
|
+
a, 305
|
|
180
|
+
a,_ 303
|
|
179
181
|
ol 302
|
|
180
|
-
ci� 300
|
|
181
182
|
aci 300
|
|
183
|
+
ci� 300
|
|
182
184
|
ció 299
|
|
183
185
|
los_ 296
|
|
184
|
-
|
|
186
|
+
_i 295
|
|
185
187
|
I 295
|
|
186
188
|
ado 294
|
|
187
|
-
|
|
189
|
+
_que_ 293
|
|
188
190
|
ur 293
|
|
189
|
-
|
|
190
|
-
|
|
191
|
+
__ 292
|
|
192
|
+
_Es 287
|
|
191
193
|
ción 285
|
|
192
|
-
_Es 282
|
|
193
194
|
su 282
|
|
194
|
-
|
|
195
|
+
_f 279
|
|
195
196
|
ña 278
|
|
197
|
+
�a 278
|
|
196
198
|
z 275
|
|
197
|
-
|
|
199
|
+
_u 275
|
|
200
|
+
on_ 273
|
|
198
201
|
ica 271
|
|
199
|
-
_u 271
|
|
200
202
|
del 270
|
|
201
|
-
on_ 270
|
|
202
|
-
cia 267
|
|
203
203
|
ce 267
|
|
204
|
+
cia 267
|
|
204
205
|
_del 266
|
|
205
206
|
del_ 261
|
|
206
207
|
é 261
|
|
207
208
|
� 261
|
|
209
|
+
_h 260
|
|
210
|
+
or_ 260
|
|
208
211
|
nci 260
|
|
209
212
|
_del_ 259
|
|
213
|
+
_o 259
|
|
214
|
+
s, 259
|
|
215
|
+
s,_ 258
|
|
210
216
|
tu 258
|
|
211
|
-
_o 258
|
|
212
|
-
_h 257
|
|
213
217
|
_los 257
|
|
214
|
-
|
|
218
|
+
_un 256
|
|
215
219
|
am 255
|
|
216
220
|
_los_ 254
|
|
217
221
|
sta 253
|
|
218
|
-
|
|
219
|
-
Espa� 250
|
|
222
|
+
ra_ 251
|
|
220
223
|
Esp 250
|
|
224
|
+
Espa� 250
|
|
221
225
|
Espa 250
|
|
222
|
-
|
|
223
|
-
s, 249
|
|
224
|
-
ra_ 248
|
|
226
|
+
ll 245
|
|
225
227
|
est 245
|
|
226
228
|
ab 245
|
|
227
|
-
ll 245
|
|
228
|
-
las 244
|
|
229
229
|
por 244
|
|
230
|
+
las 244
|
|
231
|
+
ía 243
|
|
230
232
|
�a 243
|
|
231
|
-
__ 243
|
|
232
233
|
aña 243
|
|
233
|
-
ía 243
|
|
234
234
|
r� 239
|
|
235
235
|
at 239
|
|
236
|
-
|
|
236
|
+
_A 236
|
|
237
237
|
paña 234
|
|
238
|
-
|
|
238
|
+
so 234
|
|
239
|
+
_a_ 232
|
|
239
240
|
im 231
|
|
240
241
|
las_ 230
|
|
241
|
-
|
|
242
|
+
_Esp 230
|
|
243
|
+
_Espa 230
|
|
242
244
|
ns 228
|
|
245
|
+
na_ 228
|
|
243
246
|
em 226
|
|
244
|
-
_Espa 226
|
|
245
|
-
_Esp 226
|
|
246
247
|
cu 226
|
|
247
248
|
j 224
|
|
248
|
-
|
|
249
|
+
_n 220
|
|
249
250
|
ul 220
|
|
251
|
+
ia_ 220
|
|
252
|
+
nte_ 220
|
|
250
253
|
P 219
|
|
251
254
|
ant 219
|
|
252
|
-
|
|
255
|
+
to_ 219
|
|
253
256
|
rr 218
|
|
254
|
-
|
|
255
|
-
nte_ 217
|
|
256
|
-
) 216
|
|
257
|
-
_n 216
|
|
258
|
-
( 216
|
|
259
|
-
dad 215
|
|
257
|
+
ente 218
|
|
260
258
|
se_ 215
|
|
261
|
-
|
|
262
|
-
_( 214
|
|
263
|
-
vi 213
|
|
259
|
+
dad 215
|
|
264
260
|
il 213
|
|
261
|
+
vi 213
|
|
265
262
|
L 211
|
|
266
263
|
ter 209
|
|
267
|
-
_pa
|
|
264
|
+
_pa 208
|
|
268
265
|
ada 205
|
|
266
|
+
o, 204
|
|
269
267
|
men 203
|
|
268
|
+
o,_ 202
|
|
269
|
+
da_ 202
|
|
270
270
|
era 202
|
|
271
271
|
ran 201
|
|
272
272
|
les 201
|
|
273
|
-
|
|
273
|
+
_su 200
|
|
274
274
|
ig 198
|
|
275
|
-
_su 198
|
|
276
|
-
o, 197
|
|
277
|
-
o,_ 195
|
|
278
275
|
ien 195
|
|
279
276
|
tra 193
|
|
280
277
|
res 192
|
|
281
|
-
|
|
278
|
+
_ca 191
|
|
282
279
|
com 190
|
|
283
|
-
|
|
280
|
+
cio 190
|
|
284
281
|
one 189
|
|
285
|
-
|
|
286
|
-
ed 188
|
|
282
|
+
ida 189
|
|
287
283
|
M 188
|
|
284
|
+
ed 188
|
|
288
285
|
S 187
|
|
289
|
-
|
|
286
|
+
_las_ 186
|
|
287
|
+
_las 186
|
|
290
288
|
m� 185
|
|
291
|
-
_las_ 185
|
|
292
|
-
ona 185
|
|
293
289
|
t� 185
|
|
294
|
-
|
|
290
|
+
ona 185
|
|
291
|
+
no_ 184
|
|
295
292
|
ha 184
|
|
293
|
+
ion 184
|
|
296
294
|
od 184
|
|
297
295
|
nes 183
|
|
298
|
-
no_ 182
|
|
299
296
|
ale 180
|
|
300
|
-
|
|
301
|
-
|
|
297
|
+
_com 179
|
|
298
|
+
_in 178
|
|
299
|
+
_por 178
|
|
302
300
|
br 178
|
|
303
|
-
|
|
301
|
+
sa 178
|
|
304
302
|
mp 177
|
|
305
303
|
bi 176
|
|
306
|
-
_in 176
|
|
307
|
-
ist 175
|
|
308
304
|
pro 175
|
|
309
|
-
|
|
305
|
+
ist 175
|
|
310
306
|
ació 173
|
|
311
307
|
dos 173
|
|
308
|
+
aci� 173
|
|
312
309
|
ct 172
|
|
313
|
-
des 172
|
|
314
310
|
oc 172
|
|
311
|
+
des 172
|
|
312
|
+
_I 171
|
|
313
|
+
_al 168
|
|
315
314
|
eg 167
|
|
316
|
-
|
|
317
|
-
an_ 166
|
|
318
|
-
_I 166
|
|
315
|
+
_pro 166
|
|
319
316
|
por_ 166
|
|
317
|
+
an_ 166
|
|
318
|
+
a. 165
|
|
319
|
+
_P 165
|
|
320
|
+
_se_ 165
|
|
321
|
+
_por_ 165
|
|
322
|
+
_v 165
|
|
320
323
|
ero 165
|
|
321
|
-
_pro 164
|
|
322
|
-
_se_ 164
|
|
323
|
-
_por_ 164
|
|
324
|
-
ño 163
|
|
325
324
|
�o 163
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
325
|
+
�a_ 163
|
|
326
|
+
ño 163
|
|
327
|
+
ía_ 163
|
|
329
328
|
va 161
|
|
330
329
|
ment 161
|
|
331
|
-
lo_
|
|
330
|
+
lo_ 161
|
|
332
331
|
iv 160
|
|
333
|
-
_P 160
|
|
334
332
|
gu 159
|
|
335
333
|
ndo 159
|
|
336
334
|
mu 158
|
|
337
|
-
|
|
338
|
-
|
|
335
|
+
_si 157
|
|
336
|
+
et 157
|
|
337
|
+
d_ 156
|
|
339
338
|
ici 155
|
|
340
|
-
d_ 155
|
|
341
339
|
fi 155
|
|
340
|
+
_L 154
|
|
342
341
|
ria 154
|
|
343
342
|
go 154
|
|
344
|
-
|
|
345
|
-
mo_
|
|
343
|
+
_S 153
|
|
344
|
+
mo_ 153
|
|
345
|
+
s. 152
|
|
346
346
|
ron 152
|
|
347
|
-
op 151
|
|
348
|
-
_ha 151
|
|
349
|
-
ones 151
|
|
350
347
|
ga 151
|
|
351
348
|
za 151
|
|
349
|
+
_ha 151
|
|
350
|
+
op 151
|
|
351
|
+
ones 151
|
|
352
352
|
us 150
|
|
353
353
|
mb 149
|
|
354
|
-
año 148
|
|
355
354
|
ba 148
|
|
356
355
|
Ca 148
|
|
356
|
+
año 148
|
|
357
|
+
ado_ 147
|
|
357
358
|
_pe 147
|
|
359
|
+
nes_ 147
|
|
358
360
|
sti 147
|
|
359
361
|
ncia 147
|
|
360
|
-
|
|
362
|
+
ña_ 146
|
|
361
363
|
ua 146
|
|
362
|
-
uc 146
|
|
363
364
|
ico 146
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
365
|
+
�a_ 146
|
|
366
|
+
uc 146
|
|
367
|
+
ente_ 145
|
|
368
|
+
_M 145
|
|
368
369
|
ve 144
|
|
369
|
-
ña_ 144
|
|
370
|
-
rio 143
|
|
371
370
|
cion 143
|
|
372
|
-
|
|
373
|
-
_M 142
|
|
371
|
+
rio 143
|
|
374
372
|
_con_ 142
|
|
375
373
|
con_ 142
|
|
376
|
-
ente_ 142
|
|
377
|
-
ip 141
|
|
378
374
|
rc 141
|
|
375
|
+
io_ 141
|
|
376
|
+
ip 141
|
|
377
|
+
ca_ 140
|
|
379
378
|
ntr 140
|
|
380
|
-
|
|
381
|
-
tor 139
|
|
382
|
-
ca_ 139
|
|
379
|
+
_g 139
|
|
383
380
|
nto 139
|
|
384
|
-
|
|
385
|
-
|
|
381
|
+
tor 139
|
|
382
|
+
ta_ 139
|
|
383
|
+
aña_ 138
|
|
386
384
|
par 138
|
|
387
|
-
|
|
385
|
+
ob 138
|
|
386
|
+
dos_ 137
|
|
388
387
|
ir 137
|
|
389
388
|
G 137
|
|
390
|
-
|
|
389
|
+
_. 137
|
|
390
|
+
ante 136
|
|
391
|
+
_, 136
|
|
391
392
|
bl 136
|
|
392
393
|
n� 136
|
|
393
|
-
|
|
394
|
-
dos_ 135
|
|
395
|
-
err 134
|
|
396
|
-
�_ 134
|
|
397
|
-
rm 134
|
|
398
|
-
ó_ 134
|
|
394
|
+
les_ 135
|
|
399
395
|
eri 134
|
|
400
396
|
ori 134
|
|
397
|
+
rm 134
|
|
398
|
+
�_ 134
|
|
399
|
+
ó_ 134
|
|
400
|
+
err 134
|