scylla 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -2
- data/Gemfile.lock +16 -1
- data/lib/scylla/classifier.rb +1 -1
- data/lib/scylla/generator.rb +16 -4
- data/lib/scylla/lms/afrikaans.lm +232 -232
- data/lib/scylla/lms/arabic.lm +175 -175
- data/lib/scylla/lms/bulgarian.lm +225 -225
- data/lib/scylla/lms/catalan.lm +309 -309
- data/lib/scylla/lms/danish.lm +167 -167
- data/lib/scylla/lms/english.lm +398 -398
- data/lib/scylla/lms/finnish.lm +237 -237
- data/lib/scylla/lms/french.lm +148 -148
- data/lib/scylla/lms/german.lm +258 -258
- data/lib/scylla/lms/greek.lm +236 -236
- data/lib/scylla/lms/hebrew.lm +154 -154
- data/lib/scylla/lms/hindi.lm +139 -139
- data/lib/scylla/lms/icelandic.lm +239 -239
- data/lib/scylla/lms/indonesian.lm +244 -244
- data/lib/scylla/lms/italian.lm +248 -248
- data/lib/scylla/lms/japanese.lm +90 -90
- data/lib/scylla/lms/korean.lm +306 -306
- data/lib/scylla/lms/norwegian.lm +193 -193
- data/lib/scylla/lms/polish.lm +241 -241
- data/lib/scylla/lms/portuguese.lm +232 -232
- data/lib/scylla/lms/romanian.lm +246 -246
- data/lib/scylla/lms/slovak.lm +242 -242
- data/lib/scylla/lms/slovenian.lm +229 -229
- data/lib/scylla/lms/spanish.lm +164 -164
- data/lib/scylla/lms/swedish.lm +157 -157
- data/lib/scylla/lms/tagalog.lm +247 -247
- data/lib/scylla/lms/thai.lm +252 -252
- data/lib/scylla/lms/turkish.lm +285 -285
- data/lib/scylla/lms/vietnamese.lm +250 -250
- data/lib/scylla/lms/welsh.lm +248 -248
- data/lib/scylla/resources.rb +1 -9
- data/lib/scylla.rb +4 -0
- data/scylla.gemspec +2 -120
- data/source_texts/english.txt +62 -27
- data/test/classifier_test.rb +1 -3
- data/test/fixtures/lms/danish.lm +173 -173
- data/test/fixtures/lms/english.lm +220 -220
- data/test/fixtures/lms/french.lm +175 -175
- data/test/fixtures/lms/german.lm +254 -254
- data/test/fixtures/lms/hindi.lm +139 -139
- data/test/fixtures/lms/italian.lm +236 -236
- data/test/fixtures/lms/japanese.lm +88 -88
- data/test/fixtures/lms/norwegian.lm +182 -182
- data/test/fixtures/lms/spanish.lm +164 -164
- data/test/fixtures/test_languages/spanish +0 -1
- data/test/generator_test.rb +13 -0
- data/test/helper.rb +2 -0
- metadata +18 -25
- data/.document +0 -5
- data/lib/scylla/lms/13375P33K.lm +0 -400
- data/scylla-0.1.0.gem +0 -0
- data/source_texts/13375P33K.txt +0 -199
- data/test/fixtures/lms/13375p33k.lm +0 -400
- data/test/fixtures/source_texts/13375P33K.txt +0 -199
data/lib/scylla/lms/finnish.lm
CHANGED
@@ -11,18 +11,18 @@ k 210
|
|
11
11
|
o 209
|
12
12
|
� 163
|
13
13
|
m 134
|
14
|
-
n_
|
14
|
+
n_ 130
|
15
15
|
v 99
|
16
16
|
y 82
|
17
17
|
a_ 79
|
18
|
-
r 78
|
19
18
|
an 78
|
19
|
+
r 78
|
20
20
|
ta 78
|
21
21
|
h 76
|
22
22
|
st 75
|
23
23
|
in 73
|
24
|
-
is 68
|
25
24
|
en 68
|
25
|
+
is 68
|
26
26
|
p 67
|
27
27
|
j 61
|
28
28
|
it 58
|
@@ -32,369 +32,369 @@ tt 51
|
|
32
32
|
ll 49
|
33
33
|
li 48
|
34
34
|
se 47
|
35
|
-
en_ 46
|
36
35
|
_k 46
|
36
|
+
en_ 46
|
37
37
|
mi 45
|
38
|
-
ka 45
|
39
38
|
va 45
|
39
|
+
ka 45
|
40
40
|
te 44
|
41
41
|
t� 44
|
42
42
|
i_ 43
|
43
|
-
el 42
|
44
43
|
_t 42
|
45
|
-
|
44
|
+
el 42
|
46
45
|
ai 42
|
46
|
+
tu 42
|
47
47
|
oi 40
|
48
48
|
. 40
|
49
49
|
es 39
|
50
50
|
ma 37
|
51
51
|
an_ 37
|
52
|
-
�_
|
52
|
+
�_ 36
|
53
53
|
d 33
|
54
|
-
et 33
|
55
54
|
_o 33
|
56
55
|
il 33
|
57
56
|
to 33
|
58
|
-
|
59
|
-
al 32
|
57
|
+
et 33
|
60
58
|
ki 32
|
61
59
|
_v 32
|
60
|
+
al 32
|
61
|
+
, 32
|
62
62
|
la 31
|
63
63
|
on 31
|
64
|
+
ks 30
|
65
|
+
ik 30
|
64
66
|
us 30
|
65
67
|
t_ 30
|
66
|
-
ik 30
|
67
|
-
ks 30
|
68
|
-
im 29
|
69
|
-
ja 29
|
70
68
|
ku 29
|
69
|
+
im 29
|
71
70
|
ri 29
|
71
|
+
ja 29
|
72
72
|
sa 29
|
73
|
-
ol 28
|
74
73
|
at 28
|
75
74
|
_s 28
|
76
75
|
ss 28
|
76
|
+
ol 28
|
77
|
+
,_ 27
|
77
78
|
le 27
|
78
79
|
uu 27
|
79
|
-
,_ 27
|
80
80
|
na 26
|
81
81
|
nn 26
|
82
|
+
ist 26
|
82
83
|
ti 26
|
83
84
|
_e 26
|
84
|
-
ist 26
|
85
85
|
lu 26
|
86
|
+
aan 25
|
86
87
|
ko 25
|
87
|
-
sta 25
|
88
|
-
e_ 25
|
89
88
|
._ 25
|
89
|
+
sta 25
|
90
90
|
_p 25
|
91
|
-
|
91
|
+
e_ 25
|
92
92
|
as 24
|
93
93
|
ii 23
|
94
|
-
nk 22
|
95
94
|
uo 22
|
95
|
+
nk 22
|
96
96
|
_j 22
|
97
|
-
ei 21
|
98
|
-
ne 21
|
99
97
|
un 21
|
100
|
-
ut 21
|
101
98
|
_h 21
|
99
|
+
ei 21
|
102
100
|
ke 21
|
101
|
+
ut 21
|
102
|
+
ne 21
|
103
103
|
ssa 20
|
104
|
-
|
105
|
-
in_ 19
|
106
|
-
os 19
|
104
|
+
in_ 20
|
107
105
|
__ 19
|
106
|
+
os 19
|
107
|
+
ha 19
|
108
108
|
_va 19
|
109
|
+
ak 18
|
110
|
+
_m 18
|
109
111
|
me 18
|
110
|
-
ar 18
|
111
|
-
v� 18
|
112
|
-
tta 18
|
113
112
|
am 18
|
114
|
-
|
113
|
+
tta 18
|
114
|
+
ie 18
|
115
|
+
t�_ 18
|
115
116
|
ia 18
|
116
|
-
|
117
|
-
|
118
|
-
ie 17
|
119
|
-
t�_ 17
|
120
|
-
ty 17
|
117
|
+
ar 18
|
118
|
+
v� 18
|
121
119
|
sk 17
|
122
|
-
|
120
|
+
ty 17
|
121
|
+
nt 17
|
123
122
|
oim 17
|
123
|
+
�� 17
|
124
124
|
� 17
|
125
|
-
nt 17
|
126
|
-
l� 17
|
127
125
|
ta_ 17
|
126
|
+
l� 17
|
127
|
+
er 17
|
128
|
+
_ka 16
|
128
129
|
ksi 16
|
129
130
|
_l 16
|
130
|
-
ul 16
|
131
|
-
om 16
|
132
131
|
yt 16
|
133
|
-
_ka 16
|
134
132
|
imi 16
|
133
|
+
om 16
|
134
|
+
ul 16
|
135
135
|
lli 16
|
136
|
+
toi 15
|
136
137
|
ste 15
|
137
|
-
de 15
|
138
|
-
ee 15
|
139
|
-
ise 15
|
140
|
-
aan_ 15
|
141
|
-
ais 15
|
142
|
-
ell 15
|
143
|
-
taa 15
|
144
138
|
K 15
|
145
|
-
|
139
|
+
aan_ 15
|
140
|
+
ee 15
|
146
141
|
_a 15
|
142
|
+
taa 15
|
147
143
|
vi 15
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
_on 14
|
144
|
+
ais 15
|
145
|
+
ise 15
|
146
|
+
de 15
|
147
|
+
ell 15
|
153
148
|
pa 14
|
149
|
+
au 14
|
154
150
|
ett 14
|
155
|
-
|
156
|
-
oimi 14
|
151
|
+
est 14
|
157
152
|
min 14
|
158
|
-
ni 14
|
159
153
|
ja_ 14
|
160
154
|
itt 14
|
155
|
+
ni 14
|
156
|
+
oimi 14
|
157
|
+
_on 14
|
161
158
|
lla 14
|
162
|
-
|
163
|
-
|
159
|
+
T 14
|
160
|
+
on_ 14
|
161
|
+
O 14
|
162
|
+
_ja 14
|
164
163
|
lt 13
|
165
|
-
|
166
|
-
|
164
|
+
s� 13
|
165
|
+
toim 13
|
166
|
+
em 13
|
167
|
+
kse 13
|
168
|
+
ok 13
|
169
|
+
_ol 13
|
167
170
|
je 13
|
171
|
+
su 13
|
168
172
|
all 13
|
169
|
-
|
173
|
+
_to 13
|
174
|
+
uk 13
|
170
175
|
- 13
|
171
|
-
au 13
|
172
|
-
su 13
|
173
|
-
kse 13
|
174
|
-
em 13
|
175
|
-
s� 13
|
176
|
-
nki 12
|
177
|
-
ot 12
|
178
176
|
toimi 12
|
179
|
-
|
180
|
-
|
181
|
-
vo 12
|
177
|
+
_y 12
|
178
|
+
ot 12
|
182
179
|
_ha 12
|
183
|
-
|
180
|
+
I 12
|
181
|
+
nki 12
|
184
182
|
sen 12
|
185
|
-
|
183
|
+
kk 12
|
184
|
+
vo 12
|
185
|
+
iin 12
|
186
186
|
E 12
|
187
|
-
�l 11
|
188
|
-
_ja_ 11
|
189
|
-
av 11
|
190
|
-
nna 11
|
191
|
-
�i 11
|
192
|
-
ten 11
|
193
|
-
he 11
|
194
|
-
a. 11
|
195
|
-
up 11
|
196
|
-
ht 11
|
197
|
-
lis 11
|
198
|
-
tus 11
|
199
|
-
ama 11
|
200
|
-
p� 11
|
201
187
|
_ko 11
|
202
188
|
ts 11
|
203
|
-
�n 11
|
204
189
|
tt� 11
|
205
|
-
|
206
|
-
|
190
|
+
tus 11
|
191
|
+
nna 11
|
192
|
+
p� 11
|
207
193
|
ust 11
|
208
|
-
|
209
|
-
|
194
|
+
av 11
|
195
|
+
_ja_ 11
|
196
|
+
lis 11
|
197
|
+
�n 11
|
198
|
+
�l 11
|
199
|
+
ama 11
|
200
|
+
up 11
|
201
|
+
mis 11
|
202
|
+
ht 11
|
203
|
+
he 11
|
204
|
+
�i 11
|
205
|
+
iv 11
|
206
|
+
a. 11
|
207
|
+
ten 11
|
208
|
+
_toim 10
|
209
|
+
yh 10
|
210
|
+
nne 10
|
211
|
+
_K 10
|
212
|
+
�t 10
|
213
|
+
�v 10
|
214
|
+
lle 10
|
210
215
|
sta_ 10
|
211
|
-
mm 10
|
212
|
-
a, 10
|
213
216
|
ah 10
|
214
|
-
_K 10
|
215
217
|
pi 10
|
216
|
-
nne 10
|
217
|
-
_toim 10
|
218
|
-
�v 10
|
219
218
|
si_ 10
|
220
|
-
|
221
|
-
�
|
219
|
+
yy 10
|
220
|
+
j� 10
|
221
|
+
lin 10
|
222
|
+
a, 10
|
222
223
|
aj 10
|
223
|
-
_tu 10
|
224
224
|
_toi 10
|
225
|
+
mm 10
|
226
|
+
_tu 10
|
225
227
|
ast 10
|
226
|
-
|
227
|
-
|
228
|
-
io 9
|
229
|
-
ra 9
|
230
|
-
st� 9
|
231
|
-
sa_ 9
|
232
|
-
iks 9
|
228
|
+
oj 9
|
229
|
+
uut 9
|
233
230
|
_- 9
|
234
|
-
|
235
|
-
eri 9
|
231
|
+
le_ 9
|
236
232
|
mu 9
|
237
|
-
ole 9
|
238
|
-
sti 9
|
239
|
-
uut 9
|
240
|
-
hd 9
|
241
233
|
_n 9
|
234
|
+
io 9
|
235
|
+
_ta 9
|
236
|
+
hd 9
|
237
|
+
alli 9
|
238
|
+
ra 9
|
242
239
|
V 9
|
240
|
+
st� 9
|
241
|
+
iks 9
|
243
242
|
ns 9
|
244
|
-
eh 9
|
245
243
|
aa_ 9
|
246
|
-
H 9
|
247
244
|
kau 9
|
248
245
|
uks 9
|
249
|
-
|
250
|
-
|
251
|
-
hu 9
|
252
|
-
le_ 9
|
253
|
-
oj 9
|
246
|
+
S 9
|
247
|
+
at_ 9
|
254
248
|
voi 9
|
249
|
+
eri 9
|
250
|
+
hu 9
|
251
|
+
H 9
|
252
|
+
sa_ 9
|
253
|
+
ole 9
|
254
|
+
ite 9
|
255
|
+
eh 9
|
256
|
+
sti 9
|
255
257
|
�. 9
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
258
|
+
oit 9
|
259
|
+
n. 8
|
260
|
+
van 8
|
261
|
+
kaup 8
|
262
|
+
iksi 8
|
263
|
+
�r 8
|
264
|
+
ari 8
|
265
|
+
_si 8
|
266
|
+
sia 8
|
267
|
+
ou 8
|
268
|
+
stu 8
|
269
|
+
ttaa 8
|
265
270
|
_ma 8
|
271
|
+
_i 8
|
272
|
+
no 8
|
266
273
|
ses 8
|
267
|
-
ou 8
|
268
|
-
aup 8
|
269
|
-
ine 8
|
270
274
|
esk 8
|
271
|
-
ess 8
|
272
|
-
kin 8
|
273
|
-
stu 8
|
274
275
|
lai 8
|
275
|
-
|
276
|
-
|
277
|
-
�r 8
|
278
|
-
nen_ 8
|
276
|
+
ess 8
|
277
|
+
rk 8
|
279
278
|
ila 8
|
280
|
-
|
281
|
-
|
279
|
+
ksi_ 8
|
280
|
+
hal 8
|
281
|
+
A 8
|
282
282
|
ank 8
|
283
|
+
ulu 8
|
283
284
|
isi 8
|
284
|
-
|
285
|
-
kaup 8
|
286
|
-
ttaa 8
|
287
|
-
A 8
|
288
|
-
van 8
|
289
|
-
iste 8
|
290
|
-
_on_ 8
|
291
|
-
ur 8
|
285
|
+
uv 8
|
292
286
|
L 8
|
293
|
-
_i 8
|
294
|
-
k� 8
|
295
|
-
iksi 8
|
296
287
|
n, 8
|
297
|
-
|
298
|
-
|
288
|
+
_on_ 8
|
289
|
+
k� 8
|
299
290
|
oli 8
|
291
|
+
ssa_ 8
|
292
|
+
ien 8
|
293
|
+
kin 8
|
300
294
|
M 8
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
295
|
+
aup 8
|
296
|
+
iste 8
|
297
|
+
nen 8
|
298
|
+
nta 8
|
299
|
+
nen_ 8
|
305
300
|
t�v 8
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
tel 7
|
314
|
-
m� 7
|
315
|
-
pp 7
|
316
|
-
ill 7
|
317
|
-
taan 7
|
318
|
-
s_ 7
|
301
|
+
ur 8
|
302
|
+
N 8
|
303
|
+
pe 8
|
304
|
+
ss� 8
|
305
|
+
ine 8
|
306
|
+
s_ 8
|
307
|
+
iss 7
|
319
308
|
til 7
|
320
|
-
|
321
|
-
|
309
|
+
tel 7
|
310
|
+
�v� 7
|
311
|
+
n� 7
|
312
|
+
maa 7
|
313
|
+
_r 7
|
314
|
+
ita 7
|
315
|
+
ella 7
|
322
316
|
P 7
|
323
|
-
|
324
|
-
|
325
|
-
|
317
|
+
ill 7
|
318
|
+
pu 7
|
319
|
+
_ku 7
|
320
|
+
lais 7
|
321
|
+
-_ 7
|
326
322
|
�s 7
|
323
|
+
sku 7
|
324
|
+
n,_ 7
|
327
325
|
_yh 7
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
la_ 7
|
326
|
+
a,_ 7
|
327
|
+
vai 7
|
328
|
+
ten_ 7
|
329
|
+
ri_ 7
|
333
330
|
inen_ 7
|
334
|
-
lais 7
|
335
|
-
ll� 7
|
336
|
-
inn 7
|
337
|
-
ita 7
|
338
|
-
lla_ 7
|
339
331
|
rj 7
|
332
|
+
jo 7
|
333
|
+
pp 7
|
334
|
+
oll 7
|
340
335
|
ain 7
|
341
|
-
|
342
|
-
|
343
|
-
|
336
|
+
_et 7
|
337
|
+
aka 7
|
338
|
+
kan 7
|
339
|
+
taan 7
|
340
|
+
m� 7
|
344
341
|
inen 7
|
342
|
+
ui 7
|
343
|
+
ll� 7
|
344
|
+
suu 7
|
345
|
+
a._ 7
|
346
|
+
lm 7
|
347
|
+
inn 7
|
348
|
+
la_ 7
|
349
|
+
lo 7
|
350
|
+
ava 7
|
345
351
|
tav 7
|
346
|
-
|
347
|
-
|
348
|
-
|
352
|
+
omi 7
|
353
|
+
sit 7
|
354
|
+
J 7
|
349
355
|
vat 7
|
350
|
-
|
351
|
-
oll 7
|
352
|
-
lm 7
|
353
|
-
aka 7
|
354
|
-
vai 7
|
356
|
+
lla_ 7
|
355
357
|
ost 7
|
356
|
-
|
357
|
-
iss 7
|
358
|
-
n� 7
|
359
|
-
ella 7
|
360
|
-
a._ 7
|
358
|
+
een 7
|
361
359
|
so 7
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
_T 6
|
367
|
-
�._ 6
|
368
|
-
_ett 6
|
369
|
-
utt 6
|
370
|
-
aise 6
|
371
|
-
ho 6
|
360
|
+
int 7
|
361
|
+
ukse 7
|
362
|
+
kin_ 6
|
363
|
+
nis 6
|
372
364
|
mme 6
|
373
|
-
|
374
|
-
|
375
|
-
|
365
|
+
sen_ 6
|
366
|
+
oje 6
|
367
|
+
aise 6
|
368
|
+
_voi 6
|
376
369
|
den 6
|
377
|
-
|
378
|
-
|
379
|
-
u_ 6
|
370
|
+
esi 6
|
371
|
+
lan 6
|
380
372
|
_-_ 6
|
381
373
|
sy 6
|
382
|
-
|
383
|
-
sten 6
|
384
|
-
ua 6
|
385
|
-
po 6
|
386
|
-
mari 6
|
374
|
+
nist 6
|
387
375
|
tee 6
|
388
|
-
|
376
|
+
_T 6
|
377
|
+
iku 6
|
378
|
+
_E 6
|
379
|
+
mar 6
|
380
|
+
_l� 6
|
389
381
|
�li 6
|
390
|
-
|
391
|
-
|
392
|
-
assa 6
|
393
|
-
kil 6
|
394
|
-
tet 6
|
395
|
-
skus 6
|
382
|
+
llis 6
|
383
|
+
ih 6
|
396
384
|
ima 6
|
397
|
-
|
398
|
-
|
399
|
-
|
385
|
+
Ke 6
|
386
|
+
sten 6
|
387
|
+
its 6
|
388
|
+
assa 6
|
389
|
+
op 6
|
390
|
+
_es 6
|
400
391
|
ytt 6
|
392
|
+
_vo 6
|
393
|
+
_vai 6
|
394
|
+
tar 6
|
395
|
+
mari 6
|
396
|
+
ev 6
|
397
|
+
po 6
|
398
|
+
ska 6
|
399
|
+
ista 6
|
400
|
+
na_ 6
|