scylla 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -2
- data/Gemfile.lock +16 -1
- data/lib/scylla/classifier.rb +1 -1
- data/lib/scylla/generator.rb +16 -4
- data/lib/scylla/lms/afrikaans.lm +232 -232
- data/lib/scylla/lms/arabic.lm +175 -175
- data/lib/scylla/lms/bulgarian.lm +225 -225
- data/lib/scylla/lms/catalan.lm +309 -309
- data/lib/scylla/lms/danish.lm +167 -167
- data/lib/scylla/lms/english.lm +398 -398
- data/lib/scylla/lms/finnish.lm +237 -237
- data/lib/scylla/lms/french.lm +148 -148
- data/lib/scylla/lms/german.lm +258 -258
- data/lib/scylla/lms/greek.lm +236 -236
- data/lib/scylla/lms/hebrew.lm +154 -154
- data/lib/scylla/lms/hindi.lm +139 -139
- data/lib/scylla/lms/icelandic.lm +239 -239
- data/lib/scylla/lms/indonesian.lm +244 -244
- data/lib/scylla/lms/italian.lm +248 -248
- data/lib/scylla/lms/japanese.lm +90 -90
- data/lib/scylla/lms/korean.lm +306 -306
- data/lib/scylla/lms/norwegian.lm +193 -193
- data/lib/scylla/lms/polish.lm +241 -241
- data/lib/scylla/lms/portuguese.lm +232 -232
- data/lib/scylla/lms/romanian.lm +246 -246
- data/lib/scylla/lms/slovak.lm +242 -242
- data/lib/scylla/lms/slovenian.lm +229 -229
- data/lib/scylla/lms/spanish.lm +164 -164
- data/lib/scylla/lms/swedish.lm +157 -157
- data/lib/scylla/lms/tagalog.lm +247 -247
- data/lib/scylla/lms/thai.lm +252 -252
- data/lib/scylla/lms/turkish.lm +285 -285
- data/lib/scylla/lms/vietnamese.lm +250 -250
- data/lib/scylla/lms/welsh.lm +248 -248
- data/lib/scylla/resources.rb +1 -9
- data/lib/scylla.rb +4 -0
- data/scylla.gemspec +2 -120
- data/source_texts/english.txt +62 -27
- data/test/classifier_test.rb +1 -3
- data/test/fixtures/lms/danish.lm +173 -173
- data/test/fixtures/lms/english.lm +220 -220
- data/test/fixtures/lms/french.lm +175 -175
- data/test/fixtures/lms/german.lm +254 -254
- data/test/fixtures/lms/hindi.lm +139 -139
- data/test/fixtures/lms/italian.lm +236 -236
- data/test/fixtures/lms/japanese.lm +88 -88
- data/test/fixtures/lms/norwegian.lm +182 -182
- data/test/fixtures/lms/spanish.lm +164 -164
- data/test/fixtures/test_languages/spanish +0 -1
- data/test/generator_test.rb +13 -0
- data/test/helper.rb +2 -0
- metadata +18 -25
- data/.document +0 -5
- data/lib/scylla/lms/13375P33K.lm +0 -400
- data/scylla-0.1.0.gem +0 -0
- data/source_texts/13375P33K.txt +0 -199
- data/test/fixtures/lms/13375p33k.lm +0 -400
- data/test/fixtures/source_texts/13375P33K.txt +0 -199
data/lib/scylla/lms/turkish.lm
CHANGED
@@ -3,398 +3,398 @@ i 470
|
|
3
3
|
a 439
|
4
4
|
e 316
|
5
5
|
n 276
|
6
|
-
& 256
|
7
|
-
; 256
|
8
6
|
l 244
|
9
7
|
r 239
|
10
8
|
k 175
|
11
9
|
s 172
|
12
|
-
&i; 166
|
13
|
-
&i 166
|
14
|
-
i; 166
|
15
10
|
d 148
|
16
11
|
t 119
|
12
|
+
__ 114
|
17
13
|
y 113
|
18
14
|
m 110
|
19
|
-
|
15
|
+
in 107
|
20
16
|
u 105
|
21
17
|
n_ 101
|
18
|
+
i_ 99
|
22
19
|
o 97
|
23
20
|
b 90
|
24
21
|
ar 75
|
25
|
-
g 73
|
26
22
|
la 73
|
23
|
+
g 73
|
27
24
|
e_ 71
|
25
|
+
il 67
|
28
26
|
an 66
|
29
|
-
� 65
|
30
27
|
a_ 65
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
in 59
|
35
|
-
_b 56
|
36
|
-
i_ 56
|
28
|
+
� 65
|
29
|
+
si 64
|
30
|
+
_b 57
|
37
31
|
_, 54
|
38
|
-
, 54
|
39
32
|
ra 54
|
33
|
+
, 54
|
40
34
|
le 52
|
41
|
-
|
42
|
-
;_ 49
|
35
|
+
ri 52
|
43
36
|
er 48
|
37
|
+
_,_ 48
|
38
|
+
is 48
|
44
39
|
da 48
|
45
|
-
,_
|
46
|
-
_,_ 47
|
47
|
-
&i;n 44
|
48
|
-
i;n 44
|
49
|
-
;n 44
|
40
|
+
,_ 48
|
50
41
|
z 44
|
51
|
-
&i;_ 43
|
52
|
-
i;_ 43
|
53
42
|
_y 43
|
43
|
+
ni 43
|
54
44
|
k_ 42
|
45
|
+
li 41
|
46
|
+
in_ 41
|
55
47
|
_k 41
|
56
|
-
de 40
|
57
48
|
en 40
|
58
|
-
|
49
|
+
di 40
|
59
50
|
nd 40
|
60
|
-
|
51
|
+
de 40
|
61
52
|
_. 39
|
62
53
|
p 39
|
54
|
+
. 39
|
55
|
+
ir 37
|
63
56
|
lar 37
|
64
57
|
ka 36
|
65
|
-
|
58
|
+
_s 35
|
66
59
|
r_ 35
|
67
60
|
ya 34
|
68
|
-
ri 34
|
69
|
-
c 33
|
70
61
|
v 33
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
g; 30
|
76
|
-
&g; 30
|
77
|
-
� 30
|
78
|
-
&g 30
|
62
|
+
c 33
|
63
|
+
_d 33
|
64
|
+
_i 32
|
65
|
+
ki 31
|
79
66
|
bi 30
|
67
|
+
� 30
|
68
|
+
ma 30
|
69
|
+
ak 30
|
80
70
|
ara 29
|
81
71
|
ol 29
|
82
|
-
|
72
|
+
_a 28
|
83
73
|
_t 28
|
74
|
+
as 28
|
75
|
+
gi 28
|
84
76
|
ler 28
|
85
|
-
_a 28
|
86
|
-
ki 28
|
87
|
-
in_ 28
|
88
77
|
eri 27
|
89
78
|
h 27
|
90
79
|
re 27
|
91
|
-
ir 27
|
92
|
-
si 26
|
93
80
|
ek 26
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
81
|
+
ind 26
|
82
|
+
ini 26
|
83
|
+
rin 25
|
84
|
+
ti 25
|
85
|
+
ik 24
|
98
86
|
u_ 23
|
99
|
-
ni 23
|
100
87
|
ba 23
|
101
|
-
|
88
|
+
me 23
|
89
|
+
sin 23
|
102
90
|
_g 22
|
103
91
|
leri 22
|
104
|
-
di 22
|
105
|
-
r& 22
|
106
92
|
te 22
|
107
|
-
on 22
|
108
|
-
_o 22
|
109
93
|
nda 22
|
110
|
-
|
94
|
+
_o 22
|
95
|
+
on 22
|
96
|
+
an_ 22
|
97
|
+
� 21
|
98
|
+
ve 21
|
99
|
+
mi 21
|
100
|
+
un 21
|
111
101
|
al 21
|
102
|
+
se 21
|
112
103
|
en_ 21
|
113
|
-
|
114
|
-
s&i 21
|
115
|
-
s& 21
|
116
|
-
ve 21
|
104
|
+
_bi 21
|
117
105
|
f 21
|
118
|
-
� 21
|
119
|
-
at 20
|
120
|
-
da_ 20
|
121
|
-
n&i; 20
|
122
|
-
n&i 20
|
123
|
-
n& 20
|
124
106
|
et 20
|
107
|
+
es 20
|
125
108
|
ay 20
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
ile 19
|
130
|
-
_v 19
|
131
|
-
es 19
|
132
|
-
i&s 19
|
109
|
+
at 20
|
110
|
+
_v 20
|
111
|
+
da_ 20
|
133
112
|
bu 19
|
134
|
-
|
135
|
-
|
136
|
-
&i;l 18
|
137
|
-
i;& 18
|
138
|
-
&i;& 18
|
139
|
-
i;l 18
|
140
|
-
r&i; 18
|
141
|
-
;&i; 18
|
142
|
-
;&i 18
|
143
|
-
d& 18
|
144
|
-
d&i; 18
|
145
|
-
ar& 17
|
146
|
-
l&i 17
|
147
|
-
;i 17
|
148
|
-
l&i; 17
|
149
|
-
mi 17
|
113
|
+
ile 19
|
114
|
+
si_ 18
|
150
115
|
_ya 17
|
151
|
-
nu 17
|
152
|
-
l& 17
|
153
116
|
l_ 17
|
117
|
+
_ve 17
|
154
118
|
_ol 17
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
119
|
+
sa 17
|
120
|
+
nin 17
|
121
|
+
nu 17
|
122
|
+
ta 16
|
123
|
+
ari 16
|
160
124
|
_ka 16
|
161
|
-
|
125
|
+
ad 16
|
126
|
+
inda 16
|
127
|
+
iy 16
|
162
128
|
or 16
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
129
|
+
ola 16
|
130
|
+
ye 16
|
131
|
+
ed 16
|
132
|
+
pa 16
|
133
|
+
asi 15
|
168
134
|
de_ 15
|
169
|
-
|
135
|
+
iz 15
|
136
|
+
nl 15
|
137
|
+
nin_ 15
|
170
138
|
na 15
|
171
|
-
;nd 15
|
172
139
|
el 15
|
173
|
-
|
174
|
-
ed 15
|
175
|
-
ge 15
|
176
|
-
ta 15
|
177
|
-
nl 15
|
178
|
-
i;nd 15
|
140
|
+
ke 15
|
179
141
|
am 15
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
142
|
+
_ba 15
|
143
|
+
ge 15
|
144
|
+
im 15
|
145
|
+
az 15
|
146
|
+
inin 15
|
147
|
+
rl 14
|
148
|
+
di_ 14
|
149
|
+
esi 14
|
186
150
|
ko 14
|
151
|
+
_ve_ 14
|
187
152
|
erin 14
|
188
|
-
|
189
|
-
|
190
|
-
as& 14
|
153
|
+
ve_ 14
|
154
|
+
y� 14
|
191
155
|
_._ 14
|
192
|
-
|
193
|
-
|
194
|
-
rin 14
|
195
|
-
;n& 13
|
196
|
-
i;n& 13
|
197
|
-
_ve_ 13
|
198
|
-
lar&i 13
|
199
|
-
t&i; 13
|
200
|
-
;n&i; 13
|
201
|
-
rk 13
|
202
|
-
;n_ 13
|
203
|
-
_m 13
|
204
|
-
lar& 13
|
205
|
-
t& 13
|
206
|
-
t&i 13
|
207
|
-
y� 13
|
156
|
+
yi 14
|
157
|
+
._ 14
|
208
158
|
dan 13
|
209
|
-
|
210
|
-
sa 13
|
211
|
-
yo 13
|
212
|
-
_' 13
|
213
|
-
esi 13
|
214
|
-
a&s 13
|
159
|
+
_m 13
|
215
160
|
em 13
|
216
|
-
|
217
|
-
|
218
|
-
&i;n_ 13
|
219
|
-
i;n&i 13
|
161
|
+
lari 13
|
162
|
+
ne 13
|
220
163
|
ki_ 13
|
221
|
-
|
222
|
-
|
164
|
+
inin_ 13
|
165
|
+
rk 13
|
166
|
+
yo 13
|
167
|
+
isi 13
|
223
168
|
ap 13
|
224
|
-
ne 13
|
225
169
|
ld 13
|
226
|
-
|
227
|
-
ha 12
|
228
|
-
_ko 12
|
229
|
-
y&i 12
|
230
|
-
g;&i 12
|
231
|
-
lerin 12
|
232
|
-
g;&i; 12
|
233
|
-
y&i; 12
|
170
|
+
anl 12
|
234
171
|
_h 12
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
&g;& 12
|
239
|
-
im 12
|
240
|
-
g;& 12
|
241
|
-
lara 12
|
242
|
-
ar_ 12
|
172
|
+
s_ 12
|
173
|
+
_e 12
|
174
|
+
lerin 12
|
243
175
|
lm 12
|
244
|
-
to 12
|
245
176
|
_p 12
|
246
|
-
|
177
|
+
ha 12
|
178
|
+
gin 12
|
179
|
+
to 12
|
247
180
|
_bu 12
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
181
|
+
_ko 12
|
182
|
+
lara 12
|
183
|
+
ar_ 12
|
184
|
+
ur 12
|
185
|
+
ci 11
|
252
186
|
ek_ 11
|
253
|
-
|
254
|
-
;&s 11
|
255
|
-
_� 11
|
187
|
+
kan 11
|
256
188
|
ir_ 11
|
257
|
-
|
258
|
-
|
189
|
+
ik_ 11
|
190
|
+
_� 11
|
259
191
|
_ola 11
|
260
|
-
;&s; 11
|
261
|
-
kan 11
|
262
|
-
_e 11
|
263
|
-
i;&s; 11
|
264
|
-
;r 11
|
265
192
|
t� 11
|
266
|
-
|
267
|
-
|
268
|
-
|
193
|
+
ag 11
|
194
|
+
rak 11
|
195
|
+
bir 11
|
196
|
+
ri_ 11
|
197
|
+
arin 11
|
198
|
+
tl 11
|
269
199
|
bil 10
|
270
|
-
|
200
|
+
_ge 10
|
201
|
+
_si 10
|
271
202
|
af 10
|
272
|
-
|
273
|
-
_&s 10
|
274
|
-
ru 10
|
275
|
-
_� 10
|
276
|
-
nin 10
|
277
|
-
;m 10
|
203
|
+
ada 10
|
278
204
|
�k 10
|
279
|
-
|
205
|
+
sl 10
|
280
206
|
ce 10
|
281
|
-
|
282
|
-
|
207
|
+
sini 10
|
208
|
+
_sa 10
|
283
209
|
nde 10
|
284
|
-
|
210
|
+
dan_ 10
|
211
|
+
igi 10
|
212
|
+
_� 10
|
285
213
|
lan 10
|
286
|
-
|
287
|
-
|
288
|
-
|
214
|
+
ig 10
|
215
|
+
rin_ 10
|
216
|
+
ni_ 10
|
217
|
+
za 10
|
218
|
+
_bir 10
|
289
219
|
�r 10
|
290
|
-
|
291
|
-
|
292
|
-
|
220
|
+
st 10
|
221
|
+
ru 10
|
222
|
+
su 9
|
293
223
|
_bir_ 9
|
294
|
-
|
224
|
+
iler 9
|
225
|
+
us 9
|
226
|
+
nda_ 9
|
227
|
+
�i 9
|
295
228
|
ec 9
|
296
|
-
ak_ 9
|
297
|
-
ada 9
|
298
|
-
rd 9
|
299
|
-
_bir 9
|
300
|
-
bir_ 9
|
301
|
-
lk 9
|
302
229
|
kon 9
|
303
|
-
|
304
|
-
_sa 9
|
305
|
-
s&i;n 9
|
306
|
-
&s;l 9
|
307
|
-
;la 9
|
308
|
-
_il 9
|
309
|
-
ala 9
|
310
|
-
si_ 9
|
311
|
-
s;l 9
|
312
|
-
nda_ 9
|
313
|
-
ra_ 9
|
314
|
-
iler 9
|
230
|
+
inde 9
|
315
231
|
ul 9
|
316
232
|
olar 9
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
233
|
+
_de 9
|
234
|
+
sind 9
|
235
|
+
_il 9
|
236
|
+
larin 9
|
237
|
+
isin 9
|
238
|
+
_y� 9
|
239
|
+
bir_ 9
|
240
|
+
ak_ 9
|
241
|
+
ra_ 9
|
242
|
+
li_ 9
|
243
|
+
lk 9
|
244
|
+
rd 9
|
245
|
+
ala 9
|
246
|
+
unu 8
|
247
|
+
bas 8
|
321
248
|
�e 8
|
322
|
-
|
323
|
-
|
324
|
-
esi_ 8
|
325
|
-
s&i;_ 8
|
326
|
-
_f 8
|
327
|
-
bu_ 8
|
249
|
+
asi_ 8
|
250
|
+
zi 8
|
328
251
|
e� 8
|
329
|
-
|
330
|
-
|
252
|
+
rla 8
|
253
|
+
bu_ 8
|
254
|
+
_yil 8
|
331
255
|
eni 8
|
332
|
-
|
333
|
-
|
256
|
+
ild 8
|
257
|
+
on_ 8
|
258
|
+
nla 8
|
259
|
+
igin 8
|
260
|
+
rini 8
|
261
|
+
onu 8
|
262
|
+
it 8
|
263
|
+
_l 8
|
264
|
+
_di 8
|
265
|
+
_is 8
|
266
|
+
ece 8
|
267
|
+
lik 8
|
268
|
+
lma 8
|
269
|
+
ldi 8
|
270
|
+
ab 8
|
271
|
+
_ha 8
|
334
272
|
lu 8
|
335
|
-
|
336
|
-
|
273
|
+
id 8
|
274
|
+
ama 8
|
275
|
+
um 8
|
337
276
|
edi 8
|
338
|
-
|
277
|
+
_kon 8
|
339
278
|
eki 8
|
340
|
-
|
279
|
+
_yi 8
|
341
280
|
iye 8
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
;&i;n 8
|
348
|
-
onu 8
|
349
|
-
_y&i 8
|
350
|
-
um 8
|
351
|
-
_y& 8
|
281
|
+
esi_ 8
|
282
|
+
_te 8
|
283
|
+
yil 8
|
284
|
+
sm 8
|
285
|
+
_f 8
|
352
286
|
arak 8
|
353
|
-
|
354
|
-
|
355
|
-
|
287
|
+
_pa 8
|
288
|
+
ira 8
|
289
|
+
and 7
|
290
|
+
lar_ 7
|
291
|
+
ede 7
|
292
|
+
�_ 7
|
293
|
+
_ar 7
|
294
|
+
kt 7
|
295
|
+
bin 7
|
356
296
|
mesi 7
|
357
|
-
|
358
|
-
|
359
|
-
|
297
|
+
t�rk 7
|
298
|
+
erin_ 7
|
299
|
+
ilm 7
|
300
|
+
aka 7
|
301
|
+
dil 7
|
302
|
+
gr 7
|
303
|
+
sinin 7
|
304
|
+
ge� 7
|
305
|
+
ini_ 7
|
306
|
+
ili 7
|
360
307
|
m_ 7
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
;&g;& 7
|
367
|
-
lir 7
|
368
|
-
olara 7
|
369
|
-
;&g 7
|
370
|
-
i;&g; 7
|
371
|
-
;i&s; 7
|
308
|
+
kanl 7
|
309
|
+
ey 7
|
310
|
+
leri_ 7
|
311
|
+
_li 7
|
312
|
+
aza 7
|
372
313
|
t�r 7
|
373
|
-
ndan 7
|
374
|
-
ri_ 7
|
375
|
-
n&i;n 7
|
376
|
-
eri_ 7
|
377
|
-
_l 7
|
378
|
-
mes 7
|
379
|
-
and 7
|
380
314
|
_bin_ 7
|
315
|
+
is_ 7
|
316
|
+
olara 7
|
317
|
+
ist 7
|
318
|
+
fi 7
|
319
|
+
g� 7
|
320
|
+
lik_ 7
|
321
|
+
yan 7
|
322
|
+
lke 7
|
381
323
|
un_ 7
|
382
|
-
|
383
|
-
cu 7
|
384
|
-
&i;&g 7
|
385
|
-
_bin 7
|
386
|
-
d&i;& 7
|
387
|
-
dil 7
|
388
|
-
anla 7
|
324
|
+
�r 7
|
389
325
|
�rk 7
|
390
|
-
|
326
|
+
_da 7
|
327
|
+
eri_ 7
|
328
|
+
lir 7
|
329
|
+
sil 7
|
330
|
+
�n 7
|
331
|
+
mes 7
|
332
|
+
ndan 7
|
391
333
|
bin_ 7
|
392
|
-
|
334
|
+
par 7
|
335
|
+
er_ 7
|
393
336
|
_r 7
|
394
|
-
_li 7
|
395
|
-
_da 7
|
396
|
-
_de 7
|
397
337
|
_to 7
|
398
|
-
|
399
|
-
|
400
|
-
|
338
|
+
anla 7
|
339
|
+
_bin 7
|
340
|
+
aya 7
|
341
|
+
cu 7
|
342
|
+
min 6
|
343
|
+
yl 6
|
344
|
+
rad 6
|
345
|
+
larak 6
|
346
|
+
imi 6
|
347
|
+
usu 6
|
348
|
+
i� 6
|
349
|
+
�l 6
|
350
|
+
ze 6
|
351
|
+
z_ 6
|
352
|
+
ye_ 6
|
353
|
+
ildi 6
|
354
|
+
ndan_ 6
|
355
|
+
irl 6
|
356
|
+
ta_ 6
|
357
|
+
gini 6
|
358
|
+
ard 6
|
359
|
+
re_ 6
|
360
|
+
pl 6
|
361
|
+
�y 6
|
362
|
+
�z 6
|
363
|
+
ere 6
|
364
|
+
_t� 6
|
365
|
+
sk 6
|
366
|
+
lig 6
|
367
|
+
ng 6
|
368
|
+
lara_ 6
|
369
|
+
ileri 6
|
370
|
+
nli 6
|
371
|
+
azi 6
|
372
|
+
eki_ 6
|
373
|
+
ac 6
|
374
|
+
so 6
|
375
|
+
nlar 6
|
376
|
+
isl 6
|
377
|
+
asin 6
|
378
|
+
masi 6
|
379
|
+
sinda 6
|
380
|
+
y�k 6
|
381
|
+
mas 6
|
382
|
+
rs 6
|
383
|
+
ah 6
|
384
|
+
lam 6
|
385
|
+
etl 6
|
386
|
+
inda_ 6
|
387
|
+
la_ 6
|
388
|
+
_bas 6
|
389
|
+
ks 6
|
390
|
+
ele 6
|
391
|
+
ya_ 6
|
392
|
+
ligi 6
|
393
|
+
ti_ 6
|
394
|
+
zal 6
|
395
|
+
rinin 6
|
396
|
+
sis 6
|
397
|
+
aki 6
|
398
|
+
kl 6
|
399
|
+
ma_ 6
|
400
|
+
du 6
|