scylla 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -2
- data/Gemfile.lock +16 -1
- data/lib/scylla/classifier.rb +1 -1
- data/lib/scylla/generator.rb +16 -4
- data/lib/scylla/lms/afrikaans.lm +232 -232
- data/lib/scylla/lms/arabic.lm +175 -175
- data/lib/scylla/lms/bulgarian.lm +225 -225
- data/lib/scylla/lms/catalan.lm +309 -309
- data/lib/scylla/lms/danish.lm +167 -167
- data/lib/scylla/lms/english.lm +398 -398
- data/lib/scylla/lms/finnish.lm +237 -237
- data/lib/scylla/lms/french.lm +148 -148
- data/lib/scylla/lms/german.lm +258 -258
- data/lib/scylla/lms/greek.lm +236 -236
- data/lib/scylla/lms/hebrew.lm +154 -154
- data/lib/scylla/lms/hindi.lm +139 -139
- data/lib/scylla/lms/icelandic.lm +239 -239
- data/lib/scylla/lms/indonesian.lm +244 -244
- data/lib/scylla/lms/italian.lm +248 -248
- data/lib/scylla/lms/japanese.lm +90 -90
- data/lib/scylla/lms/korean.lm +306 -306
- data/lib/scylla/lms/norwegian.lm +193 -193
- data/lib/scylla/lms/polish.lm +241 -241
- data/lib/scylla/lms/portuguese.lm +232 -232
- data/lib/scylla/lms/romanian.lm +246 -246
- data/lib/scylla/lms/slovak.lm +242 -242
- data/lib/scylla/lms/slovenian.lm +229 -229
- data/lib/scylla/lms/spanish.lm +164 -164
- data/lib/scylla/lms/swedish.lm +157 -157
- data/lib/scylla/lms/tagalog.lm +247 -247
- data/lib/scylla/lms/thai.lm +252 -252
- data/lib/scylla/lms/turkish.lm +285 -285
- data/lib/scylla/lms/vietnamese.lm +250 -250
- data/lib/scylla/lms/welsh.lm +248 -248
- data/lib/scylla/resources.rb +1 -9
- data/lib/scylla.rb +4 -0
- data/scylla.gemspec +2 -120
- data/source_texts/english.txt +62 -27
- data/test/classifier_test.rb +1 -3
- data/test/fixtures/lms/danish.lm +173 -173
- data/test/fixtures/lms/english.lm +220 -220
- data/test/fixtures/lms/french.lm +175 -175
- data/test/fixtures/lms/german.lm +254 -254
- data/test/fixtures/lms/hindi.lm +139 -139
- data/test/fixtures/lms/italian.lm +236 -236
- data/test/fixtures/lms/japanese.lm +88 -88
- data/test/fixtures/lms/norwegian.lm +182 -182
- data/test/fixtures/lms/spanish.lm +164 -164
- data/test/fixtures/test_languages/spanish +0 -1
- data/test/generator_test.rb +13 -0
- data/test/helper.rb +2 -0
- metadata +18 -25
- data/.document +0 -5
- data/lib/scylla/lms/13375P33K.lm +0 -400
- data/scylla-0.1.0.gem +0 -0
- data/source_texts/13375P33K.txt +0 -199
- data/test/fixtures/lms/13375p33k.lm +0 -400
- data/test/fixtures/source_texts/13375P33K.txt +0 -199
data/test/fixtures/lms/german.lm
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
_
|
1
|
+
_ 1208
|
2
2
|
e 644
|
3
3
|
n 386
|
4
4
|
i 318
|
@@ -9,392 +9,392 @@ a 197
|
|
9
9
|
h 190
|
10
10
|
d 187
|
11
11
|
u 161
|
12
|
-
en 137
|
13
12
|
er 137
|
13
|
+
en 137
|
14
14
|
l 131
|
15
15
|
c 125
|
16
16
|
o 116
|
17
17
|
ch 112
|
18
18
|
g 105
|
19
19
|
m 101
|
20
|
-
e_
|
21
|
-
n_
|
20
|
+
e_ 99
|
21
|
+
n_ 92
|
22
22
|
te 80
|
23
|
+
_d 78
|
23
24
|
in 77
|
24
|
-
_d 76
|
25
25
|
de 76
|
26
|
+
r_ 75
|
26
27
|
ei 73
|
27
|
-
r_ 72
|
28
28
|
k 68
|
29
29
|
b 66
|
30
30
|
f 58
|
31
|
+
er_ 56
|
31
32
|
un 56
|
32
33
|
ie 54
|
33
|
-
er_ 53
|
34
34
|
nd 52
|
35
|
+
en_ 52
|
35
36
|
w 50
|
36
|
-
|
37
|
-
_e 48
|
37
|
+
_e 49
|
38
38
|
ge 48
|
39
|
-
__
|
39
|
+
__ 48
|
40
40
|
ng 45
|
41
41
|
t_ 44
|
42
42
|
, 42
|
43
43
|
der 41
|
44
|
-
ne 39
|
45
44
|
st 39
|
46
|
-
|
47
|
-
|
48
|
-
z 36
|
45
|
+
ne 39
|
46
|
+
_s 37
|
49
47
|
he 36
|
50
|
-
|
48
|
+
z 36
|
49
|
+
,_ 36
|
50
|
+
. 36
|
51
51
|
re 34
|
52
52
|
s_ 34
|
53
|
-
m_ 34
|
54
53
|
di 34
|
55
|
-
|
56
|
-
|
54
|
+
m_ 34
|
55
|
+
es 34
|
57
56
|
v 33
|
58
|
-
|
57
|
+
it 33
|
58
|
+
_de 33
|
59
59
|
ie_ 32
|
60
60
|
_i 32
|
61
|
-
be 31
|
62
61
|
au 31
|
62
|
+
be 31
|
63
63
|
ein 31
|
64
64
|
S 31
|
65
65
|
ti 30
|
66
|
-
|
66
|
+
d_ 29
|
67
67
|
� 29
|
68
|
-
_a 29
|
69
68
|
ic 29
|
69
|
+
_a 29
|
70
70
|
_w 28
|
71
71
|
� 28
|
72
|
-
d_ 28
|
73
72
|
le 28
|
74
|
-
sch 27
|
75
73
|
sc 27
|
74
|
+
sch 27
|
76
75
|
nt 26
|
77
76
|
ung 26
|
77
|
+
is 25
|
78
78
|
si 25
|
79
|
-
ich 25
|
80
79
|
die 25
|
81
|
-
is 25
|
82
80
|
an 25
|
83
|
-
|
84
|
-
che 24
|
81
|
+
ich 25
|
85
82
|
or 24
|
83
|
+
h_ 24
|
84
|
+
p 24
|
86
85
|
el 24
|
86
|
+
der_ 24
|
87
|
+
che 24
|
88
|
+
on 23
|
87
89
|
al 23
|
88
|
-
der_ 23
|
89
|
-
_die 23
|
90
90
|
_di 23
|
91
|
+
_die 23
|
91
92
|
nd_ 23
|
92
|
-
|
93
|
-
h_ 22
|
93
|
+
_ei 22
|
94
94
|
rt 22
|
95
|
-
|
96
|
-
|
97
|
-
|
95
|
+
_ein 22
|
96
|
+
ch_ 22
|
97
|
+
ht 21
|
98
|
+
em 21
|
98
99
|
ten 21
|
99
|
-
ra 21
|
100
100
|
und 21
|
101
|
-
|
102
|
-
ht 21
|
103
|
-
_ein 21
|
101
|
+
ra 21
|
104
102
|
die_ 21
|
105
|
-
|
103
|
+
_S 21
|
104
|
+
li 21
|
106
105
|
at 20
|
107
|
-
hr 20
|
108
106
|
ke 20
|
107
|
+
hr 20
|
109
108
|
in_ 20
|
110
|
-
ur 19
|
111
109
|
_die_ 19
|
110
|
+
_der 19
|
111
|
+
me 19
|
112
|
+
ur 19
|
112
113
|
A 19
|
113
114
|
_v 19
|
114
|
-
|
115
|
+
se 18
|
116
|
+
_in 18
|
115
117
|
ns 18
|
116
118
|
ll 18
|
117
|
-
_der 18
|
118
119
|
ine 18
|
119
|
-
|
120
|
-
se 18
|
120
|
+
im 17
|
121
121
|
ter 17
|
122
|
-
|
122
|
+
eine 17
|
123
123
|
nge 17
|
124
|
+
ar 17
|
125
|
+
._ 17
|
124
126
|
ri 17
|
125
|
-
eine 17
|
126
127
|
eit 17
|
127
|
-
|
128
|
-
ar 17
|
129
|
-
den 16
|
130
|
-
gen 16
|
131
|
-
D 16
|
128
|
+
L 17
|
132
129
|
da 16
|
133
130
|
we 16
|
134
|
-
|
131
|
+
gen 16
|
132
|
+
_h 16
|
133
|
+
D 16
|
134
|
+
den 16
|
135
135
|
ig 15
|
136
|
-
eu 15
|
137
|
-
K 15
|
138
|
-
cht 15
|
139
|
-
._ 15
|
140
136
|
zu 15
|
137
|
+
K 15
|
141
138
|
_er 15
|
142
139
|
F 15
|
140
|
+
te_ 15
|
141
|
+
eu 15
|
142
|
+
cht 15
|
143
143
|
_au 14
|
144
|
-
|
145
|
-
und_ 14
|
146
|
-
te_ 14
|
147
|
-
mm 14
|
148
|
-
rs 14
|
144
|
+
ha 14
|
149
145
|
E 14
|
150
|
-
|
151
|
-
|
146
|
+
und_ 14
|
147
|
+
us 14
|
152
148
|
vo 14
|
153
|
-
ha 14
|
154
|
-
_A 14
|
155
149
|
nde 14
|
156
|
-
|
157
|
-
|
158
|
-
|
150
|
+
ach 14
|
151
|
+
_A 14
|
152
|
+
ac 14
|
153
|
+
mm 14
|
154
|
+
eh 14
|
155
|
+
rs 14
|
156
|
+
_da 13
|
157
|
+
B 13
|
158
|
+
fe 13
|
159
|
+
ru 13
|
159
160
|
_si 13
|
161
|
+
�r 13
|
162
|
+
� 13
|
160
163
|
ls 13
|
161
|
-
fe 13
|
162
|
-
ab 13
|
163
|
-
n. 13
|
164
164
|
� 13
|
165
|
-
|
166
|
-
|
165
|
+
n. 13
|
166
|
+
ab 13
|
167
|
+
_L 13
|
168
|
+
lle 12
|
169
|
+
_der_ 12
|
170
|
+
as 12
|
167
171
|
et 12
|
172
|
+
rd 12
|
173
|
+
rn 12
|
174
|
+
am 12
|
168
175
|
lic 12
|
176
|
+
g_ 12
|
177
|
+
rk 12
|
169
178
|
em_ 12
|
179
|
+
_z 12
|
180
|
+
R 12
|
181
|
+
_b 12
|
182
|
+
_D 12
|
170
183
|
wi 12
|
171
|
-
|
172
|
-
_da 12
|
173
|
-
am 12
|
184
|
+
_eine 12
|
174
185
|
ste 12
|
175
|
-
rn 12
|
176
|
-
lle 12
|
177
|
-
as 12
|
178
|
-
R 12
|
179
|
-
_z 12
|
180
|
-
rk 12
|
181
|
-
_u 12
|
182
|
-
rd 12
|
183
186
|
T 12
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
rte 11
|
189
|
-
g_ 11
|
190
|
-
ge_ 11
|
191
|
-
- 11
|
192
|
-
_eine 11
|
193
|
-
hl 11
|
194
|
-
Sc 11
|
195
|
-
ck 11
|
196
|
-
ts 11
|
187
|
+
nte 12
|
188
|
+
_B 11
|
189
|
+
_g 11
|
190
|
+
on_ 11
|
197
191
|
lich 11
|
198
|
-
ni 11
|
199
|
-
ers 11
|
200
|
-
_D 11
|
201
|
-
ve 11
|
202
|
-
Sch 11
|
203
|
-
_n 11
|
204
192
|
G 11
|
193
|
+
men 11
|
194
|
+
ers 11
|
195
|
+
U 11
|
205
196
|
ent 11
|
206
|
-
|
207
|
-
|
208
|
-
tt 11
|
197
|
+
Sch 11
|
198
|
+
ni 11
|
209
199
|
_in_ 11
|
200
|
+
uf 11
|
201
|
+
Sc 11
|
202
|
+
ge_ 11
|
210
203
|
f� 11
|
211
|
-
|
204
|
+
rte 11
|
212
205
|
ren 11
|
206
|
+
ve 11
|
213
207
|
nn 11
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
208
|
+
_u 11
|
209
|
+
tr 11
|
210
|
+
tt 11
|
211
|
+
ck 11
|
212
|
+
n, 11
|
213
|
+
ts 11
|
214
|
+
hl 11
|
215
|
+
_n 11
|
216
|
+
_F 10
|
217
|
+
den_ 10
|
218
|
+
ma 10
|
219
|
+
en. 10
|
220
|
+
_K 10
|
221
221
|
hi 10
|
222
|
+
gs 10
|
223
|
+
als 10
|
224
|
+
ut 10
|
225
|
+
nder 10
|
226
|
+
ten_ 10
|
227
|
+
e, 10
|
228
|
+
run 10
|
229
|
+
ht_ 10
|
222
230
|
sin 10
|
223
|
-
|
224
|
-
|
231
|
+
rc 10
|
232
|
+
isch 10
|
233
|
+
na 10
|
234
|
+
f�r 10
|
225
235
|
isc 10
|
236
|
+
�r_ 10
|
237
|
+
_E 10
|
226
238
|
um 10
|
227
|
-
_" 10
|
228
|
-
ut 10
|
229
|
-
isch 10
|
230
239
|
auf 10
|
231
|
-
_K 10
|
232
|
-
gs 10
|
233
|
-
n, 10
|
234
|
-
en. 10
|
235
|
-
_F 10
|
236
240
|
_vo 10
|
237
|
-
als 10
|
238
241
|
sa 10
|
239
242
|
_zu 10
|
240
|
-
nen 9
|
241
|
-
ungs 9
|
242
|
-
_be 9
|
243
|
-
_E 9
|
244
|
-
chen 9
|
245
|
-
hen 9
|
246
|
-
_Sch 9
|
247
|
-
den_ 9
|
248
|
-
it_ 9
|
249
|
-
_Sc 9
|
250
|
-
hre 9
|
251
|
-
O 9
|
252
|
-
ma 9
|
253
243
|
_T 9
|
254
|
-
|
255
|
-
ere 9
|
256
|
-
f�r_ 9
|
244
|
+
ng_ 9
|
257
245
|
k_ 9
|
258
|
-
rch 9
|
259
246
|
lt 9
|
260
|
-
|
261
|
-
|
262
|
-
e, 9
|
263
|
-
ten_ 9
|
247
|
+
om 9
|
248
|
+
chen 9
|
264
249
|
ik 9
|
265
|
-
|
250
|
+
nen 9
|
251
|
+
ere 9
|
266
252
|
erk 9
|
267
|
-
Un 9
|
268
|
-
no 9
|
269
253
|
M 9
|
254
|
+
ungs 9
|
255
|
+
ngs 9
|
256
|
+
ver 9
|
257
|
+
f�r_ 9
|
258
|
+
hre 9
|
270
259
|
so 9
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
260
|
+
no 9
|
261
|
+
_Sch 9
|
262
|
+
O 9
|
263
|
+
ze 9
|
264
|
+
rch 9
|
265
|
+
_be 9
|
266
|
+
il 9
|
267
|
+
_Sc 9
|
268
|
+
hen 9
|
269
|
+
Un 9
|
270
|
+
n,_ 9
|
271
|
+
it_ 9
|
272
|
+
e,_ 9
|
275
273
|
le_ 8
|
276
|
-
sp 8
|
277
|
-
_k 8
|
278
|
-
od 8
|
279
|
-
n,_ 8
|
280
|
-
urc 8
|
281
274
|
_ha 8
|
275
|
+
es_ 8
|
276
|
+
_f 8
|
277
|
+
sp 8
|
278
|
+
_das 8
|
282
279
|
uc 8
|
283
|
-
run 8
|
284
|
-
tra 8
|
285
|
-
ber 8
|
286
|
-
end 8
|
287
|
-
ir 8
|
288
|
-
du 8
|
289
|
-
ind 8
|
290
|
-
ng_ 8
|
291
280
|
bei 8
|
292
|
-
|
281
|
+
_G 8
|
293
282
|
sic 8
|
283
|
+
_Le 8
|
294
284
|
_Un 8
|
295
|
-
|
296
|
-
tig 8
|
297
|
-
"_ 8
|
298
|
-
es_ 8
|
299
|
-
e,_ 8
|
285
|
+
das 8
|
300
286
|
ute 8
|
301
|
-
ft 8
|
302
|
-
mme 8
|
303
287
|
wo 8
|
304
|
-
|
288
|
+
la 8
|
289
|
+
mme 8
|
290
|
+
ol 8
|
305
291
|
kei 8
|
306
|
-
|
292
|
+
od 8
|
293
|
+
ir 8
|
294
|
+
V 8
|
307
295
|
Le 8
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
296
|
+
du 8
|
297
|
+
sich 8
|
298
|
+
end 8
|
299
|
+
ind 8
|
312
300
|
rde 8
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
301
|
+
ens 8
|
302
|
+
urc 8
|
303
|
+
ner 8
|
304
|
+
ber 8
|
305
|
+
tig 8
|
306
|
+
ft 8
|
307
|
+
ls_ 8
|
308
|
+
tra 8
|
309
|
+
_k 8
|
310
|
+
_U 8
|
323
311
|
ih 7
|
324
|
-
|
312
|
+
f_ 7
|
313
|
+
ein_ 7
|
314
|
+
ba 7
|
315
|
+
durch 7
|
316
|
+
_al 7
|
317
|
+
_un 7
|
318
|
+
en, 7
|
319
|
+
_wi 7
|
320
|
+
des 7
|
325
321
|
ho 7
|
322
|
+
and 7
|
323
|
+
das_ 7
|
324
|
+
nh 7
|
325
|
+
unge 7
|
326
|
+
io 7
|
327
|
+
_o 7
|
328
|
+
l_ 7
|
329
|
+
H 7
|
326
330
|
ko 7
|
327
|
-
eb 7
|
328
|
-
des 7
|
329
|
-
ne_ 7
|
330
|
-
y 7
|
331
331
|
eut 7
|
332
|
-
|
333
|
-
|
334
|
-
|
332
|
+
durc 7
|
333
|
+
icht 7
|
334
|
+
_das_ 7
|
335
|
+
keit 7
|
336
|
+
hat 7
|
337
|
+
eute 7
|
338
|
+
_ge 7
|
339
|
+
P 7
|
335
340
|
dur 7
|
336
|
-
|
341
|
+
as_ 7
|
342
|
+
n._ 7
|
343
|
+
r� 7
|
344
|
+
rei 7
|
345
|
+
_ve 7
|
337
346
|
ert 7
|
338
|
-
|
339
|
-
von 7
|
340
|
-
_o 7
|
341
|
-
_al 7
|
347
|
+
y 7
|
342
348
|
urch 7
|
343
|
-
|
349
|
+
do 7
|
350
|
+
uch 7
|
344
351
|
ich_ 7
|
352
|
+
sti 7
|
353
|
+
von 7
|
345
354
|
tte 7
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
_das 7
|
352
|
-
ba 7
|
353
|
-
durch 7
|
354
|
-
ode 7
|
355
|
-
ing 7
|
355
|
+
St 7
|
356
|
+
ngen 7
|
357
|
+
e. 7
|
358
|
+
im_ 7
|
359
|
+
mi 7
|
356
360
|
_ver 7
|
357
361
|
_Leu 7
|
358
|
-
|
359
|
-
als_ 7
|
360
|
-
and 7
|
361
|
-
unge 7
|
362
|
-
P 7
|
363
|
-
f_ 7
|
362
|
+
ne_ 7
|
364
363
|
len 7
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
364
|
+
ode 7
|
365
|
+
Leu 7
|
366
|
+
als_ 7
|
367
|
+
ing 7
|
368
|
+
eb 7
|
369
|
+
ische 6
|
370
|
+
_St 6
|
371
|
+
bi 6
|
372
|
+
hw 6
|
373
|
+
hu 6
|
374
|
+
von_ 6
|
375
|
+
sind 6
|
376
|
+
nge_ 6
|
372
377
|
ren_ 6
|
373
|
-
|
374
|
-
|
375
|
-
|
378
|
+
mmen 6
|
379
|
+
nter 6
|
380
|
+
_sich 6
|
381
|
+
Z 6
|
382
|
+
rst 6
|
383
|
+
he_ 6
|
384
|
+
ter_ 6
|
385
|
+
_f� 6
|
386
|
+
_we 6
|
387
|
+
ute_ 6
|
388
|
+
_hat 6
|
389
|
+
vor 6
|
390
|
+
ung_ 6
|
391
|
+
th 6
|
392
|
+
eru 6
|
376
393
|
_ih 6
|
377
|
-
nge_ 6
|
378
394
|
gu 6
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
chl 6
|
383
|
-
dor 6
|
395
|
+
t. 6
|
396
|
+
_im 6
|
397
|
+
x 6
|
384
398
|
N 6
|
385
|
-
|
386
|
-
|
387
|
-
ue 6
|
388
|
-
lte 6
|
389
|
-
auc 6
|
390
|
-
sk 6
|
391
|
-
he_ 6
|
392
|
-
_we 6
|
393
|
-
_sich 6
|
394
|
-
l_ 6
|
395
|
-
_f�r 6
|
396
|
-
u_ 6
|
397
|
-
ene 6
|
398
|
-
ed 6
|
399
|
-
imm 6
|
400
|
-
e. 6
|
399
|
+
ben 6
|
400
|
+
_ers 6
|