scylla 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -2
- data/Gemfile.lock +16 -1
- data/lib/scylla/classifier.rb +1 -1
- data/lib/scylla/generator.rb +16 -4
- data/lib/scylla/lms/afrikaans.lm +232 -232
- data/lib/scylla/lms/arabic.lm +175 -175
- data/lib/scylla/lms/bulgarian.lm +225 -225
- data/lib/scylla/lms/catalan.lm +309 -309
- data/lib/scylla/lms/danish.lm +167 -167
- data/lib/scylla/lms/english.lm +398 -398
- data/lib/scylla/lms/finnish.lm +237 -237
- data/lib/scylla/lms/french.lm +148 -148
- data/lib/scylla/lms/german.lm +258 -258
- data/lib/scylla/lms/greek.lm +236 -236
- data/lib/scylla/lms/hebrew.lm +154 -154
- data/lib/scylla/lms/hindi.lm +139 -139
- data/lib/scylla/lms/icelandic.lm +239 -239
- data/lib/scylla/lms/indonesian.lm +244 -244
- data/lib/scylla/lms/italian.lm +248 -248
- data/lib/scylla/lms/japanese.lm +90 -90
- data/lib/scylla/lms/korean.lm +306 -306
- data/lib/scylla/lms/norwegian.lm +193 -193
- data/lib/scylla/lms/polish.lm +241 -241
- data/lib/scylla/lms/portuguese.lm +232 -232
- data/lib/scylla/lms/romanian.lm +246 -246
- data/lib/scylla/lms/slovak.lm +242 -242
- data/lib/scylla/lms/slovenian.lm +229 -229
- data/lib/scylla/lms/spanish.lm +164 -164
- data/lib/scylla/lms/swedish.lm +157 -157
- data/lib/scylla/lms/tagalog.lm +247 -247
- data/lib/scylla/lms/thai.lm +252 -252
- data/lib/scylla/lms/turkish.lm +285 -285
- data/lib/scylla/lms/vietnamese.lm +250 -250
- data/lib/scylla/lms/welsh.lm +248 -248
- data/lib/scylla/resources.rb +1 -9
- data/lib/scylla.rb +4 -0
- data/scylla.gemspec +2 -120
- data/source_texts/english.txt +62 -27
- data/test/classifier_test.rb +1 -3
- data/test/fixtures/lms/danish.lm +173 -173
- data/test/fixtures/lms/english.lm +220 -220
- data/test/fixtures/lms/french.lm +175 -175
- data/test/fixtures/lms/german.lm +254 -254
- data/test/fixtures/lms/hindi.lm +139 -139
- data/test/fixtures/lms/italian.lm +236 -236
- data/test/fixtures/lms/japanese.lm +88 -88
- data/test/fixtures/lms/norwegian.lm +182 -182
- data/test/fixtures/lms/spanish.lm +164 -164
- data/test/fixtures/test_languages/spanish +0 -1
- data/test/generator_test.rb +13 -0
- data/test/helper.rb +2 -0
- metadata +18 -25
- data/.document +0 -5
- data/lib/scylla/lms/13375P33K.lm +0 -400
- data/scylla-0.1.0.gem +0 -0
- data/source_texts/13375P33K.txt +0 -199
- data/test/fixtures/lms/13375p33k.lm +0 -400
- data/test/fixtures/source_texts/13375P33K.txt +0 -199
data/lib/scylla/lms/french.lm
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
_
|
|
1
|
+
_ 32316
|
|
2
2
|
e 11820
|
|
3
3
|
s 6385
|
|
4
4
|
a 6245
|
|
@@ -7,184 +7,184 @@ n 6115
|
|
|
7
7
|
t 5635
|
|
8
8
|
r 5480
|
|
9
9
|
l 4711
|
|
10
|
-
e_
|
|
10
|
+
e_ 4340
|
|
11
11
|
u 4260
|
|
12
12
|
o 4196
|
|
13
13
|
� 3243
|
|
14
14
|
d 3178
|
|
15
|
-
s_
|
|
15
|
+
s_ 2979
|
|
16
16
|
c 2591
|
|
17
17
|
p 2396
|
|
18
|
-
_d
|
|
18
|
+
_d 2365
|
|
19
19
|
m 2162
|
|
20
20
|
es 2138
|
|
21
21
|
é 2115
|
|
22
22
|
� 2115
|
|
23
|
-
_l
|
|
24
|
-
t_
|
|
25
|
-
de
|
|
26
|
-
le
|
|
27
|
-
es_
|
|
28
|
-
en
|
|
23
|
+
_l 1883
|
|
24
|
+
t_ 1786
|
|
25
|
+
de 1586
|
|
26
|
+
le 1538
|
|
27
|
+
es_ 1526
|
|
28
|
+
en 1500
|
|
29
29
|
on 1429
|
|
30
|
-
_de
|
|
31
|
-
_e
|
|
30
|
+
_de 1340
|
|
31
|
+
_e 1316
|
|
32
32
|
nt 1258
|
|
33
33
|
an 1256
|
|
34
|
+
_p 1247
|
|
34
35
|
, 1245
|
|
35
|
-
|
|
36
|
-
,_ 1207
|
|
36
|
+
,_ 1209
|
|
37
37
|
re 1163
|
|
38
|
-
n_
|
|
38
|
+
n_ 1087
|
|
39
39
|
ti 1028
|
|
40
40
|
is 981
|
|
41
41
|
de_ 975
|
|
42
|
-
la
|
|
42
|
+
la 906
|
|
43
43
|
ra 900
|
|
44
44
|
_de_ 879
|
|
45
45
|
a_ 863
|
|
46
46
|
g 846
|
|
47
|
-
_s
|
|
47
|
+
_s 835
|
|
48
48
|
v 796
|
|
49
49
|
f 777
|
|
50
|
-
_c
|
|
51
|
-
_a
|
|
50
|
+
_c 773
|
|
51
|
+
_a 749
|
|
52
52
|
ai 735
|
|
53
53
|
te 723
|
|
54
54
|
ent 708
|
|
55
55
|
� 698
|
|
56
|
+
_le 693
|
|
56
57
|
� 688
|
|
57
58
|
� 687
|
|
58
59
|
q 683
|
|
59
|
-
|
|
60
|
-
qu 672
|
|
60
|
+
_la 679
|
|
61
61
|
ar 672
|
|
62
|
-
|
|
62
|
+
qu 672
|
|
63
|
+
le_ 665
|
|
64
|
+
nt_ 664
|
|
63
65
|
in 664
|
|
64
|
-
nt_ 663
|
|
65
66
|
et 662
|
|
66
|
-
_la 661
|
|
67
|
-
me 657
|
|
68
67
|
se 657
|
|
68
|
+
me 657
|
|
69
69
|
it 656
|
|
70
|
+
’ 652
|
|
70
71
|
� 652
|
|
71
72
|
�� 652
|
|
72
|
-
’ 652
|
|
73
73
|
ur 643
|
|
74
|
-
ce
|
|
74
|
+
ce 640
|
|
75
75
|
la_ 638
|
|
76
|
+
_la_ 637
|
|
76
77
|
ne 632
|
|
77
78
|
b 630
|
|
78
|
-
_la_ 625
|
|
79
79
|
er 620
|
|
80
|
-
ue
|
|
81
|
-
io 600
|
|
80
|
+
ue 616
|
|
82
81
|
ns 600
|
|
82
|
+
io 600
|
|
83
83
|
. 599
|
|
84
|
+
__ 594
|
|
84
85
|
u_ 577
|
|
85
86
|
ion 572
|
|
86
|
-
et_
|
|
87
|
+
et_ 556
|
|
88
|
+
_� 545
|
|
87
89
|
st 545
|
|
88
|
-
_� 544
|
|
89
90
|
r_ 541
|
|
91
|
+
_m 538
|
|
90
92
|
ie 538
|
|
91
|
-
_m 537
|
|
92
93
|
ri 535
|
|
93
94
|
pa 531
|
|
94
95
|
_et 529
|
|
95
96
|
ran 527
|
|
96
|
-
at 525
|
|
97
97
|
au 525
|
|
98
|
+
at 525
|
|
98
99
|
co 521
|
|
99
100
|
_et_ 520
|
|
100
101
|
nc 519
|
|
101
102
|
les 515
|
|
103
|
+
li 512
|
|
102
104
|
ou 510
|
|
103
|
-
li 508
|
|
104
105
|
tr 498
|
|
105
106
|
al 479
|
|
106
107
|
ta 468
|
|
107
108
|
ro 467
|
|
108
109
|
h 459
|
|
109
|
-
les_
|
|
110
|
-
que
|
|
110
|
+
les_ 457
|
|
111
|
+
que 455
|
|
111
112
|
tio 431
|
|
112
113
|
eu 429
|
|
113
114
|
tion 429
|
|
114
115
|
r� 428
|
|
115
116
|
_f 424
|
|
116
|
-
re_
|
|
117
|
+
re_ 423
|
|
117
118
|
on_ 420
|
|
118
119
|
em 413
|
|
120
|
+
_r 413
|
|
121
|
+
_en 410
|
|
119
122
|
x 410
|
|
120
|
-
|
|
121
|
-
_en 406
|
|
122
|
-
_pa 403
|
|
123
|
+
_pa 404
|
|
123
124
|
or 400
|
|
124
125
|
rt 400
|
|
125
126
|
po 397
|
|
126
127
|
ll 386
|
|
127
128
|
si 382
|
|
128
129
|
des 379
|
|
129
|
-
__ 379
|
|
130
130
|
oi 378
|
|
131
131
|
pr 375
|
|
132
132
|
anc 374
|
|
133
133
|
un 373
|
|
134
|
-
ent_
|
|
134
|
+
ent_ 372
|
|
135
135
|
en_ 368
|
|
136
136
|
nce 366
|
|
137
137
|
des_ 363
|
|
138
|
+
._ 363
|
|
138
139
|
_t 362
|
|
139
|
-
|
|
140
|
-
|
|
140
|
+
e,_ 362
|
|
141
|
+
e, 362
|
|
141
142
|
� 359
|
|
142
|
-
|
|
143
|
-
|
|
143
|
+
è 359
|
|
144
|
+
ne_ 357
|
|
144
145
|
par 352
|
|
145
|
-
|
|
146
|
-
|
|
146
|
+
ir 352
|
|
147
|
+
_le_ 347
|
|
147
148
|
_des 347
|
|
148
|
-
_co
|
|
149
|
-
ce_
|
|
150
|
-
|
|
151
|
-
du 339
|
|
149
|
+
_co 345
|
|
150
|
+
ce_ 344
|
|
151
|
+
du 343
|
|
152
152
|
_des_ 338
|
|
153
|
-
ns_
|
|
153
|
+
ns_ 338
|
|
154
154
|
el 330
|
|
155
|
-
l’ 328
|
|
156
|
-
l� 328
|
|
157
155
|
l� 328
|
|
158
|
-
_l�
|
|
159
|
-
_l
|
|
160
|
-
|
|
156
|
+
_l� 328
|
|
157
|
+
_l’ 328
|
|
158
|
+
l� 328
|
|
159
|
+
_l� 328
|
|
160
|
+
l’ 328
|
|
161
|
+
ui 327
|
|
161
162
|
us 325
|
|
162
|
-
ui 325
|
|
163
163
|
F 324
|
|
164
164
|
ré 322
|
|
165
165
|
nd 321
|
|
166
166
|
ati 321
|
|
167
167
|
ance 311
|
|
168
168
|
t� 311
|
|
169
|
-
n�
|
|
169
|
+
n� 308
|
|
170
170
|
ve 307
|
|
171
|
+
_F 307
|
|
171
172
|
ion_ 306
|
|
172
|
-
|
|
173
|
-
L 305
|
|
173
|
+
_en_ 306
|
|
174
174
|
om 305
|
|
175
|
-
|
|
175
|
+
L 305
|
|
176
176
|
so 304
|
|
177
|
+
_les 304
|
|
177
178
|
is_ 302
|
|
178
|
-
|
|
179
|
+
ue_ 299
|
|
179
180
|
é_ 299
|
|
180
181
|
�_ 299
|
|
181
|
-
ue_ 298
|
|
182
182
|
iq 296
|
|
183
|
+
_les_ 296
|
|
183
184
|
iqu 296
|
|
184
|
-
�e 294
|
|
185
185
|
ée 294
|
|
186
|
-
|
|
187
|
-
|
|
186
|
+
_par 294
|
|
187
|
+
�e 294
|
|
188
188
|
ma 293
|
|
189
189
|
men 291
|
|
190
190
|
à 288
|
|
@@ -193,208 +193,208 @@ Fr 287
|
|
|
193
193
|
ique 286
|
|
194
194
|
il 284
|
|
195
195
|
Fra 283
|
|
196
|
+
_Fr 283
|
|
196
197
|
est 283
|
|
197
|
-
_Fr 282
|
|
198
|
-
à_ 280
|
|
199
198
|
i_ 280
|
|
200
|
-
Fran 280
|
|
201
199
|
�_ 280
|
|
200
|
+
Fran 280
|
|
201
|
+
à_ 280
|
|
202
|
+
_Fra 279
|
|
202
203
|
mi 279
|
|
203
|
-
_Fra 278
|
|
204
204
|
pl 278
|
|
205
|
+
_Fran 276
|
|
205
206
|
té 276
|
|
206
|
-
|
|
207
|
+
_. 275
|
|
207
208
|
_u 274
|
|
208
209
|
_à 273
|
|
209
|
-
_à_ 272
|
|
210
210
|
ranc 272
|
|
211
|
+
_à_ 272
|
|
212
|
+
di 271
|
|
211
213
|
ment 270
|
|
212
|
-
|
|
214
|
+
na 270
|
|
213
215
|
ut 269
|
|
214
|
-
na 269
|
|
215
216
|
ss 268
|
|
216
|
-
|
|
217
|
+
_du 268
|
|
217
218
|
ci 267
|
|
218
|
-
|
|
219
|
+
_, 266
|
|
219
220
|
�s 263
|
|
220
221
|
és 263
|
|
222
|
+
_au 262
|
|
221
223
|
ais 262
|
|
222
224
|
du_ 259
|
|
223
225
|
ation 258
|
|
224
226
|
atio 258
|
|
225
|
-
|
|
227
|
+
_n 258
|
|
228
|
+
_pr 258
|
|
226
229
|
_un 257
|
|
227
|
-
|
|
230
|
+
lu 257
|
|
228
231
|
_du_ 256
|
|
229
232
|
y 255
|
|
230
|
-
lu 255
|
|
231
233
|
nce_ 255
|
|
232
|
-
|
|
233
|
-
ol 252
|
|
234
|
+
que_ 253
|
|
234
235
|
Franc 252
|
|
235
|
-
|
|
236
|
+
ol 252
|
|
236
237
|
rs 251
|
|
237
238
|
rance 251
|
|
238
239
|
tion_ 250
|
|
239
240
|
lle 250
|
|
240
241
|
pe 248
|
|
242
|
+
s, 246
|
|
241
243
|
con 244
|
|
242
244
|
te_ 243
|
|
243
|
-
s,
|
|
244
|
-
|
|
245
|
+
s,_ 243
|
|
246
|
+
d� 242
|
|
245
247
|
ont 240
|
|
248
|
+
_so 240
|
|
246
249
|
ire 240
|
|
247
|
-
d� 239
|
|
248
250
|
res 239
|
|
251
|
+
_,_ 239
|
|
249
252
|
no 239
|
|
250
|
-
s,_ 238
|
|
251
|
-
_so 238
|
|
252
|
-
ic 238
|
|
253
253
|
ons 238
|
|
254
|
+
ic 238
|
|
254
255
|
mo 236
|
|
256
|
+
dé 233
|
|
255
257
|
- 232
|
|
256
|
-
dé 232
|
|
257
258
|
i� 231
|
|
258
259
|
eur 230
|
|
259
260
|
ance_ 228
|
|
260
261
|
nn 227
|
|
261
|
-
_qu 226
|
|
262
262
|
ant 226
|
|
263
|
+
_qu 226
|
|
263
264
|
_q 226
|
|
264
265
|
ct 223
|
|
265
|
-
st_ 219
|
|
266
266
|
est_ 219
|
|
267
|
-
|
|
267
|
+
st_ 219
|
|
268
268
|
ni 216
|
|
269
|
+
lo 215
|
|
269
270
|
ux 215
|
|
270
271
|
_po 213
|
|
271
|
-
lo 213
|
|
272
272
|
ch 213
|
|
273
273
|
vi 212
|
|
274
274
|
me_ 210
|
|
275
|
-
_é
|
|
275
|
+
_é 209
|
|
276
276
|
eme 208
|
|
277
|
-
_L
|
|
278
|
-
|
|
277
|
+
_L 207
|
|
278
|
+
sa 205
|
|
279
279
|
ement 205
|
|
280
|
-
|
|
280
|
+
emen 205
|
|
281
|
+
_d� 203
|
|
282
|
+
se_ 203
|
|
281
283
|
_es 202
|
|
284
|
+
�t 202
|
|
282
285
|
ec 202
|
|
283
286
|
pu 202
|
|
284
287
|
ét 202
|
|
285
|
-
�t 202
|
|
286
|
-
se_ 201
|
|
287
|
-
d� 201
|
|
288
|
-
d� 201
|
|
289
288
|
d’ 201
|
|
290
|
-
|
|
289
|
+
d� 201
|
|
290
|
+
d� 201
|
|
291
|
+
ale 196
|
|
291
292
|
ur_ 196
|
|
293
|
+
_dé 196
|
|
292
294
|
to 196
|
|
293
|
-
|
|
294
|
-
_dé 195
|
|
295
|
-
_d’ 193
|
|
295
|
+
da 194
|
|
296
296
|
_d� 193
|
|
297
297
|
_o 193
|
|
298
298
|
_d� 193
|
|
299
|
+
_d’ 193
|
|
299
300
|
_est 192
|
|
300
|
-
tre 191
|
|
301
301
|
A 191
|
|
302
302
|
op 191
|
|
303
|
-
|
|
304
|
-
da 190
|
|
303
|
+
tre 191
|
|
305
304
|
au_ 190
|
|
305
|
+
_pl 190
|
|
306
306
|
ul 189
|
|
307
307
|
_est_ 187
|
|
308
|
+
x_ 186
|
|
309
|
+
ment_ 186
|
|
310
|
+
_re 185
|
|
308
311
|
fr 185
|
|
309
|
-
ment_ 185
|
|
310
|
-
_re 184
|
|
311
|
-
x_ 184
|
|
312
|
-
bl 184
|
|
313
312
|
_i 184
|
|
313
|
+
_._ 184
|
|
314
314
|
mp 184
|
|
315
|
-
|
|
316
|
-
ts 180
|
|
315
|
+
bl 184
|
|
317
316
|
ac 180
|
|
317
|
+
ér 180
|
|
318
|
+
ts 180
|
|
318
319
|
�r 180
|
|
319
320
|
tu 180
|
|
320
|
-
ér 180
|
|
321
321
|
rti 179
|
|
322
|
-
_se
|
|
322
|
+
_se 177
|
|
323
323
|
ise 174
|
|
324
324
|
art 173
|
|
325
325
|
iè 172
|
|
326
326
|
ans 171
|
|
327
|
-
ç 170
|
|
328
327
|
� 170
|
|
328
|
+
ç 170
|
|
329
329
|
ia 170
|
|
330
|
+
l_ 169
|
|
330
331
|
_mo 169
|
|
332
|
+
ux_ 169
|
|
331
333
|
ar_ 168
|
|
332
|
-
ux_ 168
|
|
333
|
-
an� 167
|
|
334
334
|
_con 167
|
|
335
|
-
l_ 167
|
|
336
335
|
gr 167
|
|
337
|
-
|
|
336
|
+
an� 167
|
|
337
|
+
lus 167
|
|
338
338
|
ran� 166
|
|
339
|
-
|
|
339
|
+
P 166
|
|
340
340
|
nç 165
|
|
341
|
-
im 164
|
|
342
341
|
ont_ 164
|
|
343
|
-
|
|
342
|
+
im 164
|
|
344
343
|
_fr 163
|
|
344
|
+
une 163
|
|
345
|
+
_g 162
|
|
345
346
|
son 162
|
|
346
347
|
rs_ 162
|
|
347
348
|
un_ 161
|
|
348
349
|
anç 161
|
|
349
|
-
_g 161
|
|
350
|
-
ranç 160
|
|
351
350
|
C 160
|
|
352
|
-
|
|
351
|
+
_v 160
|
|
353
352
|
us_ 160
|
|
354
|
-
|
|
353
|
+
su 160
|
|
354
|
+
ranç 160
|
|
355
355
|
ill 159
|
|
356
356
|
cl 158
|
|
357
357
|
plu 158
|
|
358
358
|
par_ 158
|
|
359
359
|
_plu 158
|
|
360
360
|
as 157
|
|
361
|
+
_par_ 157
|
|
361
362
|
nça 156
|
|
362
363
|
�a 156
|
|
363
364
|
ça 156
|
|
364
|
-
|
|
365
|
-
|
|
365
|
+
ique_ 154
|
|
366
|
+
_au_ 154
|
|
367
|
+
l� 154
|
|
366
368
|
mm 153
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
plus 152
|
|
369
|
+
ança 153
|
|
370
|
+
_plus 152
|
|
370
371
|
nçai 152
|
|
371
372
|
çai 152
|
|
373
|
+
plus 152
|
|
372
374
|
�ai 152
|
|
373
375
|
çais 152
|
|
374
|
-
|
|
376
|
+
�ais 152
|
|
375
377
|
rr 152
|
|
376
378
|
fra 151
|
|
377
379
|
ge 150
|
|
378
|
-
_au_ 150
|
|
379
|
-
l� 149
|
|
380
380
|
m� 149
|
|
381
381
|
une_ 148
|
|
382
382
|
ag 147
|
|
383
|
-
ell 146
|
|
384
383
|
ions 146
|
|
384
|
+
ell 146
|
|
385
385
|
fi 146
|
|
386
|
+
ie_ 145
|
|
386
387
|
iv 145
|
|
387
|
-
dan
|
|
388
|
+
dan 145
|
|
389
|
+
ans_ 144
|
|
388
390
|
ien 144
|
|
389
|
-
ie_ 144
|
|
390
391
|
_fra 143
|
|
391
|
-
our 143
|
|
392
392
|
ain 143
|
|
393
393
|
té_ 143
|
|
394
|
-
|
|
395
|
-
elle 142
|
|
396
|
-
fran 142
|
|
394
|
+
our 143
|
|
397
395
|
ep 142
|
|
396
|
+
fran 142
|
|
397
|
+
elle 142
|
|
398
398
|
_fran 142
|
|
399
|
-
|
|
400
|
-
|
|
399
|
+
éc 141
|
|
400
|
+
�c 141
|