scylla 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -2
- data/Gemfile.lock +16 -1
- data/lib/scylla/classifier.rb +1 -1
- data/lib/scylla/generator.rb +16 -4
- data/lib/scylla/lms/afrikaans.lm +232 -232
- data/lib/scylla/lms/arabic.lm +175 -175
- data/lib/scylla/lms/bulgarian.lm +225 -225
- data/lib/scylla/lms/catalan.lm +309 -309
- data/lib/scylla/lms/danish.lm +167 -167
- data/lib/scylla/lms/english.lm +398 -398
- data/lib/scylla/lms/finnish.lm +237 -237
- data/lib/scylla/lms/french.lm +148 -148
- data/lib/scylla/lms/german.lm +258 -258
- data/lib/scylla/lms/greek.lm +236 -236
- data/lib/scylla/lms/hebrew.lm +154 -154
- data/lib/scylla/lms/hindi.lm +139 -139
- data/lib/scylla/lms/icelandic.lm +239 -239
- data/lib/scylla/lms/indonesian.lm +244 -244
- data/lib/scylla/lms/italian.lm +248 -248
- data/lib/scylla/lms/japanese.lm +90 -90
- data/lib/scylla/lms/korean.lm +306 -306
- data/lib/scylla/lms/norwegian.lm +193 -193
- data/lib/scylla/lms/polish.lm +241 -241
- data/lib/scylla/lms/portuguese.lm +232 -232
- data/lib/scylla/lms/romanian.lm +246 -246
- data/lib/scylla/lms/slovak.lm +242 -242
- data/lib/scylla/lms/slovenian.lm +229 -229
- data/lib/scylla/lms/spanish.lm +164 -164
- data/lib/scylla/lms/swedish.lm +157 -157
- data/lib/scylla/lms/tagalog.lm +247 -247
- data/lib/scylla/lms/thai.lm +252 -252
- data/lib/scylla/lms/turkish.lm +285 -285
- data/lib/scylla/lms/vietnamese.lm +250 -250
- data/lib/scylla/lms/welsh.lm +248 -248
- data/lib/scylla/resources.rb +1 -9
- data/lib/scylla.rb +4 -0
- data/scylla.gemspec +2 -120
- data/source_texts/english.txt +62 -27
- data/test/classifier_test.rb +1 -3
- data/test/fixtures/lms/danish.lm +173 -173
- data/test/fixtures/lms/english.lm +220 -220
- data/test/fixtures/lms/french.lm +175 -175
- data/test/fixtures/lms/german.lm +254 -254
- data/test/fixtures/lms/hindi.lm +139 -139
- data/test/fixtures/lms/italian.lm +236 -236
- data/test/fixtures/lms/japanese.lm +88 -88
- data/test/fixtures/lms/norwegian.lm +182 -182
- data/test/fixtures/lms/spanish.lm +164 -164
- data/test/fixtures/test_languages/spanish +0 -1
- data/test/generator_test.rb +13 -0
- data/test/helper.rb +2 -0
- metadata +18 -25
- data/.document +0 -5
- data/lib/scylla/lms/13375P33K.lm +0 -400
- data/scylla-0.1.0.gem +0 -0
- data/source_texts/13375P33K.txt +0 -199
- data/test/fixtures/lms/13375p33k.lm +0 -400
- data/test/fixtures/source_texts/13375P33K.txt +0 -199
data/lib/scylla/lms/arabic.lm
CHANGED
|
@@ -1,62 +1,62 @@
|
|
|
1
1
|
� 26053
|
|
2
2
|
� 21638
|
|
3
|
-
_
|
|
4
|
-
ا 7004
|
|
3
|
+
_ 20952
|
|
5
4
|
� 7004
|
|
6
|
-
|
|
5
|
+
ا 7004
|
|
6
|
+
_� 5950
|
|
7
7
|
� 5598
|
|
8
8
|
ل 5598
|
|
9
|
-
�� 4328
|
|
10
9
|
ا� 4328
|
|
11
|
-
|
|
10
|
+
�� 4328
|
|
11
|
+
_� 3540
|
|
12
12
|
� 3314
|
|
13
13
|
ي 3314
|
|
14
|
-
�ل 3313
|
|
15
14
|
ال 3313
|
|
15
|
+
�ل 3313
|
|
16
16
|
م 3159
|
|
17
17
|
� 3159
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
_ا
|
|
18
|
+
ل� 2847
|
|
19
|
+
�� 2847
|
|
20
|
+
_ا 2729
|
|
21
21
|
� 2654
|
|
22
22
|
و 2654
|
|
23
|
-
_ا�
|
|
24
|
-
_ال
|
|
25
|
-
� 2192
|
|
23
|
+
_ا� 2629
|
|
24
|
+
_ال 2527
|
|
26
25
|
ر 2192
|
|
27
|
-
|
|
26
|
+
� 2192
|
|
28
27
|
�ل� 2158
|
|
29
|
-
|
|
28
|
+
ال� 2158
|
|
30
29
|
� 2092
|
|
31
|
-
|
|
32
|
-
ل�
|
|
30
|
+
ن 2092
|
|
31
|
+
ل� 2059
|
|
32
|
+
�� 2059
|
|
33
33
|
ا� 2030
|
|
34
34
|
�� 2030
|
|
35
|
-
ت 1994
|
|
36
35
|
� 1994
|
|
36
|
+
ت 1994
|
|
37
37
|
� 1777
|
|
38
38
|
ب 1772
|
|
39
|
-
��
|
|
40
|
-
ي�
|
|
41
|
-
ة 1712
|
|
39
|
+
�� 1734
|
|
40
|
+
ي� 1734
|
|
42
41
|
� 1712
|
|
42
|
+
ة 1712
|
|
43
43
|
ع 1691
|
|
44
44
|
� 1691
|
|
45
45
|
د 1652
|
|
46
46
|
� 1652
|
|
47
|
-
و� 1513
|
|
48
47
|
�� 1513
|
|
49
|
-
|
|
48
|
+
و� 1513
|
|
50
49
|
�_ 1426
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
ة_ 1426
|
|
51
|
+
�� 1355
|
|
52
|
+
م� 1355
|
|
53
53
|
�� 1183
|
|
54
54
|
م� 1183
|
|
55
55
|
س 1089
|
|
56
56
|
� 1089
|
|
57
|
-
_و
|
|
58
|
-
�ل� 1026
|
|
57
|
+
_و 1080
|
|
59
58
|
ال� 1026
|
|
59
|
+
�ل� 1026
|
|
60
60
|
ق 1019
|
|
61
61
|
� 1019
|
|
62
62
|
� 976
|
|
@@ -65,129 +65,129 @@ _و 1070
|
|
|
65
65
|
� 950
|
|
66
66
|
� 943
|
|
67
67
|
ه 943
|
|
68
|
-
|
|
69
|
-
|
|
68
|
+
ر� 941
|
|
69
|
+
�� 941
|
|
70
70
|
� 925
|
|
71
|
-
ب� 918
|
|
72
71
|
�� 918
|
|
73
|
-
|
|
72
|
+
ب� 918
|
|
74
73
|
ح 916
|
|
74
|
+
� 916
|
|
75
75
|
�� 907
|
|
76
76
|
و� 907
|
|
77
77
|
ع� 879
|
|
78
78
|
�� 879
|
|
79
79
|
أ 874
|
|
80
|
-
_م
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
ر� 722
|
|
80
|
+
_م 826
|
|
81
|
+
__ 825
|
|
82
|
+
�� 812
|
|
83
|
+
ي� 812
|
|
84
|
+
ت� 811
|
|
85
|
+
�� 811
|
|
86
|
+
�_ 776
|
|
87
|
+
ن_ 776
|
|
88
|
+
. 772
|
|
89
|
+
ن� 729
|
|
90
|
+
�� 729
|
|
92
91
|
�� 722
|
|
93
|
-
|
|
94
|
-
_
|
|
92
|
+
ر� 722
|
|
93
|
+
_ب 703
|
|
94
|
+
_و� 702
|
|
95
95
|
د� 696
|
|
96
96
|
�� 696
|
|
97
97
|
�� 690
|
|
98
98
|
ب� 685
|
|
99
|
-
�_ 683
|
|
100
99
|
ي_ 683
|
|
100
|
+
�_ 683
|
|
101
101
|
�� 675
|
|
102
102
|
ت� 675
|
|
103
|
-
لم 654
|
|
104
103
|
�م 654
|
|
104
|
+
لم 654
|
|
105
105
|
ج 630
|
|
106
106
|
� 630
|
|
107
|
-
�� 621
|
|
108
107
|
ق� 621
|
|
109
|
-
|
|
108
|
+
�� 621
|
|
110
109
|
ية 609
|
|
111
|
-
|
|
112
|
-
ل_ 606
|
|
110
|
+
�ة 609
|
|
113
111
|
ع� 592
|
|
114
112
|
�� 592
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
113
|
+
ل_ 579
|
|
114
|
+
�_ 579
|
|
115
|
+
س� 556
|
|
116
|
+
�� 556
|
|
119
117
|
ط 548
|
|
118
|
+
� 548
|
|
120
119
|
ح� 546
|
|
121
120
|
�� 546
|
|
122
121
|
�� 539
|
|
122
|
+
._ 539
|
|
123
123
|
, 537
|
|
124
|
+
�_ 533
|
|
125
|
+
ا_ 533
|
|
124
126
|
,_ 532
|
|
125
|
-
�_ 532
|
|
126
|
-
ا_ 532
|
|
127
|
-
م_ 514
|
|
128
127
|
�_ 514
|
|
128
|
+
م_ 514
|
|
129
129
|
�لم 506
|
|
130
130
|
أ� 489
|
|
131
131
|
�� 487
|
|
132
132
|
ه� 487
|
|
133
|
-
�_
|
|
134
|
-
ر_
|
|
133
|
+
�_ 482
|
|
134
|
+
ر_ 482
|
|
135
135
|
وا 479
|
|
136
136
|
�ا 479
|
|
137
|
-
ية_
|
|
138
|
-
�ة_
|
|
139
|
-
ت_
|
|
140
|
-
�_
|
|
137
|
+
ية_ 472
|
|
138
|
+
�ة_ 472
|
|
139
|
+
ت_ 471
|
|
140
|
+
�_ 471
|
|
141
141
|
_أ 470
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
د� 454
|
|
142
|
+
�� 456
|
|
143
|
+
د� 456
|
|
145
144
|
�� 448
|
|
146
145
|
ف� 448
|
|
146
|
+
_م� 448
|
|
147
147
|
� 447
|
|
148
148
|
ص 447
|
|
149
149
|
إ 445
|
|
150
150
|
� 445
|
|
151
|
-
|
|
151
|
+
�� 441
|
|
152
|
+
ك� 441
|
|
152
153
|
� 439
|
|
153
|
-
�� 439
|
|
154
|
-
ك� 439
|
|
155
154
|
ش 439
|
|
156
|
-
ن� 437
|
|
157
155
|
�� 437
|
|
156
|
+
ن� 437
|
|
157
|
+
_ع 435
|
|
158
158
|
� 434
|
|
159
|
-
�� 433
|
|
160
159
|
س� 433
|
|
161
|
-
|
|
160
|
+
�� 433
|
|
162
161
|
ز 430
|
|
163
162
|
د_ 430
|
|
163
|
+
�_ 430
|
|
164
164
|
�ن 427
|
|
165
165
|
من 427
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
166
|
+
�ا 402
|
|
167
|
+
_ب� 402
|
|
168
|
+
ما 402
|
|
169
|
+
لا 398
|
|
170
|
+
�ا 398
|
|
170
171
|
ان 384
|
|
171
|
-
|
|
172
|
+
�ن 384
|
|
172
173
|
�� 382
|
|
174
|
+
ك� 382
|
|
173
175
|
�� 381
|
|
174
176
|
أ� 381
|
|
175
|
-
لا 374
|
|
176
|
-
�ا 374
|
|
177
177
|
�ي 368
|
|
178
178
|
لي 368
|
|
179
|
-
ات 366
|
|
180
179
|
�ت 366
|
|
180
|
+
ات 366
|
|
181
181
|
_ف 361
|
|
182
182
|
ار 361
|
|
183
183
|
�ر 361
|
|
184
184
|
_ل 355
|
|
185
185
|
ف� 353
|
|
186
186
|
�� 353
|
|
187
|
-
_
|
|
188
|
-
|
|
187
|
+
_ت 352
|
|
188
|
+
_م� 352
|
|
189
189
|
خ 343
|
|
190
|
-
|
|
190
|
+
� 343
|
|
191
191
|
ض 332
|
|
192
192
|
� 332
|
|
193
193
|
�ا� 329
|
|
@@ -195,206 +195,206 @@ _ت 340
|
|
|
195
195
|
، 318
|
|
196
196
|
� 318
|
|
197
197
|
�ي 317
|
|
198
|
-
في 317
|
|
199
|
-
�م� 317
|
|
200
198
|
لم� 317
|
|
199
|
+
�م� 317
|
|
200
|
+
في 317
|
|
201
201
|
� 316
|
|
202
202
|
ث 316
|
|
203
|
-
_
|
|
204
|
-
|
|
205
|
-
|
|
203
|
+
_و� 314
|
|
204
|
+
_وا 311
|
|
205
|
+
�� 310
|
|
206
|
+
ج� 310
|
|
206
207
|
�� 306
|
|
207
208
|
ح� 306
|
|
208
|
-
_و� 306
|
|
209
209
|
با 304
|
|
210
210
|
�ا 304
|
|
211
211
|
_ف� 303
|
|
212
|
-
_أ� 301
|
|
213
212
|
_ب� 301
|
|
213
|
+
_أ� 301
|
|
214
214
|
ى 293
|
|
215
215
|
� 293
|
|
216
|
-
دي 282
|
|
217
216
|
�ي 282
|
|
218
|
-
|
|
217
|
+
دي 282
|
|
219
218
|
ّ 281
|
|
219
|
+
� 281
|
|
220
220
|
�_ 280
|
|
221
221
|
،_ 280
|
|
222
|
-
يا
|
|
223
|
-
�ا
|
|
224
|
-
�� 275
|
|
222
|
+
يا 279
|
|
223
|
+
�ا 279
|
|
225
224
|
ري 275
|
|
226
|
-
�ي 275
|
|
227
225
|
ج� 275
|
|
226
|
+
�ي 275
|
|
227
|
+
�� 275
|
|
228
228
|
�� 273
|
|
229
|
-
�ت_ 273
|
|
230
|
-
ات_ 273
|
|
231
229
|
ه� 273
|
|
230
|
+
�ت_ 272
|
|
231
|
+
ات_ 272
|
|
232
|
+
�� 271
|
|
232
233
|
ق� 271
|
|
233
234
|
�ال 271
|
|
234
|
-
�� 271
|
|
235
235
|
�_ 270
|
|
236
|
+
�أ 270
|
|
237
|
+
لأ 270
|
|
236
238
|
ى_ 270
|
|
237
|
-
�أ 269
|
|
238
|
-
لأ 269
|
|
239
|
-
�ا 261
|
|
240
239
|
ها 261
|
|
240
|
+
�ا 261
|
|
241
241
|
�ن 256
|
|
242
242
|
ين 256
|
|
243
243
|
لع 255
|
|
244
244
|
�ع 255
|
|
245
|
-
ام 254
|
|
246
245
|
�م 254
|
|
246
|
+
ام 254
|
|
247
247
|
_من 251
|
|
248
|
-
�م� 248
|
|
249
248
|
لم� 248
|
|
249
|
+
�م� 248
|
|
250
250
|
_ل� 247
|
|
251
251
|
ش� 246
|
|
252
252
|
�� 246
|
|
253
253
|
_في 244
|
|
254
|
-
�� 242
|
|
255
254
|
ط� 242
|
|
256
|
-
|
|
255
|
+
�� 242
|
|
256
|
+
�ا 240
|
|
257
257
|
را 240
|
|
258
258
|
عا 240
|
|
259
|
-
�ا 240
|
|
260
|
-
_ع� 237
|
|
259
|
+
�ا 240
|
|
261
260
|
_إ 236
|
|
262
261
|
�لأ 235
|
|
263
|
-
�ل 234
|
|
264
262
|
�لع 234
|
|
263
|
+
�ل 234
|
|
265
264
|
ول 234
|
|
266
|
-
ذ 233
|
|
267
|
-
�� 233
|
|
268
|
-
� 233
|
|
269
265
|
إ� 233
|
|
270
|
-
|
|
266
|
+
� 233
|
|
267
|
+
�� 233
|
|
268
|
+
ذ 233
|
|
271
269
|
�� 230
|
|
272
|
-
|
|
270
|
+
ط� 230
|
|
273
271
|
في_ 229
|
|
274
|
-
_ك 229
|
|
275
272
|
ص� 229
|
|
273
|
+
�� 229
|
|
276
274
|
�ي_ 229
|
|
277
|
-
|
|
278
|
-
�� 224
|
|
275
|
+
_ك 229
|
|
279
276
|
_ع� 224
|
|
277
|
+
�� 224
|
|
278
|
+
خ� 224
|
|
280
279
|
�ن_ 221
|
|
281
280
|
من_ 221
|
|
281
|
+
�ع� 215
|
|
282
282
|
لع� 215
|
|
283
283
|
لك 215
|
|
284
284
|
�ك 215
|
|
285
|
-
�ع� 215
|
|
286
|
-
ز� 213
|
|
287
285
|
�� 213
|
|
288
|
-
|
|
289
|
-
�� 212
|
|
286
|
+
ز� 213
|
|
290
287
|
إ� 212
|
|
288
|
+
�� 212
|
|
289
|
+
غ 212
|
|
291
290
|
� 212
|
|
291
|
+
_ع� 211
|
|
292
292
|
لس 211
|
|
293
293
|
�س 211
|
|
294
|
-
بي 209
|
|
295
294
|
�ي 209
|
|
295
|
+
بي 209
|
|
296
296
|
�ل 208
|
|
297
297
|
عل 208
|
|
298
298
|
�� 204
|
|
299
|
-
مل 204
|
|
300
299
|
ص� 204
|
|
300
|
+
مل 204
|
|
301
301
|
�ل 204
|
|
302
|
+
_ق 201
|
|
302
303
|
�ا� 200
|
|
303
|
-
با� 200
|
|
304
304
|
� 200
|
|
305
|
+
با� 200
|
|
305
306
|
ء 197
|
|
306
|
-
_
|
|
307
|
-
�د 195
|
|
308
|
-
�لس 195
|
|
309
|
-
�_ 195
|
|
307
|
+
_ت� 197
|
|
310
308
|
ع_ 195
|
|
309
|
+
�_ 195
|
|
310
|
+
�د 195
|
|
311
311
|
اد 195
|
|
312
|
+
�لس 195
|
|
312
313
|
�د 191
|
|
313
314
|
ود 191
|
|
314
|
-
�د 187
|
|
315
315
|
عد 187
|
|
316
|
-
|
|
316
|
+
�د 187
|
|
317
317
|
_ح 183
|
|
318
318
|
و_ 181
|
|
319
319
|
�_ 181
|
|
320
|
-
�ح 180
|
|
321
320
|
لح 180
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
321
|
+
�ت 180
|
|
322
|
+
لي� 180
|
|
323
|
+
�ي� 180
|
|
324
|
+
لت 180
|
|
325
|
+
�ح 180
|
|
325
326
|
ري� 178
|
|
327
|
+
_ه 178
|
|
326
328
|
�ي� 178
|
|
327
|
-
�إ 177
|
|
328
|
-
�ي� 177
|
|
329
329
|
لإ 177
|
|
330
|
-
|
|
331
|
-
|
|
330
|
+
�ا� 177
|
|
331
|
+
لا� 177
|
|
332
|
+
�إ 177
|
|
332
333
|
�ا� 176
|
|
333
|
-
|
|
334
|
-
�ا
|
|
335
|
-
|
|
334
|
+
را� 176
|
|
335
|
+
�ا 175
|
|
336
|
+
نا 175
|
|
336
337
|
�� 173
|
|
337
|
-
|
|
338
|
-
�ب
|
|
339
|
-
|
|
338
|
+
رب 173
|
|
339
|
+
�ب 173
|
|
340
|
+
ش� 173
|
|
340
341
|
وي 172
|
|
342
|
+
�ي 172
|
|
341
343
|
اء 171
|
|
342
344
|
ها_ 171
|
|
343
|
-
�ء 171
|
|
344
345
|
�ا_ 171
|
|
346
|
+
�ء 171
|
|
347
|
+
_أ� 169
|
|
345
348
|
يا� 169
|
|
346
349
|
�ا� 169
|
|
347
|
-
_أ� 169
|
|
348
|
-
ير 166
|
|
349
350
|
�ر 166
|
|
350
|
-
|
|
351
|
+
ير 166
|
|
351
352
|
�ل 165
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
353
|
+
بل 165
|
|
354
|
+
_با 165
|
|
355
|
+
�ا 162
|
|
356
|
+
دا 162
|
|
357
|
+
ه_ 160
|
|
358
|
+
دي� 160
|
|
355
359
|
�_ 160
|
|
356
360
|
�ي� 160
|
|
357
|
-
دي� 160
|
|
358
|
-
ه_ 160
|
|
359
361
|
ني 158
|
|
360
362
|
�ي 158
|
|
361
363
|
مي 157
|
|
362
364
|
�ي 157
|
|
363
365
|
�لح 156
|
|
364
|
-
_ت� 155
|
|
365
|
-
�_ 155
|
|
366
366
|
ء_ 155
|
|
367
|
+
�_ 155
|
|
368
|
+
_ت� 155
|
|
367
369
|
�لإ 154
|
|
368
|
-
ف_ 153
|
|
369
|
-
لا� 153
|
|
370
|
-
�ا� 153
|
|
371
370
|
�_ 153
|
|
371
|
+
ف_ 153
|
|
372
372
|
�ل� 152
|
|
373
373
|
مل� 152
|
|
374
|
-
�ة 151
|
|
375
374
|
رة 151
|
|
375
|
+
�ة 151
|
|
376
|
+
مع 149
|
|
376
377
|
لا� 149
|
|
377
|
-
�ا� 149
|
|
378
378
|
�ع 149
|
|
379
|
-
|
|
379
|
+
�ا� 149
|
|
380
|
+
لأ� 148
|
|
380
381
|
�ال 148
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
382
|
+
�أ� 148
|
|
383
|
+
�_ 147
|
|
384
|
+
�م 147
|
|
384
385
|
لى 147
|
|
385
386
|
�ى 147
|
|
387
|
+
ب_ 147
|
|
388
|
+
تم 147
|
|
386
389
|
_ي 147
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
_ج 145
|
|
392
|
-
�ا_ 145
|
|
390
|
+
�ا_ 146
|
|
391
|
+
ما_ 146
|
|
392
|
+
_ق� 146
|
|
393
|
+
_ج 146
|
|
393
394
|
�ا� 144
|
|
394
|
-
عا� 144
|
|
395
|
-
لب 144
|
|
396
|
-
_ق� 144
|
|
397
395
|
�ب 144
|
|
398
|
-
|
|
399
|
-
|
|
396
|
+
لب 144
|
|
397
|
+
عا� 144
|
|
400
398
|
اس 143
|
|
399
|
+
سي 143
|
|
400
|
+
�س 143
|