scylla 0.4.3 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -0
- data/Gemfile.lock +10 -0
- data/VERSION +1 -1
- data/lib/scylla/generator.rb +1 -1
- data/lib/scylla/lms/13375P33K.lm +156 -156
- data/lib/scylla/lms/arabic.lm +133 -133
- data/lib/scylla/lms/bulgarian.lm +122 -122
- data/lib/scylla/lms/catalan.lm +151 -151
- data/lib/scylla/lms/danish.lm +137 -137
- data/lib/scylla/lms/english.lm +207 -207
- data/lib/scylla/lms/french.lm +400 -400
- data/lib/scylla/lms/japanese.lm +400 -400
- data/lib/scylla/lms/korean.lm +233 -233
- data/lib/scylla/lms/norwegian.lm +398 -398
- data/lib/scylla/lms/spanish.lm +98 -98
- data/lib/scylla/lms/swedish.lm +123 -123
- data/lib/scylla/lms/tagalog.lm +223 -223
- data/lib/scylla/lms/welsh.lm +234 -234
- data/lib/scylla/resources.rb +10 -10
- data/scylla.gemspec +17 -40
- data/source_texts/catalan.txt +28 -28
- data/source_texts/danish.txt +62 -62
- data/source_texts/english.txt +10 -10
- data/source_texts/french.txt +470 -77
- data/source_texts/japanese.txt +453 -199
- data/source_texts/norwegian.txt +96 -63
- data/source_texts/spanish.txt +269 -269
- data/test/classifier_test.rb +2 -2
- data/test/fixtures/lms/13375p33k.lm +156 -156
- data/test/fixtures/lms/danish.lm +137 -137
- data/test/fixtures/lms/english.lm +207 -207
- data/test/fixtures/lms/french.lm +400 -400
- data/test/fixtures/lms/hindi.lm +400 -0
- data/test/fixtures/lms/italian.lm +400 -0
- data/test/fixtures/lms/japanese.lm +400 -400
- data/test/fixtures/lms/norwegian.lm +400 -0
- data/test/fixtures/lms/spanish.lm +98 -98
- data/test/fixtures/source_texts/danish.txt +62 -62
- data/test/fixtures/source_texts/english.txt +10 -10
- data/test/fixtures/source_texts/french.txt +470 -77
- data/test/fixtures/source_texts/hindi.txt +199 -0
- data/test/fixtures/source_texts/italian.txt +120 -0
- data/test/fixtures/source_texts/japanese.txt +453 -199
- data/test/fixtures/source_texts/norwegian.txt +190 -0
- data/test/fixtures/source_texts/spanish.txt +269 -269
- data/test/fixtures/test_languages/english +61 -0
- data/test/fixtures/test_languages/french +0 -0
- data/test/fixtures/test_languages/german +29 -0
- data/test/fixtures/test_languages/hindi +3 -0
- data/test/fixtures/test_languages/italian +6 -0
- data/test/fixtures/test_languages/japanese +79 -0
- data/test/fixtures/test_languages/norwegian +14 -0
- data/test/fixtures/test_languages/spanish +22 -0
- data/test/generator_test.rb +0 -1
- data/test/language_test.rb +28 -0
- metadata +20 -43
- data/lib/scylla/lms/esperanto.lm +0 -400
- data/lib/scylla/lms/hungarian.lm +0 -400
- data/lib/scylla/lms/irish.lm +0 -400
- data/lib/scylla/lms/kannada.lm +0 -400
- data/lib/scylla/lms/latin.lm +0 -400
- data/lib/scylla/lms/malay.lm +0 -400
- data/lib/scylla/lms/marathi.lm +0 -400
- data/lib/scylla/lms/mingo.lm +0 -400
- data/lib/scylla/lms/nepali.lm +0 -400
- data/lib/scylla/lms/quechua.lm +0 -400
- data/lib/scylla/lms/rumantsch.lm +0 -400
- data/lib/scylla/lms/sanskrit.lm +0 -400
- data/lib/scylla/lms/scots_gaelic.lm +0 -400
- data/lib/scylla/lms/serbian.lm +0 -400
- data/lib/scylla/lms/swahili.lm +0 -400
- data/lib/scylla/lms/tamil.lm +0 -400
- data/lib/scylla/lms/ukrainian.lm +0 -400
- data/lib/scylla/lms/yiddish.lm +0 -400
- data/source_texts/esperanto.txt +0 -199
- data/source_texts/hungarian.txt +0 -102
- data/source_texts/irish.txt +0 -209
- data/source_texts/kannada.txt +0 -283
- data/source_texts/latin.txt +0 -120
- data/source_texts/malay.txt +0 -108
- data/source_texts/marathi.txt +0 -100
- data/source_texts/mingo.txt +0 -146
- data/source_texts/nepali.txt +0 -131
- data/source_texts/quechua.txt +0 -108
- data/source_texts/rumantsch.txt +0 -110
- data/source_texts/sanskrit.txt +0 -135
- data/source_texts/scots_gaelic.txt +0 -93
- data/source_texts/serbian.txt +0 -121
- data/source_texts/swahili.txt +0 -120
- data/source_texts/tamil.txt +0 -167
- data/source_texts/ukrainian.txt +0 -214
- data/source_texts/yiddish-utf.txt +0 -83
- data/test/fixtures/lms/kannada.lm +0 -400
- data/test/fixtures/source_texts/kannada.txt +0 -283
data/lib/scylla/lms/arabic.lm
CHANGED
@@ -1,57 +1,57 @@
|
|
1
1
|
� 26053
|
2
2
|
� 21638
|
3
|
-
_
|
4
|
-
� 7004
|
3
|
+
_ 21106
|
5
4
|
ا 7004
|
6
|
-
|
5
|
+
� 7004
|
6
|
+
_� 5968
|
7
7
|
� 5598
|
8
8
|
ل 5598
|
9
9
|
�� 4328
|
10
10
|
ا� 4328
|
11
|
-
_�
|
12
|
-
ي 3314
|
11
|
+
_� 3517
|
13
12
|
� 3314
|
14
|
-
|
13
|
+
ي 3314
|
15
14
|
�ل 3313
|
15
|
+
ال 3313
|
16
16
|
م 3159
|
17
17
|
� 3159
|
18
18
|
�� 2820
|
19
19
|
ل� 2820
|
20
|
-
_ا
|
20
|
+
_ا 2741
|
21
21
|
� 2654
|
22
22
|
و 2654
|
23
|
-
_ا�
|
24
|
-
_ال
|
25
|
-
ر 2192
|
23
|
+
_ا� 2641
|
24
|
+
_ال 2539
|
26
25
|
� 2192
|
26
|
+
ر 2192
|
27
27
|
ال� 2158
|
28
28
|
�ل� 2158
|
29
|
-
� 2092
|
30
29
|
ن 2092
|
30
|
+
� 2092
|
31
31
|
�� 2058
|
32
32
|
ل� 2058
|
33
|
-
�� 2030
|
34
33
|
ا� 2030
|
35
|
-
|
34
|
+
�� 2030
|
36
35
|
ت 1994
|
36
|
+
� 1994
|
37
37
|
� 1777
|
38
38
|
ب 1772
|
39
|
-
ي� 1727
|
40
39
|
�� 1727
|
41
|
-
|
40
|
+
ي� 1727
|
42
41
|
ة 1712
|
42
|
+
� 1712
|
43
43
|
ع 1691
|
44
44
|
� 1691
|
45
|
-
� 1652
|
46
45
|
د 1652
|
47
|
-
|
46
|
+
� 1652
|
48
47
|
و� 1513
|
49
|
-
|
50
|
-
|
51
|
-
|
48
|
+
�� 1513
|
49
|
+
ة_ 1426
|
50
|
+
�_ 1426
|
52
51
|
�� 1353
|
53
|
-
م�
|
52
|
+
م� 1353
|
54
53
|
�� 1183
|
54
|
+
م� 1183
|
55
55
|
س 1089
|
56
56
|
� 1089
|
57
57
|
_و 1070
|
@@ -61,8 +61,8 @@ _و 1070
|
|
61
61
|
� 1019
|
62
62
|
� 976
|
63
63
|
ف 976
|
64
|
-
� 950
|
65
64
|
ك 950
|
65
|
+
� 950
|
66
66
|
� 943
|
67
67
|
ه 943
|
68
68
|
�� 938
|
@@ -72,57 +72,57 @@ _و 1070
|
|
72
72
|
�� 918
|
73
73
|
� 916
|
74
74
|
ح 916
|
75
|
-
و� 907
|
76
75
|
�� 907
|
76
|
+
و� 907
|
77
77
|
ع� 879
|
78
78
|
�� 879
|
79
79
|
أ 874
|
80
80
|
_م 816
|
81
|
-
ي� 811
|
82
81
|
�� 811
|
82
|
+
ي� 811
|
83
83
|
ت� 810
|
84
84
|
�� 810
|
85
|
-
__
|
85
|
+
__ 804
|
86
86
|
. 775
|
87
87
|
�_ 773
|
88
88
|
ن_ 773
|
89
89
|
�� 726
|
90
90
|
ن� 726
|
91
|
-
�� 722
|
92
91
|
ر� 722
|
92
|
+
�� 722
|
93
|
+
_ب 701
|
93
94
|
_و� 700
|
94
|
-
_ب 699
|
95
|
-
�� 696
|
96
95
|
د� 696
|
96
|
+
�� 696
|
97
97
|
�� 690
|
98
98
|
ب� 685
|
99
|
-
|
100
|
-
|
101
|
-
ت� 675
|
99
|
+
�_ 683
|
100
|
+
ي_ 683
|
102
101
|
�� 675
|
103
|
-
|
102
|
+
ت� 675
|
104
103
|
لم 654
|
105
|
-
|
104
|
+
�م 654
|
106
105
|
ج 630
|
107
|
-
|
106
|
+
� 630
|
108
107
|
�� 621
|
108
|
+
ق� 621
|
109
109
|
�ة 609
|
110
110
|
ية 609
|
111
|
+
�_ 606
|
112
|
+
ل_ 606
|
111
113
|
ع� 592
|
112
114
|
�� 592
|
113
|
-
|
114
|
-
ل_ 577
|
115
|
+
._ 557
|
115
116
|
س� 554
|
116
117
|
�� 554
|
117
118
|
� 548
|
118
119
|
ط 548
|
119
|
-
�� 546
|
120
120
|
ح� 546
|
121
|
+
�� 546
|
121
122
|
�� 539
|
122
123
|
, 537
|
123
|
-
._ 536
|
124
|
-
�_ 532
|
125
124
|
,_ 532
|
125
|
+
�_ 532
|
126
126
|
ا_ 532
|
127
127
|
م_ 514
|
128
128
|
�_ 514
|
@@ -130,271 +130,271 @@ _ب 699
|
|
130
130
|
أ� 489
|
131
131
|
�� 487
|
132
132
|
ه� 487
|
133
|
-
|
134
|
-
|
135
|
-
�ا 479
|
133
|
+
�_ 484
|
134
|
+
ر_ 484
|
136
135
|
وا 479
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
_
|
142
|
-
|
136
|
+
�ا 479
|
137
|
+
ية_ 473
|
138
|
+
�ة_ 473
|
139
|
+
ت_ 472
|
140
|
+
�_ 472
|
141
|
+
_أ 470
|
142
|
+
_ع 461
|
143
143
|
�� 454
|
144
|
-
|
144
|
+
د� 454
|
145
145
|
�� 448
|
146
|
+
ف� 448
|
146
147
|
� 447
|
147
148
|
ص 447
|
148
|
-
_م� 445
|
149
149
|
إ 445
|
150
150
|
� 445
|
151
|
-
|
151
|
+
_م� 445
|
152
152
|
� 439
|
153
153
|
�� 439
|
154
|
+
ك� 439
|
154
155
|
ش 439
|
155
156
|
ن� 437
|
156
157
|
�� 437
|
157
158
|
� 434
|
158
159
|
�� 433
|
159
160
|
س� 433
|
160
|
-
_ع 432
|
161
|
-
ز 430
|
162
161
|
�_ 430
|
162
|
+
ز 430
|
163
163
|
د_ 430
|
164
|
-
من 427
|
165
164
|
�ن 427
|
166
|
-
|
165
|
+
من 427
|
167
166
|
ما 401
|
168
|
-
|
167
|
+
�ا 401
|
168
|
+
_ب� 400
|
169
169
|
�ن 384
|
170
170
|
ان 384
|
171
171
|
ك� 382
|
172
172
|
�� 382
|
173
173
|
�� 381
|
174
174
|
أ� 381
|
175
|
-
�ا 374
|
176
175
|
لا 374
|
177
|
-
|
176
|
+
�ا 374
|
178
177
|
�ي 368
|
178
|
+
لي 368
|
179
179
|
ات 366
|
180
180
|
�ت 366
|
181
|
-
�ر 361
|
182
|
-
ار 361
|
183
181
|
_ف 361
|
182
|
+
ار 361
|
183
|
+
�ر 361
|
184
184
|
_ل 355
|
185
185
|
ف� 353
|
186
186
|
�� 353
|
187
187
|
_م� 345
|
188
|
-
خ 343
|
189
188
|
� 343
|
190
|
-
|
191
|
-
|
189
|
+
خ 343
|
190
|
+
_ت 340
|
192
191
|
ض 332
|
193
|
-
|
192
|
+
� 332
|
194
193
|
�ا� 329
|
194
|
+
وا� 329
|
195
195
|
، 318
|
196
196
|
� 318
|
197
|
+
�ي 317
|
197
198
|
في 317
|
198
199
|
�م� 317
|
199
|
-
�ي 317
|
200
200
|
لم� 317
|
201
201
|
� 316
|
202
202
|
ث 316
|
203
203
|
_وا 310
|
204
|
-
ج� 309
|
205
204
|
�� 309
|
205
|
+
ج� 309
|
206
|
+
�� 306
|
206
207
|
ح� 306
|
207
208
|
_و� 306
|
208
|
-
�� 306
|
209
209
|
با 304
|
210
210
|
�ا 304
|
211
211
|
_ف� 303
|
212
|
+
_أ� 301
|
212
213
|
_ب� 301
|
213
|
-
_أ� 300
|
214
|
-
� 293
|
215
214
|
ى 293
|
215
|
+
� 293
|
216
216
|
دي 282
|
217
217
|
�ي 282
|
218
|
-
ّ 281
|
219
218
|
� 281
|
220
|
-
|
219
|
+
ّ 281
|
221
220
|
�_ 280
|
222
|
-
|
221
|
+
،_ 280
|
223
222
|
يا 276
|
223
|
+
�ا 276
|
224
224
|
�� 275
|
225
|
-
�ي 275
|
226
225
|
ري 275
|
226
|
+
�ي 275
|
227
227
|
ج� 275
|
228
|
-
ه� 273
|
229
228
|
�� 273
|
230
|
-
|
231
|
-
|
229
|
+
�ت_ 273
|
230
|
+
ات_ 273
|
231
|
+
ه� 273
|
232
232
|
ق� 271
|
233
233
|
�ال 271
|
234
234
|
�� 271
|
235
|
-
ى_ 270
|
236
235
|
�_ 270
|
237
|
-
|
236
|
+
ى_ 270
|
238
237
|
�أ 269
|
238
|
+
لأ 269
|
239
239
|
�ا 261
|
240
240
|
ها 261
|
241
241
|
�ن 256
|
242
242
|
ين 256
|
243
|
-
�ع 255
|
244
243
|
لع 255
|
244
|
+
�ع 255
|
245
245
|
ام 254
|
246
246
|
�م 254
|
247
247
|
_من 251
|
248
248
|
�م� 248
|
249
249
|
لم� 248
|
250
250
|
_ل� 247
|
251
|
-
�� 246
|
252
251
|
ش� 246
|
252
|
+
�� 246
|
253
253
|
_في 244
|
254
254
|
�� 242
|
255
255
|
ط� 242
|
256
|
-
|
256
|
+
�ا 240
|
257
257
|
را 240
|
258
|
+
عا 240
|
258
259
|
�ا 240
|
259
|
-
|
260
|
+
_ع� 237
|
260
261
|
_إ 236
|
261
262
|
�لأ 235
|
262
263
|
�ل 234
|
263
264
|
�لع 234
|
264
265
|
ول 234
|
265
266
|
ذ 233
|
266
|
-
إ� 233
|
267
267
|
�� 233
|
268
268
|
� 233
|
269
|
+
إ� 233
|
269
270
|
ط� 230
|
270
271
|
�� 230
|
271
|
-
�ي_ 229
|
272
|
-
ص� 229
|
273
272
|
�� 229
|
274
273
|
في_ 229
|
275
274
|
_ك 229
|
276
|
-
|
275
|
+
ص� 229
|
276
|
+
�ي_ 229
|
277
277
|
خ� 224
|
278
|
+
�� 224
|
278
279
|
_ع� 224
|
279
280
|
�ن_ 221
|
280
281
|
من_ 221
|
281
|
-
لك 215
|
282
|
-
�ع� 215
|
283
282
|
لع� 215
|
283
|
+
لك 215
|
284
284
|
�ك 215
|
285
|
-
|
285
|
+
�ع� 215
|
286
286
|
ز� 213
|
287
|
-
��
|
288
|
-
� 212
|
287
|
+
�� 213
|
289
288
|
غ 212
|
289
|
+
�� 212
|
290
290
|
إ� 212
|
291
|
-
|
291
|
+
� 212
|
292
292
|
لس 211
|
293
|
-
|
293
|
+
�س 211
|
294
294
|
بي 209
|
295
|
-
|
295
|
+
�ي 209
|
296
296
|
�ل 208
|
297
|
-
|
298
|
-
|
297
|
+
عل 208
|
298
|
+
�� 204
|
299
299
|
مل 204
|
300
300
|
ص� 204
|
301
|
-
|
301
|
+
�ل 204
|
302
|
+
�ا� 200
|
302
303
|
با� 200
|
303
304
|
� 200
|
304
|
-
�ا� 200
|
305
|
-
_ق 197
|
306
305
|
ء 197
|
307
|
-
|
306
|
+
_ق 197
|
308
307
|
�د 195
|
309
308
|
�لس 195
|
310
|
-
ع_ 195
|
311
309
|
�_ 195
|
310
|
+
ع_ 195
|
311
|
+
اد 195
|
312
312
|
�د 191
|
313
313
|
ود 191
|
314
314
|
�د 187
|
315
315
|
عد 187
|
316
|
-
_ت�
|
316
|
+
_ت� 185
|
317
317
|
_ح 183
|
318
|
-
�_ 181
|
319
318
|
و_ 181
|
320
|
-
|
319
|
+
�_ 181
|
321
320
|
�ح 180
|
321
|
+
لح 180
|
322
|
+
_ه 179
|
322
323
|
�ت 179
|
323
324
|
لت 179
|
324
325
|
ري� 178
|
325
|
-
_ه 178
|
326
326
|
�ي� 178
|
327
|
-
|
327
|
+
�إ 177
|
328
328
|
�ي� 177
|
329
329
|
لإ 177
|
330
|
-
|
330
|
+
لي� 177
|
331
331
|
را� 176
|
332
332
|
�ا� 176
|
333
|
-
�ا 174
|
334
333
|
نا 174
|
335
|
-
|
334
|
+
�ا 174
|
336
335
|
ش� 173
|
337
|
-
|
336
|
+
�� 173
|
338
337
|
�ي 172
|
339
|
-
وي 172
|
340
338
|
�ب 172
|
339
|
+
رب 172
|
340
|
+
وي 172
|
341
|
+
اء 171
|
341
342
|
ها_ 171
|
342
343
|
�ء 171
|
343
|
-
اء 171
|
344
344
|
�ا_ 171
|
345
345
|
يا� 169
|
346
346
|
�ا� 169
|
347
|
-
_أ�
|
348
|
-
�ر 166
|
347
|
+
_أ� 169
|
349
348
|
ير 166
|
349
|
+
�ر 166
|
350
350
|
بل 165
|
351
351
|
�ل 165
|
352
352
|
_با 162
|
353
353
|
�ا 161
|
354
354
|
دا 161
|
355
|
+
�_ 160
|
356
|
+
�ي� 160
|
355
357
|
دي� 160
|
356
358
|
ه_ 160
|
357
|
-
�ي� 160
|
358
|
-
�_ 160
|
359
359
|
ني 158
|
360
360
|
�ي 158
|
361
361
|
مي 157
|
362
362
|
�ي 157
|
363
363
|
�لح 156
|
364
|
-
|
364
|
+
_ت� 155
|
365
365
|
�_ 155
|
366
|
-
_
|
366
|
+
ء_ 155
|
367
367
|
�لإ 154
|
368
|
+
ف_ 153
|
368
369
|
لا� 153
|
369
370
|
�ا� 153
|
370
|
-
�_
|
371
|
-
ف_ 152
|
371
|
+
�_ 153
|
372
372
|
�ل� 152
|
373
373
|
مل� 152
|
374
|
-
رة 151
|
375
374
|
�ة 151
|
375
|
+
رة 151
|
376
|
+
لا� 149
|
376
377
|
�ا� 149
|
377
|
-
مع 149
|
378
378
|
�ع 149
|
379
|
-
|
379
|
+
مع 149
|
380
380
|
�ال 148
|
381
|
-
لأ� 147
|
382
|
-
�م 147
|
383
|
-
�_ 147
|
384
381
|
تم 147
|
385
382
|
�أ� 147
|
386
|
-
�ى 147
|
387
|
-
لى 147
|
388
383
|
ب_ 147
|
384
|
+
لى 147
|
385
|
+
�ى 147
|
389
386
|
_ي 147
|
387
|
+
�_ 147
|
388
|
+
لأ� 147
|
389
|
+
�م 147
|
390
390
|
ما_ 145
|
391
|
+
_ج 145
|
391
392
|
�ا_ 145
|
392
|
-
عا� 144
|
393
|
-
_ج 144
|
394
|
-
�ب 144
|
395
393
|
�ا� 144
|
394
|
+
عا� 144
|
396
395
|
لب 144
|
397
396
|
_ق� 144
|
398
|
-
|
399
|
-
|
397
|
+
�ب 144
|
398
|
+
لس� 143
|
400
399
|
�ي 143
|
400
|
+
اس 143
|