scylla 0.4.3 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -0
- data/Gemfile.lock +10 -0
- data/VERSION +1 -1
- data/lib/scylla/generator.rb +1 -1
- data/lib/scylla/lms/13375P33K.lm +156 -156
- data/lib/scylla/lms/arabic.lm +133 -133
- data/lib/scylla/lms/bulgarian.lm +122 -122
- data/lib/scylla/lms/catalan.lm +151 -151
- data/lib/scylla/lms/danish.lm +137 -137
- data/lib/scylla/lms/english.lm +207 -207
- data/lib/scylla/lms/french.lm +400 -400
- data/lib/scylla/lms/japanese.lm +400 -400
- data/lib/scylla/lms/korean.lm +233 -233
- data/lib/scylla/lms/norwegian.lm +398 -398
- data/lib/scylla/lms/spanish.lm +98 -98
- data/lib/scylla/lms/swedish.lm +123 -123
- data/lib/scylla/lms/tagalog.lm +223 -223
- data/lib/scylla/lms/welsh.lm +234 -234
- data/lib/scylla/resources.rb +10 -10
- data/scylla.gemspec +17 -40
- data/source_texts/catalan.txt +28 -28
- data/source_texts/danish.txt +62 -62
- data/source_texts/english.txt +10 -10
- data/source_texts/french.txt +470 -77
- data/source_texts/japanese.txt +453 -199
- data/source_texts/norwegian.txt +96 -63
- data/source_texts/spanish.txt +269 -269
- data/test/classifier_test.rb +2 -2
- data/test/fixtures/lms/13375p33k.lm +156 -156
- data/test/fixtures/lms/danish.lm +137 -137
- data/test/fixtures/lms/english.lm +207 -207
- data/test/fixtures/lms/french.lm +400 -400
- data/test/fixtures/lms/hindi.lm +400 -0
- data/test/fixtures/lms/italian.lm +400 -0
- data/test/fixtures/lms/japanese.lm +400 -400
- data/test/fixtures/lms/norwegian.lm +400 -0
- data/test/fixtures/lms/spanish.lm +98 -98
- data/test/fixtures/source_texts/danish.txt +62 -62
- data/test/fixtures/source_texts/english.txt +10 -10
- data/test/fixtures/source_texts/french.txt +470 -77
- data/test/fixtures/source_texts/hindi.txt +199 -0
- data/test/fixtures/source_texts/italian.txt +120 -0
- data/test/fixtures/source_texts/japanese.txt +453 -199
- data/test/fixtures/source_texts/norwegian.txt +190 -0
- data/test/fixtures/source_texts/spanish.txt +269 -269
- data/test/fixtures/test_languages/english +61 -0
- data/test/fixtures/test_languages/french +0 -0
- data/test/fixtures/test_languages/german +29 -0
- data/test/fixtures/test_languages/hindi +3 -0
- data/test/fixtures/test_languages/italian +6 -0
- data/test/fixtures/test_languages/japanese +79 -0
- data/test/fixtures/test_languages/norwegian +14 -0
- data/test/fixtures/test_languages/spanish +22 -0
- data/test/generator_test.rb +0 -1
- data/test/language_test.rb +28 -0
- metadata +20 -43
- data/lib/scylla/lms/esperanto.lm +0 -400
- data/lib/scylla/lms/hungarian.lm +0 -400
- data/lib/scylla/lms/irish.lm +0 -400
- data/lib/scylla/lms/kannada.lm +0 -400
- data/lib/scylla/lms/latin.lm +0 -400
- data/lib/scylla/lms/malay.lm +0 -400
- data/lib/scylla/lms/marathi.lm +0 -400
- data/lib/scylla/lms/mingo.lm +0 -400
- data/lib/scylla/lms/nepali.lm +0 -400
- data/lib/scylla/lms/quechua.lm +0 -400
- data/lib/scylla/lms/rumantsch.lm +0 -400
- data/lib/scylla/lms/sanskrit.lm +0 -400
- data/lib/scylla/lms/scots_gaelic.lm +0 -400
- data/lib/scylla/lms/serbian.lm +0 -400
- data/lib/scylla/lms/swahili.lm +0 -400
- data/lib/scylla/lms/tamil.lm +0 -400
- data/lib/scylla/lms/ukrainian.lm +0 -400
- data/lib/scylla/lms/yiddish.lm +0 -400
- data/source_texts/esperanto.txt +0 -199
- data/source_texts/hungarian.txt +0 -102
- data/source_texts/irish.txt +0 -209
- data/source_texts/kannada.txt +0 -283
- data/source_texts/latin.txt +0 -120
- data/source_texts/malay.txt +0 -108
- data/source_texts/marathi.txt +0 -100
- data/source_texts/mingo.txt +0 -146
- data/source_texts/nepali.txt +0 -131
- data/source_texts/quechua.txt +0 -108
- data/source_texts/rumantsch.txt +0 -110
- data/source_texts/sanskrit.txt +0 -135
- data/source_texts/scots_gaelic.txt +0 -93
- data/source_texts/serbian.txt +0 -121
- data/source_texts/swahili.txt +0 -120
- data/source_texts/tamil.txt +0 -167
- data/source_texts/ukrainian.txt +0 -214
- data/source_texts/yiddish-utf.txt +0 -83
- data/test/fixtures/lms/kannada.lm +0 -400
- data/test/fixtures/source_texts/kannada.txt +0 -283
data/lib/scylla/lms/bulgarian.lm
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
� 73879
|
|
2
|
-
_
|
|
2
|
+
_ 41744
|
|
3
3
|
� 26019
|
|
4
|
-
_�
|
|
4
|
+
_� 13141
|
|
5
5
|
� 11448
|
|
6
6
|
а 11447
|
|
7
7
|
и 10454
|
|
8
8
|
� 10454
|
|
9
|
-
е 8498
|
|
10
9
|
� 8498
|
|
10
|
+
е 8498
|
|
11
11
|
� 8354
|
|
12
12
|
о 8353
|
|
13
13
|
т 7459
|
|
@@ -20,31 +20,32 @@ _
|
|
|
20
20
|
р 5460
|
|
21
21
|
т� 5390
|
|
22
22
|
�� 5390
|
|
23
|
-
� 5070
|
|
24
23
|
с 5070
|
|
24
|
+
� 5070
|
|
25
25
|
�� 4637
|
|
26
26
|
р� 4636
|
|
27
|
-
�� 4209
|
|
28
27
|
е� 4209
|
|
28
|
+
�� 4209
|
|
29
29
|
� 4121
|
|
30
30
|
в 4119
|
|
31
31
|
�� 4029
|
|
32
32
|
а� 4029
|
|
33
|
-
�_
|
|
34
|
-
а_
|
|
35
|
-
о� 3839
|
|
33
|
+
�_ 3970
|
|
34
|
+
а_ 3969
|
|
36
35
|
�� 3839
|
|
36
|
+
о� 3839
|
|
37
|
+
_� 3761
|
|
37
38
|
и� 3688
|
|
38
39
|
�� 3688
|
|
39
40
|
� 3685
|
|
40
41
|
к 3685
|
|
41
|
-
_� 3683
|
|
42
42
|
и� 3582
|
|
43
43
|
�� 3582
|
|
44
|
-
� 3450
|
|
45
44
|
л 3450
|
|
46
|
-
|
|
45
|
+
� 3450
|
|
47
46
|
� 3071
|
|
47
|
+
д 3071
|
|
48
|
+
__ 2975
|
|
48
49
|
�� 2942
|
|
49
50
|
а� 2942
|
|
50
51
|
�� 2907
|
|
@@ -53,132 +54,131 @@ _
|
|
|
53
54
|
�� 2905
|
|
54
55
|
�� 2813
|
|
55
56
|
с� 2813
|
|
56
|
-
к� 2793
|
|
57
57
|
�� 2793
|
|
58
|
-
|
|
59
|
-
и_
|
|
60
|
-
|
|
61
|
-
п 2507
|
|
58
|
+
к� 2793
|
|
59
|
+
и_ 2640
|
|
60
|
+
�_ 2640
|
|
62
61
|
� 2507
|
|
62
|
+
п 2507
|
|
63
63
|
д� 2355
|
|
64
64
|
�� 2355
|
|
65
65
|
�� 2274
|
|
66
66
|
о� 2274
|
|
67
67
|
на 2261
|
|
68
68
|
�а 2261
|
|
69
|
+
е_ 2194
|
|
70
|
+
�_ 2194
|
|
69
71
|
з 2176
|
|
70
72
|
� 2176
|
|
71
|
-
е_ 2116
|
|
72
|
-
�_ 2116
|
|
73
|
-
я 2001
|
|
74
73
|
� 2001
|
|
74
|
+
я 2001
|
|
75
75
|
�� 1974
|
|
76
76
|
с� 1974
|
|
77
|
-
�_
|
|
78
|
-
о_
|
|
79
|
-
_с
|
|
77
|
+
�_ 1966
|
|
78
|
+
о_ 1965
|
|
79
|
+
_с 1948
|
|
80
80
|
м 1872
|
|
81
81
|
� 1872
|
|
82
82
|
�� 1806
|
|
83
83
|
е� 1806
|
|
84
|
-
з� 1756
|
|
85
84
|
�� 1756
|
|
85
|
+
з� 1756
|
|
86
86
|
_н 1747
|
|
87
87
|
_н� 1660
|
|
88
|
-
ни 1604
|
|
89
88
|
�и 1604
|
|
90
|
-
|
|
89
|
+
ни 1604
|
|
91
90
|
� 1598
|
|
92
|
-
|
|
91
|
+
ъ 1598
|
|
93
92
|
�� 1546
|
|
94
|
-
|
|
93
|
+
м� 1546
|
|
95
94
|
� 1534
|
|
96
|
-
|
|
95
|
+
г 1534
|
|
97
96
|
�т 1463
|
|
97
|
+
ст 1463
|
|
98
98
|
, 1462
|
|
99
|
-
п� 1461
|
|
100
99
|
�� 1461
|
|
100
|
+
п� 1461
|
|
101
101
|
�т 1457
|
|
102
102
|
ат 1457
|
|
103
|
-
,_
|
|
103
|
+
,_ 1453
|
|
104
104
|
_п 1435
|
|
105
|
-
то 1405
|
|
106
105
|
�о 1405
|
|
107
|
-
|
|
106
|
+
то 1405
|
|
108
107
|
�а_ 1376
|
|
109
|
-
|
|
108
|
+
на_ 1376
|
|
110
109
|
�а 1372
|
|
111
|
-
|
|
110
|
+
ра 1372
|
|
112
111
|
те 1359
|
|
112
|
+
�е 1359
|
|
113
113
|
_и 1344
|
|
114
|
-
�н 1332
|
|
115
114
|
ан 1332
|
|
116
|
-
|
|
115
|
+
�н 1332
|
|
117
116
|
г� 1296
|
|
118
117
|
_на 1296
|
|
119
|
-
|
|
118
|
+
�� 1296
|
|
120
119
|
та 1264
|
|
120
|
+
�а 1264
|
|
121
121
|
� 1235
|
|
122
122
|
б 1234
|
|
123
|
-
�н 1178
|
|
124
123
|
ен 1178
|
|
124
|
+
�н 1178
|
|
125
125
|
� 1167
|
|
126
126
|
у 1165
|
|
127
127
|
�_ 1161
|
|
128
128
|
т_ 1161
|
|
129
|
-
ия 1149
|
|
130
129
|
�я 1149
|
|
130
|
+
ия 1149
|
|
131
131
|
ч 1105
|
|
132
132
|
� 1105
|
|
133
133
|
_в 1095
|
|
134
|
-
ат� 1091
|
|
135
134
|
�т� 1091
|
|
136
|
-
|
|
135
|
+
ат� 1091
|
|
137
136
|
�а 1082
|
|
138
|
-
ва 1082
|
|
139
137
|
�т 1082
|
|
138
|
+
ит 1082
|
|
139
|
+
ва 1082
|
|
140
140
|
ре 1077
|
|
141
141
|
�е 1077
|
|
142
|
-
|
|
142
|
+
_с� 1062
|
|
143
143
|
�� 1059
|
|
144
|
-
|
|
145
|
-
�� 1037
|
|
144
|
+
ч� 1059
|
|
146
145
|
п� 1037
|
|
147
|
-
|
|
148
|
-
ан� 1034
|
|
149
|
-
�т� 1034
|
|
146
|
+
�� 1037
|
|
150
147
|
ит� 1034
|
|
148
|
+
�т� 1034
|
|
149
|
+
ан� 1034
|
|
150
|
+
�н� 1034
|
|
151
151
|
. 1032
|
|
152
|
-
�а� 1015
|
|
153
152
|
ра� 1015
|
|
154
|
-
|
|
153
|
+
�а� 1015
|
|
155
154
|
�о 1010
|
|
156
|
-
|
|
155
|
+
но 1010
|
|
157
156
|
ст� 1007
|
|
157
|
+
�т� 1007
|
|
158
158
|
_о 957
|
|
159
|
-
�� 927
|
|
160
159
|
ъ� 927
|
|
160
|
+
�� 927
|
|
161
161
|
ки 919
|
|
162
162
|
�и 919
|
|
163
163
|
_д 889
|
|
164
|
-
�е� 882
|
|
165
|
-
то_ 882
|
|
166
|
-
ре� 882
|
|
167
164
|
�о_ 882
|
|
168
|
-
|
|
165
|
+
ре� 882
|
|
166
|
+
то_ 882
|
|
167
|
+
�е� 882
|
|
169
168
|
ри 874
|
|
169
|
+
�и 874
|
|
170
170
|
ли 873
|
|
171
171
|
�и 873
|
|
172
|
-
�к 872
|
|
173
172
|
ск 872
|
|
173
|
+
�к 872
|
|
174
174
|
�т 863
|
|
175
175
|
от 863
|
|
176
176
|
пр 857
|
|
177
177
|
�р 857
|
|
178
|
-
ск� 853
|
|
179
178
|
�к� 853
|
|
180
|
-
|
|
179
|
+
ск� 853
|
|
181
180
|
пр� 840
|
|
181
|
+
�р� 840
|
|
182
182
|
�а 832
|
|
183
183
|
ка 832
|
|
184
184
|
�� 816
|
|
@@ -191,54 +191,54 @@ _д 889
|
|
|
191
191
|
ко 778
|
|
192
192
|
т� 769
|
|
193
193
|
�� 769
|
|
194
|
-
�о 767
|
|
195
194
|
по 767
|
|
195
|
+
�о 767
|
|
196
196
|
� 756
|
|
197
197
|
ц 756
|
|
198
|
-
не 755
|
|
199
198
|
�е 755
|
|
199
|
+
не 755
|
|
200
200
|
�те 744
|
|
201
201
|
�а 736
|
|
202
202
|
да 736
|
|
203
|
-
�� 731
|
|
204
203
|
б� 731
|
|
204
|
+
�� 731
|
|
205
205
|
_п� 730
|
|
206
|
+
._ 729
|
|
206
207
|
_и_ 725
|
|
207
208
|
_к 720
|
|
208
209
|
_с� 716
|
|
209
210
|
_е 711
|
|
210
|
-
у� 706
|
|
211
211
|
�� 706
|
|
212
|
-
|
|
212
|
+
у� 706
|
|
213
213
|
�� 704
|
|
214
|
+
ц� 704
|
|
214
215
|
_п� 703
|
|
215
216
|
�н� 698
|
|
216
217
|
ен� 698
|
|
217
218
|
�е_ 684
|
|
218
219
|
те_ 684
|
|
219
|
-
�р 683
|
|
220
220
|
ар 683
|
|
221
|
-
|
|
221
|
+
�р 683
|
|
222
222
|
�в 682
|
|
223
|
-
|
|
224
|
-
ъ� 671
|
|
223
|
+
ов 682
|
|
225
224
|
_о� 671
|
|
225
|
+
ъ� 671
|
|
226
226
|
�� 671
|
|
227
|
-
й 670
|
|
228
227
|
� 670
|
|
229
228
|
_д� 670
|
|
229
|
+
й 670
|
|
230
230
|
�о 667
|
|
231
231
|
ро 667
|
|
232
|
-
ед� 659
|
|
233
232
|
�д� 659
|
|
234
|
-
|
|
233
|
+
ед� 659
|
|
235
234
|
�� 643
|
|
236
|
-
|
|
235
|
+
в� 643
|
|
237
236
|
�_ 640
|
|
237
|
+
я_ 640
|
|
238
238
|
�� 635
|
|
239
239
|
р� 634
|
|
240
|
-
к� 628
|
|
241
240
|
�� 628
|
|
241
|
+
к� 628
|
|
242
242
|
ж 627
|
|
243
243
|
� 627
|
|
244
244
|
_пр 620
|
|
@@ -248,113 +248,113 @@ _пр 620
|
|
|
248
248
|
�а_ 611
|
|
249
249
|
ик 599
|
|
250
250
|
�к 599
|
|
251
|
-
ов� 598
|
|
252
251
|
�в� 598
|
|
253
|
-
|
|
252
|
+
ов� 598
|
|
254
253
|
�� 596
|
|
254
|
+
ж� 596
|
|
255
255
|
_к� 591
|
|
256
|
-
ни� 581
|
|
257
256
|
�и� 581
|
|
257
|
+
ни� 581
|
|
258
258
|
_и� 580
|
|
259
259
|
_по 568
|
|
260
260
|
за 565
|
|
261
261
|
�а 565
|
|
262
|
-
ия_ 562
|
|
263
262
|
�� 562
|
|
264
|
-
�я_ 562
|
|
265
263
|
я� 562
|
|
264
|
+
ия_ 562
|
|
265
|
+
�я_ 562
|
|
266
266
|
ве 561
|
|
267
267
|
�е 561
|
|
268
|
-
ел 559
|
|
269
268
|
�л 559
|
|
269
|
+
ел 559
|
|
270
270
|
�т� 558
|
|
271
271
|
ет� 558
|
|
272
|
-
се 551
|
|
273
272
|
�е 551
|
|
273
|
+
се 551
|
|
274
274
|
�р 535
|
|
275
275
|
ор 535
|
|
276
|
+
�и 527
|
|
277
|
+
ди 527
|
|
276
278
|
�в 527
|
|
277
279
|
ав 527
|
|
278
|
-
ди 527
|
|
279
|
-
�и 527
|
|
280
|
-
�н 523
|
|
281
280
|
ин 523
|
|
282
|
-
|
|
281
|
+
�н 523
|
|
283
282
|
�ки 523
|
|
283
|
+
�_ 523
|
|
284
284
|
в_ 521
|
|
285
|
-
|
|
285
|
+
_р 519
|
|
286
286
|
�з 518
|
|
287
|
+
из 518
|
|
287
288
|
�е 507
|
|
288
289
|
ле 507
|
|
289
|
-
�� 506
|
|
290
|
-
д� 506
|
|
291
290
|
�о� 506
|
|
291
|
+
д� 506
|
|
292
|
+
�� 506
|
|
292
293
|
ро� 506
|
|
293
|
-
|
|
294
|
-
|
|
294
|
+
�и_ 502
|
|
295
|
+
ни_ 502
|
|
295
296
|
_з 500
|
|
297
|
+
я� 499
|
|
296
298
|
_з� 499
|
|
297
299
|
�� 499
|
|
298
|
-
я� 499
|
|
299
|
-
б� 497
|
|
300
300
|
�� 497
|
|
301
|
+
_р� 497
|
|
302
|
+
б� 497
|
|
301
303
|
�та 497
|
|
302
304
|
ко� 496
|
|
303
|
-
�о� 496
|
|
304
305
|
_от 496
|
|
305
|
-
|
|
306
|
+
�о� 496
|
|
306
307
|
во 486
|
|
308
|
+
�о 486
|
|
307
309
|
ци 485
|
|
308
310
|
�и 485
|
|
309
|
-
ле� 462
|
|
310
311
|
�е� 462
|
|
312
|
+
ле� 462
|
|
311
313
|
га 461
|
|
312
314
|
�а 461
|
|
313
|
-
�с 459
|
|
314
315
|
ос 459
|
|
316
|
+
�с 459
|
|
315
317
|
�р 454
|
|
316
318
|
тр 454
|
|
317
|
-
од 448
|
|
318
319
|
�д 448
|
|
319
|
-
|
|
320
|
-
�л 446
|
|
320
|
+
од 448
|
|
321
321
|
ла 446
|
|
322
|
+
�л 446
|
|
323
|
+
ол 446
|
|
322
324
|
�а 446
|
|
323
|
-
_р 446
|
|
324
325
|
на� 445
|
|
325
326
|
�а� 445
|
|
326
327
|
_т 444
|
|
327
|
-
�и� 440
|
|
328
328
|
ни� 440
|
|
329
|
+
�и� 440
|
|
329
330
|
�р� 437
|
|
330
331
|
ор� 437
|
|
331
|
-
|
|
332
|
+
ик� 435
|
|
332
333
|
�р 435
|
|
333
334
|
ер 435
|
|
334
|
-
|
|
335
|
+
�к� 435
|
|
335
336
|
_в_ 435
|
|
336
337
|
ри� 435
|
|
337
338
|
�и� 435
|
|
338
339
|
ар� 431
|
|
339
340
|
�р� 431
|
|
340
|
-
|
|
341
|
+
ес 428
|
|
341
342
|
�с 428
|
|
342
343
|
�а 428
|
|
343
|
-
|
|
344
|
-
�е� 426
|
|
344
|
+
ма 428
|
|
345
345
|
те� 426
|
|
346
|
+
�е� 426
|
|
346
347
|
�л� 424
|
|
347
|
-
_р� 424
|
|
348
348
|
ел� 424
|
|
349
|
-
об 424
|
|
350
349
|
�б 424
|
|
350
|
+
об 424
|
|
351
351
|
�в� 422
|
|
352
352
|
ав� 422
|
|
353
353
|
_за 416
|
|
354
354
|
тр� 415
|
|
355
355
|
�р� 415
|
|
356
|
-
�л 409
|
|
357
356
|
ал 409
|
|
357
|
+
�л 409
|
|
358
358
|
�� 407
|
|
359
359
|
л� 407
|
|
360
360
|
_м 406
|
|
@@ -362,39 +362,39 @@ _м 406
|
|
|
362
362
|
�т_ 405
|
|
363
363
|
� 404
|
|
364
364
|
щ 400
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
ва_ 399
|
|
365
|
+
ва_ 400
|
|
366
|
+
�а_ 400
|
|
368
367
|
�е 399
|
|
368
|
+
че 399
|
|
369
|
+
но_ 398
|
|
370
|
+
�ни 398
|
|
369
371
|
�з 398
|
|
372
|
+
�о_ 398
|
|
370
373
|
ез 398
|
|
371
|
-
�ни 398
|
|
372
|
-
но_ 397
|
|
373
|
-
�о_ 397
|
|
374
374
|
ви 395
|
|
375
375
|
�и 395
|
|
376
376
|
н_ 393
|
|
377
377
|
�_ 393
|
|
378
|
-
из� 392
|
|
379
378
|
�з� 392
|
|
380
|
-
|
|
381
|
-
�т 390
|
|
382
|
-
ме 390
|
|
379
|
+
из� 392
|
|
383
380
|
�е 390
|
|
381
|
+
ме 390
|
|
382
|
+
�т 390
|
|
383
|
+
ят 390
|
|
384
384
|
_се 388
|
|
385
|
-
�� 387
|
|
386
385
|
у� 387
|
|
386
|
+
�� 387
|
|
387
387
|
�е 385
|
|
388
388
|
ие 385
|
|
389
389
|
- 383
|
|
390
390
|
�л 374
|
|
391
391
|
ъл 374
|
|
392
|
-
�ан 372
|
|
393
392
|
�о� 372
|
|
394
393
|
по� 372
|
|
394
|
+
�ан 372
|
|
395
395
|
�е 368
|
|
396
396
|
де 368
|
|
397
|
-
ъл� 365
|
|
398
397
|
да_ 365
|
|
399
|
-
�а_ 365
|
|
400
398
|
�и� 365
|
|
399
|
+
�а_ 365
|
|
400
|
+
�л� 365
|