scylla 0.4.3 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. data/Gemfile +1 -0
  2. data/Gemfile.lock +10 -0
  3. data/VERSION +1 -1
  4. data/lib/scylla/generator.rb +1 -1
  5. data/lib/scylla/lms/13375P33K.lm +156 -156
  6. data/lib/scylla/lms/arabic.lm +133 -133
  7. data/lib/scylla/lms/bulgarian.lm +122 -122
  8. data/lib/scylla/lms/catalan.lm +151 -151
  9. data/lib/scylla/lms/danish.lm +137 -137
  10. data/lib/scylla/lms/english.lm +207 -207
  11. data/lib/scylla/lms/french.lm +400 -400
  12. data/lib/scylla/lms/japanese.lm +400 -400
  13. data/lib/scylla/lms/korean.lm +233 -233
  14. data/lib/scylla/lms/norwegian.lm +398 -398
  15. data/lib/scylla/lms/spanish.lm +98 -98
  16. data/lib/scylla/lms/swedish.lm +123 -123
  17. data/lib/scylla/lms/tagalog.lm +223 -223
  18. data/lib/scylla/lms/welsh.lm +234 -234
  19. data/lib/scylla/resources.rb +10 -10
  20. data/scylla.gemspec +17 -40
  21. data/source_texts/catalan.txt +28 -28
  22. data/source_texts/danish.txt +62 -62
  23. data/source_texts/english.txt +10 -10
  24. data/source_texts/french.txt +470 -77
  25. data/source_texts/japanese.txt +453 -199
  26. data/source_texts/norwegian.txt +96 -63
  27. data/source_texts/spanish.txt +269 -269
  28. data/test/classifier_test.rb +2 -2
  29. data/test/fixtures/lms/13375p33k.lm +156 -156
  30. data/test/fixtures/lms/danish.lm +137 -137
  31. data/test/fixtures/lms/english.lm +207 -207
  32. data/test/fixtures/lms/french.lm +400 -400
  33. data/test/fixtures/lms/hindi.lm +400 -0
  34. data/test/fixtures/lms/italian.lm +400 -0
  35. data/test/fixtures/lms/japanese.lm +400 -400
  36. data/test/fixtures/lms/norwegian.lm +400 -0
  37. data/test/fixtures/lms/spanish.lm +98 -98
  38. data/test/fixtures/source_texts/danish.txt +62 -62
  39. data/test/fixtures/source_texts/english.txt +10 -10
  40. data/test/fixtures/source_texts/french.txt +470 -77
  41. data/test/fixtures/source_texts/hindi.txt +199 -0
  42. data/test/fixtures/source_texts/italian.txt +120 -0
  43. data/test/fixtures/source_texts/japanese.txt +453 -199
  44. data/test/fixtures/source_texts/norwegian.txt +190 -0
  45. data/test/fixtures/source_texts/spanish.txt +269 -269
  46. data/test/fixtures/test_languages/english +61 -0
  47. data/test/fixtures/test_languages/french +0 -0
  48. data/test/fixtures/test_languages/german +29 -0
  49. data/test/fixtures/test_languages/hindi +3 -0
  50. data/test/fixtures/test_languages/italian +6 -0
  51. data/test/fixtures/test_languages/japanese +79 -0
  52. data/test/fixtures/test_languages/norwegian +14 -0
  53. data/test/fixtures/test_languages/spanish +22 -0
  54. data/test/generator_test.rb +0 -1
  55. data/test/language_test.rb +28 -0
  56. metadata +20 -43
  57. data/lib/scylla/lms/esperanto.lm +0 -400
  58. data/lib/scylla/lms/hungarian.lm +0 -400
  59. data/lib/scylla/lms/irish.lm +0 -400
  60. data/lib/scylla/lms/kannada.lm +0 -400
  61. data/lib/scylla/lms/latin.lm +0 -400
  62. data/lib/scylla/lms/malay.lm +0 -400
  63. data/lib/scylla/lms/marathi.lm +0 -400
  64. data/lib/scylla/lms/mingo.lm +0 -400
  65. data/lib/scylla/lms/nepali.lm +0 -400
  66. data/lib/scylla/lms/quechua.lm +0 -400
  67. data/lib/scylla/lms/rumantsch.lm +0 -400
  68. data/lib/scylla/lms/sanskrit.lm +0 -400
  69. data/lib/scylla/lms/scots_gaelic.lm +0 -400
  70. data/lib/scylla/lms/serbian.lm +0 -400
  71. data/lib/scylla/lms/swahili.lm +0 -400
  72. data/lib/scylla/lms/tamil.lm +0 -400
  73. data/lib/scylla/lms/ukrainian.lm +0 -400
  74. data/lib/scylla/lms/yiddish.lm +0 -400
  75. data/source_texts/esperanto.txt +0 -199
  76. data/source_texts/hungarian.txt +0 -102
  77. data/source_texts/irish.txt +0 -209
  78. data/source_texts/kannada.txt +0 -283
  79. data/source_texts/latin.txt +0 -120
  80. data/source_texts/malay.txt +0 -108
  81. data/source_texts/marathi.txt +0 -100
  82. data/source_texts/mingo.txt +0 -146
  83. data/source_texts/nepali.txt +0 -131
  84. data/source_texts/quechua.txt +0 -108
  85. data/source_texts/rumantsch.txt +0 -110
  86. data/source_texts/sanskrit.txt +0 -135
  87. data/source_texts/scots_gaelic.txt +0 -93
  88. data/source_texts/serbian.txt +0 -121
  89. data/source_texts/swahili.txt +0 -120
  90. data/source_texts/tamil.txt +0 -167
  91. data/source_texts/ukrainian.txt +0 -214
  92. data/source_texts/yiddish-utf.txt +0 -83
  93. data/test/fixtures/lms/kannada.lm +0 -400
  94. data/test/fixtures/source_texts/kannada.txt +0 -283
@@ -1,4 +1,4 @@
1
- _ 1278
1
+ _ 1282
2
2
  � 290
3
3
  � 184
4
4
  � 181
@@ -11,7 +11,7 @@ _ 1278
11
11
  � 99
12
12
  � 90
13
13
  � 89
14
- __ 86
14
+ __ 87
15
15
  � 78
16
16
  _� 77
17
17
  � 75
@@ -24,377 +24,377 @@ _
24
24
  �� 65
25
25
  � 62
26
26
  � 55
27
- � 54
28
27
  � 54
28
+ � 54
29
29
  � 53
30
30
  � 52
31
31
  � 50
32
- � 49
33
32
  � 49
34
- 48
33
+ 49
35
34
  _� 48
35
+ � 48
36
36
  � 46
37
37
  � 45
38
38
  . 44
39
39
  � 40
40
- � 39
41
- ��. 39
42
40
  � 39
43
41
  �. 39
44
42
  � 39
43
+ � 39
44
+ ��. 39
45
45
  � 37
46
46
  �� 36
47
- � 35
48
47
  � 35
49
- �� 34
50
- _� 34
48
+ 35
51
49
  � 34
50
+ _� 34
51
+ �� 34
52
52
  � 33
53
53
  �_ 32
54
54
  � 32
55
- _� 31
55
+ _� 32
56
56
  �_ 31
57
57
  � 31
58
+ ��_ 29
58
59
  �� 29
60
+ �_ 29
59
61
  �� 29
60
- ��_ 29
61
62
  � 29
62
- ��_ 29
63
- �_ 29
64
- _� 27
63
+ ��_ 29
65
64
  _� 27
66
- _� 27
67
65
  �� 27
68
- �� 26
66
+ _� 27
67
+ _� 27
69
68
  � 26
70
- �� 25
71
- �� 25
72
- _� 25
73
- _� 25
69
+ �� 26
74
70
  �� 25
75
71
  _� 25
76
- _ 24
72
+ _25
73
+ _� 25
74
+ �� 25
75
+ �� 25
77
76
  �_ 24
78
77
  �_ 24
79
- �� 23
80
- � 23
81
- _� 23
78
+ �_ 24
82
79
  ��_ 23
83
- �� 23
84
80
  ��_ 23
85
81
  �� 23
82
+ �� 23
86
83
  � 23
84
+ �� 23
85
+ � 23
86
+ _� 23
87
87
  �� 22
88
- � 22
89
- � 22
90
88
  �� 22
91
- �� 22
92
89
  �_ 22
90
+ � 22
91
+ � 22
92
+ �� 22
93
93
  �� 22
94
+ _� 21
94
95
  ��_ 21
95
96
  _� 21
96
- �� 21
97
97
  �� 21
98
- _� 21
99
- � 20
100
- ��_ 20
98
+ �� 21
99
+ _� 20
101
100
  �_ 20
101
+ � 20
102
102
  ��_ 20
103
- _ 20
103
+ ��_ 20
104
+ � 19
105
+ _�� 19
104
106
  �� 19
105
107
  � 19
106
- � 19
107
108
  ._ 19
108
- _�� 19
109
109
  ִ 18
110
- _�� 18
111
- ��_ 18
112
- �� 18
113
110
  �_ 18
114
- �_ 18
115
- �_ 18
116
- � 18
117
111
  � 18
112
+ ��_ 18
118
113
  �� 18
114
+ �_ 18
115
+ ��_ 18
116
+ � 18
119
117
  � 18
120
- ��_ 18
121
- _ 17
122
- 17
123
- �� 17
118
+ �� 18
119
+ _�� 18
120
+ _ 18
124
121
  �_ 17
122
+ � 17
125
123
  �ִ 17
124
+ �� 17
125
+ _� 17
126
+ �� 16
126
127
  " 16
127
128
  �� 16
128
- � 16
129
129
  �� 16
130
130
  �� 16
131
- �� 16
131
+ 16
132
+ _� 15
133
+ , 15
134
+ ��_ 15
135
+ ' 15
132
136
  �� 15
133
- �� 15
134
- � 15
135
137
  � 15
136
- ��_ 15
137
- , 15
138
- �._ 15
139
- _� 15
138
+ �� 15
140
139
  ��._ 15
141
- ' 15
140
+ �._ 15
141
+ � 15
142
+ �� 14
143
+ �� 14
144
+ � 14
142
145
  �� 14
143
- _�� 14
144
- �� 14
145
- ��� 14
146
- � 14
146
+ �� 14
147
+ �� 14
147
148
  �� 14
149
+ ��� 14
148
150
  _�ִ 14
149
- �� 14
150
- �� 14
151
- �� 14
152
151
  � 14
153
- 14
154
- �� 14
155
- �� 14
156
- ִ� 13
157
- �ִ� 13
158
- ߴ 13
159
- ȸ 13
160
- �� 13
152
+ �� 14
153
+ �� 14
154
+ _�� 14
155
+ 14
161
156
  � 13
157
+ ȸ 13
162
158
  �� 13
163
- �� 13
164
159
  �� 13
160
+ �� 13
161
+ �ִ� 13
162
+ ߴ 13
163
+ ִ� 13
164
+ �� 13
165
165
  �� 13
166
166
  �_ 13
167
- � 12
168
167
  � 12
169
- �� 12
170
- � 12
171
- �ߴ� 12
168
+ 12
172
169
  ,_ 12
170
+ � 12
173
171
  � 12
172
+ �ߴ� 12
173
+ �_ 12
174
174
  � 12
175
175
  �ߴ 12
176
- �_ 12
177
176
  ߴ� 12
178
- ��_ 11
179
- ��� 11
180
- ��� 11
181
- _�� 11
182
- �� 11
183
- �ߴ�. 11
184
- ��_ 11
185
- ߴ�. 11
186
- �� 11
187
- _� 11
177
+ �� 12
178
+ �� 11
188
179
  _� 11
189
- _�� 11
190
180
  �� 11
181
+ �� 11
182
+ �ߴ�. 11
183
+ _�� 11
184
+ ��_ 11
185
+ ��_ 11
186
+ ��� 11
187
+ �� 11
191
188
  �� 11
189
+ _� 11
192
190
  ���� 11
191
+ �� 11
193
192
  _�� 11
194
- �� 11
195
- �� 11
196
- �� 10
193
+ �_ 11
194
+ ߴ�. 11
195
+ _�� 11
196
+ ��� 11
197
197
  _�ִ� 10
198
- ��� 10
199
- � 10
200
- �� 10
201
- �� 10
202
- �� 10
203
198
  � 10
199
+ ��� 10
204
200
  õ 10
205
- �� 10
201
+ �� 10
202
+ � 10
203
+ � 10
206
204
  ��� 10
207
- �_ 10
208
- _ 10
209
- ���� 10
210
- _�� 10
205
+ �� 10
206
+ � 10
207
+ �� 10
211
208
  �� 10
212
- � 10
213
- � 10
214
- �� 10
215
209
  �� 10
216
- �� 10
210
+ _�� 10
211
+ �� 10
212
+ �_ 10
213
+ ���� 10
214
+ �� 10
217
215
  �� 10
218
- ϰ 9
219
- �� 9
220
- ����_ 9
221
- � 9
222
- _� 9
216
+ �� 10
217
+ �� 10
223
218
  _�� 9
224
- � 9
225
- �� 9
219
+ � 9
220
+ ����_ 9
226
221
  ���_ 9
227
- �� 9
228
- �� 9
229
- � 9
230
222
  �� 9
231
- �� 9
223
+ _� 9
232
224
  �� 9
233
- �� 8
225
+ ϰ 9
226
+ � 9
227
+ �� 9
228
+ �� 9
229
+ �� 9
230
+ �� 9
231
+ �� 9
232
+ � 9
234
233
  �_ 8
235
- ���_ 8
236
234
  �� 8
237
- _ 8
238
- ����_ 8
239
- � 8
235
+ ���_ 8
236
+ 8
240
237
  � 8
241
- �� 8
242
- ġ 8
243
- �� 8
244
238
  �� 8
239
+ �� 8
240
+ � 8
241
+ ��� 8
242
+ �� 8
243
+ � 8
245
244
  ��_ 8
246
- �� 8
245
+ �� 8
247
246
  �� 8
248
- �� 8
249
- �� 8
247
+ �_ 8
248
+ 8
249
+ �� 8
250
+ �� 8
251
+ �_ 8
252
+ �� 8
253
+ ġ 8
250
254
  ��� 8
251
- �_ 8
252
- �� 8
253
255
  �_ 8
254
- �� 8
256
+ �� 8
255
257
  �� 8
256
- 8
257
- �� 8
258
- �_ 8
258
+ �� 8
259
259
  �ϰ 8
260
- �� 8
260
+ �_ 8
261
+ ����_ 8
261
262
  �� 8
262
- _' 8
263
+ _�� 8
264
+ �� 8
265
+ �� 8
263
266
  � 8
264
- 8
265
- � 8
266
- ��� 8
267
- �� 7
268
- �� 7
267
+ _' 8
269
268
  ���� 7
270
- �� 7
269
+ _�� 7
271
270
  ��� 7
271
+ �Ӵ 7
272
+ ��_ 7
272
273
  �� 7
273
- Ʈ 7
274
- _�� 7
275
- Ӵ� 7
274
+ ��� 7
276
275
  �Ӵ 7
277
- �Ӵ 7
278
- � 7
276
+ ����_ 7
279
277
  _��_ 7
280
- �Ӵ� 7
281
- �� 7
278
+ �� 7
282
279
  �_ 7
283
- ��Ӵ 7
284
- ��_ 7
285
- ��_ 7
286
- ��� 7
287
- �� 7
288
- ��_ 7
280
+ ��� 7
289
281
  ��_ 7
290
- �� 7
282
+ �� 7
291
283
  � 7
292
- ( 7
293
- ü 7
294
- ����_ 7
295
- _�� 7
296
- �� 7
297
- �� 7
298
- �ִ�. 7
299
- �� 7
300
- ��� 7
301
284
  �� 7
285
+ �Ӵ� 7
286
+ ��_ 7
287
+ ��Ӵ 7
302
288
  _�� 7
303
- _�� 7
304
- �� 7
289
+ �� 7
290
+ �� 7
305
291
  ��� 7
292
+ � 7
293
+ �� 7
294
+ �� 7
295
+ ü 7
296
+ �� 7
297
+ ��_ 7
298
+ ( 7
299
+ _�� 7
300
+ �� 7
306
301
  Ӵ 7
302
+ �� 7
303
+ Ӵ� 7
304
+ Ʈ 7
305
+ _�� 7
307
306
  �Ӵ� 7
308
- _�� 7
309
- ) 7
307
+ �ִ�. 7
310
308
  _��� 7
311
- ִ�. 7
309
+ �� 7
312
310
  �� 7
313
311
  ���_ 7
314
- �� 7
315
- 6
316
- _" 6
317
- �Բ 6
312
+ ) 7
313
+ ִ�. 7
314
+ �� 7
315
+ ��� 6
316
+ _�� 6
317
+ ȯ 6
318
318
  �� 6
319
+ �Բ 6
319
320
  �� 6
320
- _� 6
321
- �� 6
322
- �� 6
323
321
  �_ 6
324
- �� 6
325
- �� 6
326
- _�� 6
327
- ��_ 6
322
+ Բ� 6
323
+ �� 6
328
324
  �Բ�_ 6
329
- �� 6
330
- �� 6
331
- �� 6
332
- _�� 6
333
- 6
325
+ �� 6
326
+ .� 6
327
+ _� 6
328
+ �� 6
329
+ ��_ 6
330
+ _� 6
331
+ �� 6
334
332
  _�� 6
333
+ �� 6
334
+ _��� 6
335
+ ��_ 6
336
+ Ÿ 6
337
+ Բ 6
335
338
  �� 6
336
- �� 6
339
+ �� 6
340
+ � 6
341
+ �� 6
342
+ �� 6
343
+ _õ 6
344
+ �� 6
345
+ �� 6
346
+ _�� 6
347
+ _��� 6
348
+ �� 6
349
+ _" 6
337
350
  _�� 6
338
- ��� 6
339
- Բ� 6
340
- ��_ 6
341
351
  ��� 6
342
- ȯ 6
343
- �� 6
344
- _�� 6
345
- Բ�_ 6
346
- _��� 6
347
- �� 6
352
+ ��� 6
353
+ �� 6
354
+ �� 6
348
355
  ̴ 6
349
- ��� 6
350
- .� 6
351
- ��_ 6
352
- �� 6
353
- �Բ� 6
354
356
  > 6
355
- �� 6
356
- _�� 6
357
- _��� 6
357
+ Բ�_ 6
358
358
  �_ 6
359
- �_ 6
360
- ̵ 6
359
+ �� 6
361
360
  �� 6
361
+ �� 6
362
+ �Բ� 6
362
363
  �.� 6
363
- �� 6
364
- ��_ 6
364
+ �� 6
365
365
  �� 6
366
+ _�� 6
367
+ � 6
366
368
  � 6
367
- �� 6
368
- �_ 6
369
- �� 6
370
- _��_ 6
371
- �� 6
372
- �� 6
373
- < 6
369
+ ̵ 6
374
370
  �̴ 6
375
- �� 6
376
- �� 6
377
- 6
378
- Բ 6
371
+ �_ 6
372
+ _�� 6
373
+ �� 6
374
+ �� 6
379
375
  _�� 6
380
- _ 6
376
+ ��_ 6
377
+ ��_ 6
378
+ �� 6
379
+ _��_ 6
381
380
  ��.� 6
382
- �� 6
381
+ < 6
383
382
  � 6
384
- Ÿ 6
385
- �� 5
386
- �� 5
387
- �̴�. 5
388
- �� 5
389
- �� 5
390
- ���� 5
383
+ �� 6
384
+ �_ 6
391
385
  ־ 5
392
- �� 5
393
- �� 5
394
- ��ߴ 5
395
- �� 5
386
+ ��_ 5
387
+ �� 5
388
+ ��� 5
389
+ �� 5
390
+ ��ߴ� 5
396
391
  �� 5
392
+ _�� 5
393
+ �� 5
394
+ �� 5
395
+ ��_ 5
397
396
  �� 5
398
- ��� 5
399
- �� 5
400
- �� 5
397
+ ��ߴ 5
398
+ �_ 5
399
+ �ѱ 5
400
+ �� 5