scylla 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. data/Gemfile +0 -1
  2. data/Gemfile.lock +0 -10
  3. data/README.rdoc +22 -0
  4. data/VERSION +1 -1
  5. data/bin/scylla +13 -0
  6. data/lib/scylla/classifier.rb +2 -2
  7. data/lib/scylla/generator.rb +1 -1
  8. data/lib/scylla/lms/13375P33K.lm +400 -0
  9. data/lib/scylla/lms/afrikaans.lm +400 -0
  10. data/lib/scylla/lms/arabic.lm +400 -0
  11. data/lib/scylla/lms/bulgarian.lm +400 -0
  12. data/lib/scylla/lms/catalan.lm +400 -0
  13. data/lib/scylla/lms/chinese.lm +400 -0
  14. data/lib/scylla/lms/danish.lm +400 -0
  15. data/lib/scylla/lms/english.lm +400 -0
  16. data/lib/scylla/lms/esperanto.lm +400 -0
  17. data/lib/scylla/lms/finnish.lm +400 -0
  18. data/lib/scylla/lms/french.lm +400 -0
  19. data/lib/scylla/lms/german.lm +400 -0
  20. data/lib/scylla/lms/greek-iso8859-7.lm +400 -0
  21. data/lib/scylla/lms/hebrew.lm +400 -0
  22. data/lib/scylla/lms/hindi.lm +400 -0
  23. data/lib/scylla/lms/hungarian.lm +400 -0
  24. data/lib/scylla/lms/icelandic.lm +400 -0
  25. data/lib/scylla/lms/indonesian.lm +400 -0
  26. data/lib/scylla/lms/irish.lm +400 -0
  27. data/lib/scylla/lms/italian.lm +400 -0
  28. data/lib/scylla/lms/japanese.lm +400 -0
  29. data/lib/scylla/lms/kannada.lm +400 -0
  30. data/lib/scylla/lms/korean.lm +400 -0
  31. data/lib/scylla/lms/latin.lm +400 -0
  32. data/lib/scylla/lms/malay.lm +400 -0
  33. data/lib/scylla/lms/marathi.lm +400 -0
  34. data/lib/scylla/lms/mingo.lm +400 -0
  35. data/lib/scylla/lms/nepali.lm +400 -0
  36. data/lib/scylla/lms/norwegian.lm +400 -0
  37. data/lib/scylla/lms/polish.lm +400 -0
  38. data/lib/scylla/lms/portuguese.lm +400 -0
  39. data/lib/scylla/lms/quechua.lm +400 -0
  40. data/lib/scylla/lms/romanian.lm +400 -0
  41. data/lib/scylla/lms/rumantsch.lm +400 -0
  42. data/lib/scylla/lms/russian.lm +400 -0
  43. data/lib/scylla/lms/sanskrit.lm +400 -0
  44. data/lib/scylla/lms/scots_gaelic.lm +400 -0
  45. data/lib/scylla/lms/serbian-ascii.lm +400 -0
  46. data/lib/scylla/lms/slovak-ascii.lm +400 -0
  47. data/lib/scylla/lms/slovenian-ascii.lm +400 -0
  48. data/lib/scylla/lms/spanish.lm +400 -0
  49. data/lib/scylla/lms/swahili.lm +400 -0
  50. data/lib/scylla/lms/swedish.lm +400 -0
  51. data/lib/scylla/lms/tagalog.lm +400 -0
  52. data/lib/scylla/lms/tamil.lm +400 -0
  53. data/lib/scylla/lms/thai.lm +400 -0
  54. data/lib/scylla/lms/turkish.lm +400 -0
  55. data/lib/scylla/lms/ukrainian-koi8_u.lm +400 -0
  56. data/lib/scylla/lms/vietnamese.lm +400 -0
  57. data/lib/scylla/lms/welsh.lm +400 -0
  58. data/lib/scylla/lms/yiddish-utf.lm +400 -0
  59. data/lib/scylla/loader.rb +8 -1
  60. data/scylla-0.1.0.gem +0 -0
  61. data/scylla.gemspec +69 -3
  62. data/source_texts/kannada.txt +283 -0
  63. data/test/classifier_test.rb +7 -0
  64. data/test/fixtures/lms/13375p33k.lm +400 -0
  65. data/test/fixtures/lms/danish.lm +400 -0
  66. data/test/fixtures/lms/english.lm +400 -0
  67. data/test/fixtures/lms/french.lm +400 -0
  68. data/test/fixtures/lms/german.lm +400 -0
  69. data/test/fixtures/lms/japanese.lm +400 -0
  70. data/test/fixtures/lms/kannada.lm +400 -0
  71. data/test/fixtures/lms/spanish.lm +400 -0
  72. data/test/fixtures/source_texts/13375P33K.txt +199 -0
  73. data/test/fixtures/source_texts/japanese.txt +199 -0
  74. data/test/fixtures/source_texts/kannada.txt +283 -0
  75. data/test/generator_test.rb +10 -7
  76. data/test/helper.rb +5 -6
  77. data/test/loader_test.rb +1 -0
  78. data/test/scylla_test.rb +1 -0
  79. metadata +78 -14
  80. data/source_texts/armenian.txt +0 -86
@@ -0,0 +1,400 @@
1
+ _ 1998
2
+ n 418
3
+ h 330
4
+ i 203
5
+ t 202
6
+ c 191
7
+ g 183
8
+ ng 147
9
+ _t 134
10
+ a 123
11
+ _n 117
12
+ u 113
13
+ _c 107
14
+ nh 105
15
+ g_ 101
16
+ ng_ 101
17
+ i_ 101
18
+ n_ 99
19
+ � 92
20
+ m 74
21
+ o 70
22
+ r 65
23
+ � 64
24
+ . 64
25
+ � 63
26
+ l 63
27
+ _� 59
28
+ th 57
29
+ c_ 55
30
+ _th 53
31
+ h_ 52
32
+ s 51
33
+ _nh 50
34
+ t_ 47
35
+ , 47
36
+ y 47
37
+ _l 46
38
+ � 45
39
+ _s 45
40
+ a_ 44
41
+ nh_ 43
42
+ ch 42
43
+ ,_ 41
44
+ k 39
45
+ _h 38
46
+ ._ 38
47
+ u_ 37
48
+ hi 37
49
+ o_ 37
50
+ v 37
51
+ _k 36
52
+ _v 35
53
+ T 35
54
+ �_ 35
55
+ kh 34
56
+ _m 34
57
+ � 34
58
+ � 33
59
+ _kh 33
60
+ p 32
61
+ � 31
62
+ b 31
63
+ _ch 31
64
+ � 30
65
+ � 30
66
+ _b 30
67
+ d 30
68
+ _ng 29
69
+ __ 29
70
+ y_ 28
71
+ N 27
72
+ tr 27
73
+ _T 26
74
+ l� 26
75
+ e 26
76
+ _d 26
77
+ m_ 25
78
+ _g 25
79
+ � 25
80
+ � 24
81
+ _tr 24
82
+ _l� 24
83
+ � 24
84
+ � 24
85
+ � 23
86
+ an 23
87
+ � 23
88
+ �_ 22
89
+ �n 22
90
+ C 22
91
+ �n 22
92
+ ܩ 21
93
+ � 21
94
+ ph 21
95
+ �a 20
96
+ c�a 20
97
+ _p 20
98
+ H 20
99
+ h� 20
100
+ _ph 20
101
+ � 20
102
+ c� 20
103
+ �i 19
104
+ l�_ 19
105
+ _c�a 18
106
+ ho 18
107
+ �c 18
108
+ � 18
109
+ c�a_ 18
110
+ �a_ 18
111
+ _c� 18
112
+ ha 18
113
+ �ng 17
114
+ _r 17
115
+ _l�_ 17
116
+ c� 17
117
+ gi 17
118
+ c�_ 17
119
+ �n 17
120
+ _c� 16
121
+ � 16
122
+ _c�_ 16
123
+ �i_ 16
124
+ in 16
125
+ � 16
126
+ _c�a_ 16
127
+ � 16
128
+ " 15
129
+ i� 15
130
+ i� 15
131
+ _gi 15
132
+ nhi 15
133
+ q 15
134
+ uy 15
135
+ h� 15
136
+ qu 15
137
+ h� 15
138
+ g� 15
139
+ _C 15
140
+ � 15
141
+ �ng_ 15
142
+ ra 14
143
+ � 14
144
+ �n 14
145
+ � 14
146
+ _nhi 14
147
+ h� 14
148
+ .. 14
149
+ �t 14
150
+ gܩi 13
151
+ �nh 13
152
+ ܩi 13
153
+ �u 13
154
+ ܩi_ 13
155
+ _qu 13
156
+ �t_ 13
157
+ _q 13
158
+ �n 13
159
+ gܩ 13
160
+ � 13
161
+ v� 13
162
+ gܩi_ 13
163
+ �i 13
164
+ � 13
165
+ �n_ 13
166
+ �i_ 13
167
+ _v� 12
168
+ h�n 12
169
+ on 12
170
+ � 12
171
+ _N 12
172
+ i�u 12
173
+ hi�u 12
174
+ �u_ 12
175
+ hi� 12
176
+ inh 12
177
+ h�ng 12
178
+ x 11
179
+ nhi� 11
180
+ hi�u_ 11
181
+ kh� 11
182
+ kh�n 11
183
+ nhi�u 11
184
+ kh�ng 11
185
+ �n 11
186
+ c� 11
187
+ �_ 11
188
+ �t 11
189
+ ay 11
190
+ � 11
191
+ �o 11
192
+ sa 11
193
+ �n_ 11
194
+ _m� 11
195
+ m� 11
196
+ _c� 11
197
+ _sa 11
198
+ ܮ 11
199
+ i�u_ 11
200
+ � 11
201
+ _H 11
202
+ �c_ 10
203
+ � 10
204
+ �nh_ 10
205
+ �n 10
206
+ i� 10
207
+ �o_ 10
208
+ _kh�n 10
209
+ _m�t_ 10
210
+ ng� 10
211
+ �n 10
212
+ ngܩi 10
213
+ �ܮ 10
214
+ _ng� 10
215
+ �t 10
216
+ V 10
217
+ m�t 10
218
+ ngܩ 10
219
+ h�ng_ 10
220
+ �� 10
221
+ �ng 10
222
+ �t_ 10
223
+ _ngܩ 10
224
+ m�t_ 10
225
+ � 10
226
+ �ܮc 10
227
+ _nhi� 10
228
+ ܮc 10
229
+ �n 10
230
+ _m�t 10
231
+ _kh� 10
232
+ h� 10
233
+ �c 10
234
+ _x 10
235
+ _c�c 9
236
+ � 9
237
+ nh� 9
238
+ �_ 9
239
+ ho_ 9
240
+ ti 9
241
+ _�ܮc 9
242
+ : 9
243
+ �_ 9
244
+ inh_ 9
245
+ ch_ 9
246
+ _�ܮ 9
247
+ �ng_ 9
248
+ �n 9
249
+ L 9
250
+ f 9
251
+ ra_ 9
252
+ n� 9
253
+ _V 9
254
+ an_ 9
255
+ � 9
256
+ _�� 9
257
+ th� 9
258
+ ܮc_ 9
259
+ � 9
260
+ �ܮc_ 9
261
+ _n� 9
262
+ h�n 9
263
+ �c_ 9
264
+ ay_ 9
265
+ A 9
266
+ c�c 9
267
+ S 9
268
+ _nh� 9
269
+ _th� 9
270
+ �_ 8
271
+ rong_ 8
272
+ _cho 8
273
+ �c 8
274
+ _s� 8
275
+ ong_ 8
276
+ _th�_ 8
277
+ ro 8
278
+ ron 8
279
+ h� 8
280
+ h�_ 8
281
+ rong 8
282
+ p_ 8
283
+ �� 8
284
+ �ng_ 8
285
+ ܧ 8
286
+ un 8
287
+ au 8
288
+ s� 8
289
+ �_ 8
290
+ ܩn 8
291
+ �u 8
292
+ h� 8
293
+ ��_ 8
294
+ ong 8
295
+ G 8
296
+ ܩng_ 8
297
+ �n 8
298
+ �c_ 8
299
+ th�_ 8
300
+ _ti 8
301
+ ta 8
302
+ ܩng 8
303
+ hu 8
304
+ �nh 8
305
+ M 8
306
+ �y 8
307
+ Th 8
308
+ h�_ 8
309
+ �_ 8
310
+ cho 8
311
+ �n_ 8
312
+ �ng 8
313
+ ua 7
314
+ � 7
315
+ � 7
316
+ �n_ 7
317
+ ao_ 7
318
+ �ch 7
319
+ � 7
320
+ �nh_ 7
321
+ ar 7
322
+ � 7
323
+ i� 7
324
+ hay 7
325
+ ao 7
326
+ �t_ 7
327
+ �ch_ 7
328
+ anh 7
329
+ _L 7
330
+ _nh�_ 7
331
+ _Th 7
332
+ nh�_ 7
333
+ kh� 7
334
+ _�� 7
335
+ cho_ 7
336
+ vi 7
337
+ l� 7
338
+ �n 7
339
+ � 7
340
+ ai_ 7
341
+ _��_ 7
342
+ ? 7
343
+ Ng 7
344
+ �n 7
345
+ re 7
346
+ _kh� 7
347
+ �n 7
348
+ � 7
349
+ _cho_ 7
350
+ � 7
351
+ �i 7
352
+ ... 7
353
+ _vi 7
354
+ ai 7
355
+ _ra_ 6
356
+ ��n 6
357
+ l�i 6
358
+ s� 6
359
+ �n 6
360
+ _r�t_ 6
361
+ l�i_ 6
362
+ _� 6
363
+ h� 6
364
+ _th� 6
365
+ ��ng 6
366
+ _ra 6
367
+ Nj_ 6
368
+ trong 6
369
+ ngu 6
370
+ h� 6
371
+ _Nj 6
372
+ �u_ 6
373
+ Tr 6
374
+ _ljn_ 6
375
+ _r� 6
376
+ al 6
377
+ �_ 6
378
+ d� 6
379
+ r�t 6
380
+ � 6
381
+ ljn_ 6
382
+ Nj 6
383
+ lj 6
384
+ hay_ 6
385
+ _M 6
386
+ h�nh_ 6
387
+ _lj 6
388
+ �i_ 6
389
+ � 6
390
+ _Nj_ 6
391
+ _s� 6
392
+ s�_ 6
393
+ �� 6
394
+ _r�t 6
395
+ hi� 6
396
+ _s�_ 6
397
+ v�_ 6
398
+ gu 6
399
+ ljn 6
400
+ ia 6