scylla 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. data/Gemfile +0 -1
  2. data/Gemfile.lock +0 -10
  3. data/README.rdoc +22 -0
  4. data/VERSION +1 -1
  5. data/bin/scylla +13 -0
  6. data/lib/scylla/classifier.rb +2 -2
  7. data/lib/scylla/generator.rb +1 -1
  8. data/lib/scylla/lms/13375P33K.lm +400 -0
  9. data/lib/scylla/lms/afrikaans.lm +400 -0
  10. data/lib/scylla/lms/arabic.lm +400 -0
  11. data/lib/scylla/lms/bulgarian.lm +400 -0
  12. data/lib/scylla/lms/catalan.lm +400 -0
  13. data/lib/scylla/lms/chinese.lm +400 -0
  14. data/lib/scylla/lms/danish.lm +400 -0
  15. data/lib/scylla/lms/english.lm +400 -0
  16. data/lib/scylla/lms/esperanto.lm +400 -0
  17. data/lib/scylla/lms/finnish.lm +400 -0
  18. data/lib/scylla/lms/french.lm +400 -0
  19. data/lib/scylla/lms/german.lm +400 -0
  20. data/lib/scylla/lms/greek-iso8859-7.lm +400 -0
  21. data/lib/scylla/lms/hebrew.lm +400 -0
  22. data/lib/scylla/lms/hindi.lm +400 -0
  23. data/lib/scylla/lms/hungarian.lm +400 -0
  24. data/lib/scylla/lms/icelandic.lm +400 -0
  25. data/lib/scylla/lms/indonesian.lm +400 -0
  26. data/lib/scylla/lms/irish.lm +400 -0
  27. data/lib/scylla/lms/italian.lm +400 -0
  28. data/lib/scylla/lms/japanese.lm +400 -0
  29. data/lib/scylla/lms/kannada.lm +400 -0
  30. data/lib/scylla/lms/korean.lm +400 -0
  31. data/lib/scylla/lms/latin.lm +400 -0
  32. data/lib/scylla/lms/malay.lm +400 -0
  33. data/lib/scylla/lms/marathi.lm +400 -0
  34. data/lib/scylla/lms/mingo.lm +400 -0
  35. data/lib/scylla/lms/nepali.lm +400 -0
  36. data/lib/scylla/lms/norwegian.lm +400 -0
  37. data/lib/scylla/lms/polish.lm +400 -0
  38. data/lib/scylla/lms/portuguese.lm +400 -0
  39. data/lib/scylla/lms/quechua.lm +400 -0
  40. data/lib/scylla/lms/romanian.lm +400 -0
  41. data/lib/scylla/lms/rumantsch.lm +400 -0
  42. data/lib/scylla/lms/russian.lm +400 -0
  43. data/lib/scylla/lms/sanskrit.lm +400 -0
  44. data/lib/scylla/lms/scots_gaelic.lm +400 -0
  45. data/lib/scylla/lms/serbian-ascii.lm +400 -0
  46. data/lib/scylla/lms/slovak-ascii.lm +400 -0
  47. data/lib/scylla/lms/slovenian-ascii.lm +400 -0
  48. data/lib/scylla/lms/spanish.lm +400 -0
  49. data/lib/scylla/lms/swahili.lm +400 -0
  50. data/lib/scylla/lms/swedish.lm +400 -0
  51. data/lib/scylla/lms/tagalog.lm +400 -0
  52. data/lib/scylla/lms/tamil.lm +400 -0
  53. data/lib/scylla/lms/thai.lm +400 -0
  54. data/lib/scylla/lms/turkish.lm +400 -0
  55. data/lib/scylla/lms/ukrainian-koi8_u.lm +400 -0
  56. data/lib/scylla/lms/vietnamese.lm +400 -0
  57. data/lib/scylla/lms/welsh.lm +400 -0
  58. data/lib/scylla/lms/yiddish-utf.lm +400 -0
  59. data/lib/scylla/loader.rb +8 -1
  60. data/scylla-0.1.0.gem +0 -0
  61. data/scylla.gemspec +69 -3
  62. data/source_texts/kannada.txt +283 -0
  63. data/test/classifier_test.rb +7 -0
  64. data/test/fixtures/lms/13375p33k.lm +400 -0
  65. data/test/fixtures/lms/danish.lm +400 -0
  66. data/test/fixtures/lms/english.lm +400 -0
  67. data/test/fixtures/lms/french.lm +400 -0
  68. data/test/fixtures/lms/german.lm +400 -0
  69. data/test/fixtures/lms/japanese.lm +400 -0
  70. data/test/fixtures/lms/kannada.lm +400 -0
  71. data/test/fixtures/lms/spanish.lm +400 -0
  72. data/test/fixtures/source_texts/13375P33K.txt +199 -0
  73. data/test/fixtures/source_texts/japanese.txt +199 -0
  74. data/test/fixtures/source_texts/kannada.txt +283 -0
  75. data/test/generator_test.rb +10 -7
  76. data/test/helper.rb +5 -6
  77. data/test/loader_test.rb +1 -0
  78. data/test/scylla_test.rb +1 -0
  79. metadata +78 -14
  80. data/source_texts/armenian.txt +0 -86
@@ -0,0 +1,400 @@
1
+ _ 1282
2
+ a 810
3
+ i 478
4
+ a_ 310
5
+ u 246
6
+ k 242
7
+ n 187
8
+ w 176
9
+ o 159
10
+ e 158
11
+ m 158
12
+ h 151
13
+ wa 147
14
+ l 131
15
+ s 130
16
+ t 125
17
+ i_ 116
18
+ _k 102
19
+ y 97
20
+ ku 76
21
+ wa_ 75
22
+ _w 75
23
+ li 73
24
+ r 72
25
+ z 69
26
+ ka 68
27
+ _wa 66
28
+ an 63
29
+ ma 61
30
+ ya 59
31
+ ik 58
32
+ b 57
33
+ o_ 57
34
+ _m 56
35
+ g 55
36
+ ha 54
37
+ na 52
38
+ j 51
39
+ am 50
40
+ d 49
41
+ f 48
42
+ at 48
43
+ is 47
44
+ hi 46
45
+ _ku 46
46
+ ya_ 45
47
+ u_ 43
48
+ al 41
49
+ ki 41
50
+ ta 40
51
+ sh 40
52
+ _y 39
53
+ ia 39
54
+ _ya 39
55
+ ri 39
56
+ _n 39
57
+ il 39
58
+ ti 37
59
+ e_ 36
60
+ A 35
61
+ ni 34
62
+ _h 34
63
+ . 34
64
+ zi 34
65
+ ng 33
66
+ _ya_ 33
67
+ ak 33
68
+ _wa_ 31
69
+ ba 30
70
+ ali 30
71
+ la 30
72
+ p 30
73
+ , 30
74
+ ar 29
75
+ na_ 28
76
+ v 28
77
+ mb 28
78
+ _ma 28
79
+ ika 28
80
+ ili 27
81
+ in 26
82
+ _a 26
83
+ _u 25
84
+ as 25
85
+ ati 25
86
+ kw 25
87
+ ia_ 25
88
+ ,_ 24
89
+ kwa 23
90
+ si 23
91
+ za 22
92
+ ni_ 22
93
+ _ka 22
94
+ ka_ 21
95
+ sa 21
96
+ hu 21
97
+ ana 21
98
+ zi_ 20
99
+ M 20
100
+ fa 20
101
+ un 20
102
+ _kw 20
103
+ ut 20
104
+ kat 20
105
+ c 20
106
+ se 20
107
+ ish 20
108
+ sha 19
109
+ di 19
110
+ _v 19
111
+ _s 19
112
+ ch 19
113
+ _l 19
114
+ mba 19
115
+ nd 19
116
+ kati 19
117
+ K 19
118
+ _na 19
119
+ _na_ 18
120
+ ja 18
121
+ amb 18
122
+ li_ 18
123
+ _kwa 18
124
+ ai 18
125
+ er 18
126
+ it 18
127
+ yo 18
128
+ ma_ 18
129
+ aa 17
130
+ ika_ 17
131
+ mi 16
132
+ rik 16
133
+ bi 16
134
+ fu 16
135
+ tu 16
136
+ W 16
137
+ kwa_ 16
138
+ a. 16
139
+ _hi 16
140
+ ama 16
141
+ on 16
142
+ aj 16
143
+ en 16
144
+ iki 15
145
+ _t 15
146
+ iw 15
147
+ he 15
148
+ iz 15
149
+ la_ 15
150
+ ao 15
151
+ ji 15
152
+ ko 15
153
+ amba 15
154
+ _M 15
155
+ ._ 15
156
+ if 15
157
+ tik 14
158
+ em 14
159
+ _ki 14
160
+ _A 14
161
+ ek 14
162
+ eri 14
163
+ za_ 14
164
+ um 14
165
+ iwa 14
166
+ _la 14
167
+ B 14
168
+ us 14
169
+ le 14
170
+ ul 13
171
+ tika 13
172
+ ny 13
173
+ wan 13
174
+ atika 13
175
+ uu 13
176
+ _i 13
177
+ N 13
178
+ go 13
179
+ atik 13
180
+ _z 13
181
+ ari 12
182
+ _ha 12
183
+ ini 12
184
+ iy 12
185
+ pi 12
186
+ _kwa_ 12
187
+ ye 12
188
+ tika_ 12
189
+ ua 12
190
+ _K 12
191
+ af 12
192
+ aw 12
193
+ vy 12
194
+ _j 11
195
+ uu_ 11
196
+ vi 11
197
+ ifa 11
198
+ au 11
199
+ kuwa 11
200
+ Wa 11
201
+ ra 11
202
+ I 11
203
+ ad 11
204
+ to 11
205
+ im 11
206
+ kut 11
207
+ aki 11
208
+ mu 11
209
+ ga 11
210
+ ke 11
211
+ az 11
212
+ ah 11
213
+ uwa 11
214
+ uw 11
215
+ ti_ 11
216
+ kuw 11
217
+ katik 11
218
+ ndi 11
219
+ ha_ 10
220
+ we 10
221
+ me 10
222
+ da 10
223
+ uz 10
224
+ ang 10
225
+ di_ 10
226
+ _B 10
227
+ _kat 10
228
+ _se 10
229
+ ay 10
230
+ a, 10
231
+ ab 10
232
+ _ali 10
233
+ uta 10
234
+ _kati 10
235
+ ao_ 10
236
+ rika 10
237
+ _al 10
238
+ ge 10
239
+ sa_ 10
240
+ ho 10
241
+ no 10
242
+ _W 10
243
+ es 10
244
+ _vy 10
245
+ uh 10
246
+ iyo 10
247
+ iku 10
248
+ kuu 9
249
+ O 9
250
+ T 9
251
+ hi_ 9
252
+ uo 9
253
+ ur 9
254
+ wana 9
255
+ ata 9
256
+ o. 9
257
+ lis 9
258
+ ani 9
259
+ cha 9
260
+ gu 9
261
+ wak 9
262
+ _ta 9
263
+ id 9
264
+ ngo 9
265
+ _vya 9
266
+ no_ 9
267
+ vya 9
268
+ awa 9
269
+ dh 9
270
+ _am 9
271
+ pa 9
272
+ _vi 9
273
+ kal 9
274
+ ez 9
275
+ je 9
276
+ ali_ 8
277
+ _za 8
278
+ uwa_ 8
279
+ a,_ 8
280
+ ili_ 8
281
+ atu 8
282
+ _Wa 8
283
+ P 8
284
+ ok 8
285
+ kam 8
286
+ Ka 8
287
+ zo 8
288
+ aji 8
289
+ mar 8
290
+ _ni 8
291
+ iwa_ 8
292
+ azi 8
293
+ ash 8
294
+ nga 8
295
+ tu_ 8
296
+ sha_ 8
297
+ kuwa_ 8
298
+ ib 8
299
+ isha 8
300
+ aka 8
301
+ ikal 8
302
+ wi 8
303
+ ana_ 8
304
+ ot 8
305
+ isa 8
306
+ aku 8
307
+ ag 8
308
+ su 8
309
+ kuu_ 8
310
+ wal 8
311
+ ama_ 8
312
+ ini_ 8
313
+ _mk 8
314
+ o, 8
315
+ sem 8
316
+ th 8
317
+ ema 8
318
+ kali 8
319
+ nz 8
320
+ mk 8
321
+ liw 7
322
+ heri 7
323
+ el 7
324
+ ju 7
325
+ Bw 7
326
+ ikali 7
327
+ erika 7
328
+ _la_ 7
329
+ fa_ 7
330
+ ita 7
331
+ nj 7
332
+ ho_ 7
333
+ S 7
334
+ mo 7
335
+ ara 7
336
+ ip 7
337
+ hus 7
338
+ uhu 7
339
+ _hiyo 7
340
+ _wak 7
341
+ hiy 7
342
+ R 7
343
+ _ili 7
344
+ erik 7
345
+ hiyo 7
346
+ ung 7
347
+ E 7
348
+ Wak 7
349
+ liwa 7
350
+ asi 7
351
+ uli 7
352
+ har 7
353
+ _hiy 7
354
+ her 7
355
+ rikal 7
356
+ _kam 7
357
+ ir 7
358
+ hil 7
359
+ sema 7
360
+ _za_ 7
361
+ om 7
362
+ _il 7
363
+ ndi_ 7
364
+ kis 7
365
+ uf 7
366
+ ati_ 7
367
+ ala 7
368
+ kali_ 7
369
+ nye 7
370
+ te 7
371
+ taifa 6
372
+ io 6
373
+ ifa_ 6
374
+ ema_ 6
375
+ undi_ 6
376
+ _Ka 6
377
+ yo_ 6
378
+ tia 6
379
+ asha 6
380
+ o._ 6
381
+ ala_ 6
382
+ izo 6
383
+ alis 6
384
+ dhi 6
385
+ ani_ 6
386
+ Bw_ 6
387
+ eh 6
388
+ ye_ 6
389
+ aifa 6
390
+ _N 6
391
+ sia 6
392
+ ahi 6
393
+ ria 6
394
+ undi 6
395
+ eke 6
396
+ _hu 6
397
+ tai 6
398
+ aif 6
399
+ izi 6
400
+ ong 6