scylla 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. data/Gemfile +0 -1
  2. data/Gemfile.lock +0 -10
  3. data/README.rdoc +22 -0
  4. data/VERSION +1 -1
  5. data/bin/scylla +13 -0
  6. data/lib/scylla/classifier.rb +2 -2
  7. data/lib/scylla/generator.rb +1 -1
  8. data/lib/scylla/lms/13375P33K.lm +400 -0
  9. data/lib/scylla/lms/afrikaans.lm +400 -0
  10. data/lib/scylla/lms/arabic.lm +400 -0
  11. data/lib/scylla/lms/bulgarian.lm +400 -0
  12. data/lib/scylla/lms/catalan.lm +400 -0
  13. data/lib/scylla/lms/chinese.lm +400 -0
  14. data/lib/scylla/lms/danish.lm +400 -0
  15. data/lib/scylla/lms/english.lm +400 -0
  16. data/lib/scylla/lms/esperanto.lm +400 -0
  17. data/lib/scylla/lms/finnish.lm +400 -0
  18. data/lib/scylla/lms/french.lm +400 -0
  19. data/lib/scylla/lms/german.lm +400 -0
  20. data/lib/scylla/lms/greek-iso8859-7.lm +400 -0
  21. data/lib/scylla/lms/hebrew.lm +400 -0
  22. data/lib/scylla/lms/hindi.lm +400 -0
  23. data/lib/scylla/lms/hungarian.lm +400 -0
  24. data/lib/scylla/lms/icelandic.lm +400 -0
  25. data/lib/scylla/lms/indonesian.lm +400 -0
  26. data/lib/scylla/lms/irish.lm +400 -0
  27. data/lib/scylla/lms/italian.lm +400 -0
  28. data/lib/scylla/lms/japanese.lm +400 -0
  29. data/lib/scylla/lms/kannada.lm +400 -0
  30. data/lib/scylla/lms/korean.lm +400 -0
  31. data/lib/scylla/lms/latin.lm +400 -0
  32. data/lib/scylla/lms/malay.lm +400 -0
  33. data/lib/scylla/lms/marathi.lm +400 -0
  34. data/lib/scylla/lms/mingo.lm +400 -0
  35. data/lib/scylla/lms/nepali.lm +400 -0
  36. data/lib/scylla/lms/norwegian.lm +400 -0
  37. data/lib/scylla/lms/polish.lm +400 -0
  38. data/lib/scylla/lms/portuguese.lm +400 -0
  39. data/lib/scylla/lms/quechua.lm +400 -0
  40. data/lib/scylla/lms/romanian.lm +400 -0
  41. data/lib/scylla/lms/rumantsch.lm +400 -0
  42. data/lib/scylla/lms/russian.lm +400 -0
  43. data/lib/scylla/lms/sanskrit.lm +400 -0
  44. data/lib/scylla/lms/scots_gaelic.lm +400 -0
  45. data/lib/scylla/lms/serbian-ascii.lm +400 -0
  46. data/lib/scylla/lms/slovak-ascii.lm +400 -0
  47. data/lib/scylla/lms/slovenian-ascii.lm +400 -0
  48. data/lib/scylla/lms/spanish.lm +400 -0
  49. data/lib/scylla/lms/swahili.lm +400 -0
  50. data/lib/scylla/lms/swedish.lm +400 -0
  51. data/lib/scylla/lms/tagalog.lm +400 -0
  52. data/lib/scylla/lms/tamil.lm +400 -0
  53. data/lib/scylla/lms/thai.lm +400 -0
  54. data/lib/scylla/lms/turkish.lm +400 -0
  55. data/lib/scylla/lms/ukrainian-koi8_u.lm +400 -0
  56. data/lib/scylla/lms/vietnamese.lm +400 -0
  57. data/lib/scylla/lms/welsh.lm +400 -0
  58. data/lib/scylla/lms/yiddish-utf.lm +400 -0
  59. data/lib/scylla/loader.rb +8 -1
  60. data/scylla-0.1.0.gem +0 -0
  61. data/scylla.gemspec +69 -3
  62. data/source_texts/kannada.txt +283 -0
  63. data/test/classifier_test.rb +7 -0
  64. data/test/fixtures/lms/13375p33k.lm +400 -0
  65. data/test/fixtures/lms/danish.lm +400 -0
  66. data/test/fixtures/lms/english.lm +400 -0
  67. data/test/fixtures/lms/french.lm +400 -0
  68. data/test/fixtures/lms/german.lm +400 -0
  69. data/test/fixtures/lms/japanese.lm +400 -0
  70. data/test/fixtures/lms/kannada.lm +400 -0
  71. data/test/fixtures/lms/spanish.lm +400 -0
  72. data/test/fixtures/source_texts/13375P33K.txt +199 -0
  73. data/test/fixtures/source_texts/japanese.txt +199 -0
  74. data/test/fixtures/source_texts/kannada.txt +283 -0
  75. data/test/generator_test.rb +10 -7
  76. data/test/helper.rb +5 -6
  77. data/test/loader_test.rb +1 -0
  78. data/test/scylla_test.rb +1 -0
  79. metadata +78 -14
  80. data/source_texts/armenian.txt +0 -86
@@ -0,0 +1,400 @@
1
+ _ 1780
2
+ e 577
3
+ n 391
4
+ r 299
5
+ t 268
6
+ a 248
7
+ s 208
8
+ o 194
9
+ l 175
10
+ i 173
11
+ n_ 158
12
+ g 152
13
+ k 150
14
+ r_ 145
15
+ en 135
16
+ e_ 128
17
+ m 126
18
+ d 120
19
+ h 117
20
+ er 116
21
+ _h 101
22
+ t_ 98
23
+ v 96
24
+ an 89
25
+ er_ 85
26
+ _s 81
27
+ u 80
28
+ en_ 76
29
+ f 68
30
+ . 67
31
+ g_ 65
32
+ � 63
33
+ p 62
34
+ ha 62
35
+ te 59
36
+ _ha 59
37
+ _. 58
38
+ et 57
39
+ or 54
40
+ de 54
41
+ , 53
42
+ _o 52
43
+ ne 52
44
+ _e 51
45
+ _, 51
46
+ ,_ 49
47
+ _,_ 47
48
+ an_ 47
49
+ et_ 45
50
+ nn 44
51
+ _f 44
52
+ ke 43
53
+ _d 42
54
+ _i 41
55
+ re 39
56
+ �_ 39
57
+ b 39
58
+ st 38
59
+ me 38
60
+ og 37
61
+ ar 37
62
+ om 37
63
+ m_ 37
64
+ se 36
65
+ el 34
66
+ _m 34
67
+ ge 34
68
+ _og 34
69
+ _v 33
70
+ in 32
71
+ og_ 32
72
+ ng 32
73
+ _og_ 31
74
+ _t 31
75
+ � 30
76
+ fo 30
77
+ han 30
78
+ _han 30
79
+ _a 30
80
+ _k 29
81
+ _de 29
82
+ for 29
83
+ il 29
84
+ le 28
85
+ H 28
86
+ sk 28
87
+ un 28
88
+ _b 27
89
+ l_ 27
90
+ i_ 27
91
+ _er 26
92
+ j 26
93
+ ve 25
94
+ _for 25
95
+ kk 25
96
+ _fo 25
97
+ al 25
98
+ om_ 24
99
+ ne_ 24
100
+ ti 24
101
+ _er_ 23
102
+ li 23
103
+ _g 23
104
+ han_ 23
105
+ _han_ 23
106
+ kke 22
107
+ tt 22
108
+ nne 22
109
+ va 21
110
+ is 21
111
+ rt 21
112
+ at 21
113
+ y 21
114
+ " 21
115
+ ar_ 21
116
+ _p 20
117
+ _n 20
118
+ _me 20
119
+ or_ 20
120
+ d_ 19
121
+ eg 19
122
+ ed 19
123
+ te_ 19
124
+ un_ 19
125
+ ke_ 18
126
+ �r 18
127
+ _l 18
128
+ vi 18
129
+ so 18
130
+ he 18
131
+ Ha 18
132
+ nge 18
133
+ es 18
134
+ ka 18
135
+ ll 17
136
+ det 17
137
+ il_ 17
138
+ pe 17
139
+ la 16
140
+ som 16
141
+ Han 16
142
+ ter 16
143
+ _so 16
144
+ hu 16
145
+ D 16
146
+ ig 16
147
+ ste 16
148
+ Han_ 15
149
+ s_ 15
150
+ for_ 15
151
+ _se 15
152
+ _som 15
153
+ til 15
154
+ ik 15
155
+ nd 15
156
+ am 15
157
+ "_ 15
158
+ ns 15
159
+ har 15
160
+ _ti 15
161
+ ten 14
162
+ har_ 14
163
+ enn 14
164
+ ikk 14
165
+ _har 14
166
+ ene 14
167
+ _� 14
168
+ det_ 14
169
+ _i_ 14
170
+ p� 14
171
+ gen 14
172
+ av 14
173
+ nt 14
174
+ _til 14
175
+ _som_ 13
176
+ hun 13
177
+ �r 13
178
+ _r 13
179
+ re_ 13
180
+ de_ 13
181
+ a_ 13
182
+ _ka 13
183
+ ut 13
184
+ _p� 13
185
+ mm 13
186
+ p�_ 13
187
+ v_ 13
188
+ _har_ 13
189
+ mme 13
190
+ som_ 13
191
+ eg_ 13
192
+ on 13
193
+ je 13
194
+ _hu 13
195
+ ta 13
196
+ hun_ 13
197
+ _ik 12
198
+ hen 12
199
+ no 12
200
+ kke_ 12
201
+ ko 12
202
+ De 12
203
+ _av 12
204
+ lig 12
205
+ til_ 12
206
+ ikke 12
207
+ _p�_ 12
208
+ _ikk 12
209
+ em 12
210
+ enne 12
211
+ _he 12
212
+ _hun 12
213
+ _hun_ 12
214
+ _ikke 12
215
+ _for_ 12
216
+ pen 11
217
+ tte 11
218
+ _en 11
219
+ lo 11
220
+ den_ 11
221
+ ni 11
222
+ _vi 11
223
+ S 11
224
+ den 11
225
+ nne_ 11
226
+ _til_ 11
227
+ ei 11
228
+ ham 11
229
+ tr 11
230
+ ing 11
231
+ _det 11
232
+ �r_ 11
233
+ ed_ 10
234
+ _hen 10
235
+ _" 10
236
+ inn 10
237
+ pen_ 10
238
+ ere 10
239
+ ten_ 10
240
+ ans 10
241
+ be 10
242
+ � 10
243
+ _u 10
244
+ _ham 10
245
+ _at 10
246
+ _st 10
247
+ am_ 10
248
+ k_ 10
249
+ it 10
250
+ ste_ 10
251
+ �r 10
252
+ ham_ 10
253
+ ikke_ 10
254
+ s� 9
255
+ rd 9
256
+ le_ 9
257
+ ra 9
258
+ da 9
259
+ _den 9
260
+ _et 9
261
+ _ve 9
262
+ av_ 9
263
+ els 9
264
+ _kan 9
265
+ _.H 9
266
+ men 9
267
+ rs 9
268
+ at_ 9
269
+ ord 9
270
+ ig_ 9
271
+ du 9
272
+ kan 9
273
+ s�_ 9
274
+ ak 9
275
+ _den_ 9
276
+ E 9
277
+ lt 9
278
+ ls 9
279
+ seg 9
280
+ .H 9
281
+ ang 9
282
+ oe 9
283
+ _ham_ 9
284
+ hv 9
285
+ ser 9
286
+ nni 9
287
+ est 9
288
+ eng 9
289
+ ld 9
290
+ med 9
291
+ .. 9
292
+ enne_ 8
293
+ rt_ 8
294
+ Unn 8
295
+ ske 8
296
+ Unni 8
297
+ ser_ 8
298
+ _in 8
299
+ _s� 8
300
+ ng_ 8
301
+ gan 8
302
+ _s�_ 8
303
+ _va 8
304
+ henne 8
305
+ nde 8
306
+ U 8
307
+ ga 8
308
+ _henn 8
309
+ sv 8
310
+ ss 8
311
+ _det_ 8
312
+ _kan_ 8
313
+ _be 8
314
+ ngen 8
315
+ sp 8
316
+ kan_ 8
317
+ ie 8
318
+ _med 8
319
+ lle 8
320
+ sa 8
321
+ kt 8
322
+ pp 8
323
+ _om 8
324
+ end 8
325
+ _�_ 8
326
+ _av_ 8
327
+ _at_ 8
328
+ Un 8
329
+ der 8
330
+ tt_ 8
331
+ henn 8
332
+ gen_ 7
333
+ mer_ 7
334
+ _.Ha 7
335
+ F 7
336
+ lan 7
337
+ ger 7
338
+ Hu 7
339
+ ek 7
340
+ inge 7
341
+ .Han_ 7
342
+ ner 7
343
+ ene_ 7
344
+ .Ha 7
345
+ ann 7
346
+ ._ 7
347
+ _.Han 7
348
+ .Han 7
349
+ si 7
350
+ var 7
351
+ sl 7
352
+ noe 7
353
+ vo 7
354
+ ro 7
355
+ lig_ 7
356
+ _seg 7
357
+ lv 7
358
+ o_ 7
359
+ ri 7
360
+ mer 7
361
+ ker 7
362
+ _no 7
363
+ bl 7
364
+ ort 7
365
+ _du 7
366
+ _hv 7
367
+ _med_ 7
368
+ gj 7
369
+ der_ 7
370
+ med_ 7
371
+ seg_ 7
372
+ men_ 7
373
+ _et_ 7
374
+ _om_ 7
375
+ ger_ 6
376
+ j� 6
377
+ nk 6
378
+ nen_ 6
379
+ mel 6
380
+ eli 6
381
+ dd 6
382
+ _tr 6
383
+ �r_ 6
384
+ nn_ 6
385
+ sva 6
386
+ ter_ 6
387
+ ell 6
388
+ ma 6
389
+ O 6
390
+ rg 6
391
+ _men 6
392
+ oen 6
393
+ ver 6
394
+ _ser 6
395
+ vet 6
396
+ gu 6
397
+ att 6
398
+ vil 6
399
+ g� 6
400
+ ye 6