scylla 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. data/Gemfile +0 -1
  2. data/Gemfile.lock +0 -10
  3. data/README.rdoc +22 -0
  4. data/VERSION +1 -1
  5. data/bin/scylla +13 -0
  6. data/lib/scylla/classifier.rb +2 -2
  7. data/lib/scylla/generator.rb +1 -1
  8. data/lib/scylla/lms/13375P33K.lm +400 -0
  9. data/lib/scylla/lms/afrikaans.lm +400 -0
  10. data/lib/scylla/lms/arabic.lm +400 -0
  11. data/lib/scylla/lms/bulgarian.lm +400 -0
  12. data/lib/scylla/lms/catalan.lm +400 -0
  13. data/lib/scylla/lms/chinese.lm +400 -0
  14. data/lib/scylla/lms/danish.lm +400 -0
  15. data/lib/scylla/lms/english.lm +400 -0
  16. data/lib/scylla/lms/esperanto.lm +400 -0
  17. data/lib/scylla/lms/finnish.lm +400 -0
  18. data/lib/scylla/lms/french.lm +400 -0
  19. data/lib/scylla/lms/german.lm +400 -0
  20. data/lib/scylla/lms/greek-iso8859-7.lm +400 -0
  21. data/lib/scylla/lms/hebrew.lm +400 -0
  22. data/lib/scylla/lms/hindi.lm +400 -0
  23. data/lib/scylla/lms/hungarian.lm +400 -0
  24. data/lib/scylla/lms/icelandic.lm +400 -0
  25. data/lib/scylla/lms/indonesian.lm +400 -0
  26. data/lib/scylla/lms/irish.lm +400 -0
  27. data/lib/scylla/lms/italian.lm +400 -0
  28. data/lib/scylla/lms/japanese.lm +400 -0
  29. data/lib/scylla/lms/kannada.lm +400 -0
  30. data/lib/scylla/lms/korean.lm +400 -0
  31. data/lib/scylla/lms/latin.lm +400 -0
  32. data/lib/scylla/lms/malay.lm +400 -0
  33. data/lib/scylla/lms/marathi.lm +400 -0
  34. data/lib/scylla/lms/mingo.lm +400 -0
  35. data/lib/scylla/lms/nepali.lm +400 -0
  36. data/lib/scylla/lms/norwegian.lm +400 -0
  37. data/lib/scylla/lms/polish.lm +400 -0
  38. data/lib/scylla/lms/portuguese.lm +400 -0
  39. data/lib/scylla/lms/quechua.lm +400 -0
  40. data/lib/scylla/lms/romanian.lm +400 -0
  41. data/lib/scylla/lms/rumantsch.lm +400 -0
  42. data/lib/scylla/lms/russian.lm +400 -0
  43. data/lib/scylla/lms/sanskrit.lm +400 -0
  44. data/lib/scylla/lms/scots_gaelic.lm +400 -0
  45. data/lib/scylla/lms/serbian-ascii.lm +400 -0
  46. data/lib/scylla/lms/slovak-ascii.lm +400 -0
  47. data/lib/scylla/lms/slovenian-ascii.lm +400 -0
  48. data/lib/scylla/lms/spanish.lm +400 -0
  49. data/lib/scylla/lms/swahili.lm +400 -0
  50. data/lib/scylla/lms/swedish.lm +400 -0
  51. data/lib/scylla/lms/tagalog.lm +400 -0
  52. data/lib/scylla/lms/tamil.lm +400 -0
  53. data/lib/scylla/lms/thai.lm +400 -0
  54. data/lib/scylla/lms/turkish.lm +400 -0
  55. data/lib/scylla/lms/ukrainian-koi8_u.lm +400 -0
  56. data/lib/scylla/lms/vietnamese.lm +400 -0
  57. data/lib/scylla/lms/welsh.lm +400 -0
  58. data/lib/scylla/lms/yiddish-utf.lm +400 -0
  59. data/lib/scylla/loader.rb +8 -1
  60. data/scylla-0.1.0.gem +0 -0
  61. data/scylla.gemspec +69 -3
  62. data/source_texts/kannada.txt +283 -0
  63. data/test/classifier_test.rb +7 -0
  64. data/test/fixtures/lms/13375p33k.lm +400 -0
  65. data/test/fixtures/lms/danish.lm +400 -0
  66. data/test/fixtures/lms/english.lm +400 -0
  67. data/test/fixtures/lms/french.lm +400 -0
  68. data/test/fixtures/lms/german.lm +400 -0
  69. data/test/fixtures/lms/japanese.lm +400 -0
  70. data/test/fixtures/lms/kannada.lm +400 -0
  71. data/test/fixtures/lms/spanish.lm +400 -0
  72. data/test/fixtures/source_texts/13375P33K.txt +199 -0
  73. data/test/fixtures/source_texts/japanese.txt +199 -0
  74. data/test/fixtures/source_texts/kannada.txt +283 -0
  75. data/test/generator_test.rb +10 -7
  76. data/test/helper.rb +5 -6
  77. data/test/loader_test.rb +1 -0
  78. data/test/scylla_test.rb +1 -0
  79. metadata +78 -14
  80. data/source_texts/armenian.txt +0 -86
@@ -0,0 +1,400 @@
1
+ _ 1354
2
+ a 856
3
+ n 460
4
+ i 336
5
+ g 302
6
+ ng 210
7
+ t 183
8
+ g_ 175
9
+ o 173
10
+ ng_ 169
11
+ a_ 156
12
+ an 155
13
+ k 155
14
+ s 150
15
+ l 147
16
+ _n 132
17
+ m 131
18
+ y 105
19
+ p 105
20
+ u 103
21
+ na 101
22
+ ang 98
23
+ . 85
24
+ ang_ 83
25
+ la 80
26
+ in 79
27
+ _a 77
28
+ r 77
29
+ ka 72
30
+ b 72
31
+ sa 67
32
+ ak 67
33
+ _m 65
34
+ pa 65
35
+ _s 64
36
+ ma 64
37
+ h 63
38
+ _na 61
39
+ d 57
40
+ al 57
41
+ at 54
42
+ _p 52
43
+ ag 52
44
+ ta 49
45
+ _k 45
46
+ ni 45
47
+ t_ 44
48
+ _sa 43
49
+ n_ 42
50
+ " 42
51
+ ya 42
52
+ _ng 42
53
+ ay 41
54
+ it 40
55
+ _pa 40
56
+ ga 39
57
+ on 39
58
+ e 39
59
+ sa_ 38
60
+ y_ 37
61
+ iy 36
62
+ il 36
63
+ ala 35
64
+ o_ 35
65
+ w 35
66
+ _ng_ 34
67
+ am 34
68
+ _sa_ 33
69
+ _ma 32
70
+ na_ 32
71
+ i_ 32
72
+ ._ 32
73
+ _an 31
74
+ ra 31
75
+ , 31
76
+ ba 30
77
+ _ang 29
78
+ _ang_ 29
79
+ _ni 29
80
+ ar 29
81
+ iya 29
82
+ _t 29
83
+ _i 28
84
+ _ka 28
85
+ li 27
86
+ _l 27
87
+ ha 27
88
+ as 27
89
+ is 27
90
+ aka 27
91
+ hi 27
92
+ ,_ 27
93
+ ong 27
94
+ _na_ 26
95
+ ko 26
96
+ gi 26
97
+ ri 25
98
+ ap 25
99
+ to 25
100
+ ong_ 24
101
+ un 24
102
+ ah 24
103
+ lan 24
104
+ ay_ 24
105
+ ata 23
106
+ um 23
107
+ si 23
108
+ wa 23
109
+ o. 23
110
+ ing 23
111
+ di 23
112
+ ki 22
113
+ ti 22
114
+ s_ 22
115
+ _niy 21
116
+ niy 21
117
+ _d 21
118
+ ab 21
119
+ _niya 21
120
+ an_ 21
121
+ niya 21
122
+ mo 21
123
+ at_ 20
124
+ N 20
125
+ a. 20
126
+ ila 19
127
+ ot 19
128
+ yo 19
129
+ - 19
130
+ aw 19
131
+ nga 18
132
+ _ak 18
133
+ ig 18
134
+ _mo 18
135
+ lang 18
136
+ A 18
137
+ nag 17
138
+ ama 17
139
+ tu 17
140
+ in_ 16
141
+ _b 16
142
+ ali 16
143
+ ina 16
144
+ aki 16
145
+ ara 16
146
+ P 16
147
+ "_ 16
148
+ _at 15
149
+ ' 15
150
+ ing_ 15
151
+ lang_ 15
152
+ _si 15
153
+ ai 14
154
+ bi 14
155
+ aa 14
156
+ _r 14
157
+ ga_ 14
158
+ o._ 14
159
+ nd 14
160
+ ib 14
161
+ da 14
162
+ hin 13
163
+ ro 13
164
+ iya_ 13
165
+ lo 13
166
+ ito 13
167
+ _at_ 13
168
+ ik 13
169
+ _h 13
170
+ ako 13
171
+ ya_ 13
172
+ _ta 12
173
+ ul 12
174
+ la_ 12
175
+ _la 12
176
+ Na 12
177
+ mag 12
178
+ S 12
179
+ rin 12
180
+ a, 12
181
+ K 12
182
+ ahi 12
183
+ man 11
184
+ a,_ 11
185
+ _ri 11
186
+ k_ 11
187
+ lu 11
188
+ 'y 11
189
+ ." 11
190
+ gk 11
191
+ kin 11
192
+ mi 11
193
+ go 11
194
+ 'y_ 11
195
+ asa 11
196
+ M 11
197
+ _mag 11
198
+ ilan 11
199
+ apa 11
200
+ ung 11
201
+ _tu 11
202
+ pan 11
203
+ _di 11
204
+ ot_ 10
205
+ Ma 10
206
+ is_ 10
207
+ wal 10
208
+ nak 10
209
+ mg 10
210
+ mu 10
211
+ _is 10
212
+ yon 10
213
+ I 10
214
+ mga 10
215
+ ir 10
216
+ ilang 10
217
+ mga_ 10
218
+ su 10
219
+ uma 10
220
+ yan 10
221
+ agk 10
222
+ .. 10
223
+ pa_ 10
224
+ awa 10
225
+ Ka 10
226
+ wala 10
227
+ pi 10
228
+ akin 10
229
+ ari 10
230
+ _aki 9
231
+ An 9
232
+ _K 9
233
+ ini 9
234
+ pu 9
235
+ no 9
236
+ aba 9
237
+ _mg 9
238
+ n. 9
239
+ _mga_ 9
240
+ _mga 9
241
+ agka 9
242
+ gka 9
243
+ isa 9
244
+ n, 9
245
+ _A 9
246
+ pag 9
247
+ alan 9
248
+ nan 9
249
+ san 9
250
+ it_ 9
251
+ _N 9
252
+ _akin 9
253
+ ayo 8
254
+ _lu 8
255
+ naka 8
256
+ _wala 8
257
+ t. 8
258
+ aha 8
259
+ _ako 8
260
+ ob 8
261
+ si_ 8
262
+ mo_ 8
263
+ _nag 8
264
+ _wa 8
265
+ dib 8
266
+ niya_ 8
267
+ mat 8
268
+ alang 8
269
+ aman 8
270
+ ngi 8
271
+ ili 8
272
+ aga 8
273
+ as_ 8
274
+ Pa 8
275
+ l_ 8
276
+ di_ 8
277
+ tay 8
278
+ _Ka 8
279
+ iti 8
280
+ _w 8
281
+ aking 8
282
+ igi 8
283
+ abi 8
284
+ _ba 8
285
+ umi 8
286
+ _wal 8
287
+ a' 8
288
+ _P 8
289
+ king 8
290
+ tan 8
291
+ p_ 8
292
+ yang 8
293
+ pak 7
294
+ _isa 7
295
+ er 7
296
+ _rin 7
297
+ _pag 7
298
+ tak 7
299
+ anga 7
300
+ tum 7
301
+ B 7
302
+ tat 7
303
+ _An 7
304
+ _mo_ 7
305
+ ail 7
306
+ o, 7
307
+ king_ 7
308
+ ut 7
309
+ a._ 7
310
+ iyo 7
311
+ mata 7
312
+ n,_ 7
313
+ o' 7
314
+ kan 7
315
+ _pa_ 7
316
+ ala_ 7
317
+ bo 7
318
+ .n 7
319
+ ra_ 7
320
+ e_ 7
321
+ g- 7
322
+ oo 7
323
+ gin 7
324
+ tang 7
325
+ _ko 7
326
+ lis 7
327
+ ban 7
328
+ _si_ 7
329
+ ram 7
330
+ ip 7
331
+ ka_ 7
332
+ tin 7
333
+ ana 7
334
+ to_ 7
335
+ _M 7
336
+ ku 7
337
+ kat 7
338
+ kak 6
339
+ ula 6
340
+ ago 6
341
+ lis_ 6
342
+ _nga 6
343
+ .na 6
344
+ siy 6
345
+ isang 6
346
+ ung_ 6
347
+ par 6
348
+ _I 6
349
+ man_ 6
350
+ up 6
351
+ pal 6
352
+ nang 6
353
+ ig_ 6
354
+ _" 6
355
+ kah 6
356
+ aila 6
357
+ tang_ 6
358
+ agi 6
359
+ nt 6
360
+ siya 6
361
+ nit 6
362
+ isan 6
363
+ sang 6
364
+ _iy 6
365
+ _mu 6
366
+ o'y 6
367
+ Sa 6
368
+ _S 6
369
+ c 6
370
+ _tum 6
371
+ mb 6
372
+ kun 6
373
+ tata 6
374
+ _par 6
375
+ _ha 6
376
+ ? 6
377
+ mba 6
378
+ yang_ 6
379
+ gay 6
380
+ ap_ 6
381
+ bang 6
382
+ _Na 6
383
+ im 6
384
+ d_ 6
385
+ o,_ 6
386
+ os 6
387
+ ag- 6
388
+ una 6
389
+ _Ma 6
390
+ ak_ 6
391
+ _hi 6
392
+ st 6
393
+ o'y_ 6
394
+ ita 6
395
+ _ku 6
396
+ gan 6
397
+ walan 6
398
+ kaka 6
399
+ _iyo 6
400
+ o." 6