scylla 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. data/Gemfile +0 -1
  2. data/Gemfile.lock +0 -10
  3. data/README.rdoc +22 -0
  4. data/VERSION +1 -1
  5. data/bin/scylla +13 -0
  6. data/lib/scylla/classifier.rb +2 -2
  7. data/lib/scylla/generator.rb +1 -1
  8. data/lib/scylla/lms/13375P33K.lm +400 -0
  9. data/lib/scylla/lms/afrikaans.lm +400 -0
  10. data/lib/scylla/lms/arabic.lm +400 -0
  11. data/lib/scylla/lms/bulgarian.lm +400 -0
  12. data/lib/scylla/lms/catalan.lm +400 -0
  13. data/lib/scylla/lms/chinese.lm +400 -0
  14. data/lib/scylla/lms/danish.lm +400 -0
  15. data/lib/scylla/lms/english.lm +400 -0
  16. data/lib/scylla/lms/esperanto.lm +400 -0
  17. data/lib/scylla/lms/finnish.lm +400 -0
  18. data/lib/scylla/lms/french.lm +400 -0
  19. data/lib/scylla/lms/german.lm +400 -0
  20. data/lib/scylla/lms/greek-iso8859-7.lm +400 -0
  21. data/lib/scylla/lms/hebrew.lm +400 -0
  22. data/lib/scylla/lms/hindi.lm +400 -0
  23. data/lib/scylla/lms/hungarian.lm +400 -0
  24. data/lib/scylla/lms/icelandic.lm +400 -0
  25. data/lib/scylla/lms/indonesian.lm +400 -0
  26. data/lib/scylla/lms/irish.lm +400 -0
  27. data/lib/scylla/lms/italian.lm +400 -0
  28. data/lib/scylla/lms/japanese.lm +400 -0
  29. data/lib/scylla/lms/kannada.lm +400 -0
  30. data/lib/scylla/lms/korean.lm +400 -0
  31. data/lib/scylla/lms/latin.lm +400 -0
  32. data/lib/scylla/lms/malay.lm +400 -0
  33. data/lib/scylla/lms/marathi.lm +400 -0
  34. data/lib/scylla/lms/mingo.lm +400 -0
  35. data/lib/scylla/lms/nepali.lm +400 -0
  36. data/lib/scylla/lms/norwegian.lm +400 -0
  37. data/lib/scylla/lms/polish.lm +400 -0
  38. data/lib/scylla/lms/portuguese.lm +400 -0
  39. data/lib/scylla/lms/quechua.lm +400 -0
  40. data/lib/scylla/lms/romanian.lm +400 -0
  41. data/lib/scylla/lms/rumantsch.lm +400 -0
  42. data/lib/scylla/lms/russian.lm +400 -0
  43. data/lib/scylla/lms/sanskrit.lm +400 -0
  44. data/lib/scylla/lms/scots_gaelic.lm +400 -0
  45. data/lib/scylla/lms/serbian-ascii.lm +400 -0
  46. data/lib/scylla/lms/slovak-ascii.lm +400 -0
  47. data/lib/scylla/lms/slovenian-ascii.lm +400 -0
  48. data/lib/scylla/lms/spanish.lm +400 -0
  49. data/lib/scylla/lms/swahili.lm +400 -0
  50. data/lib/scylla/lms/swedish.lm +400 -0
  51. data/lib/scylla/lms/tagalog.lm +400 -0
  52. data/lib/scylla/lms/tamil.lm +400 -0
  53. data/lib/scylla/lms/thai.lm +400 -0
  54. data/lib/scylla/lms/turkish.lm +400 -0
  55. data/lib/scylla/lms/ukrainian-koi8_u.lm +400 -0
  56. data/lib/scylla/lms/vietnamese.lm +400 -0
  57. data/lib/scylla/lms/welsh.lm +400 -0
  58. data/lib/scylla/lms/yiddish-utf.lm +400 -0
  59. data/lib/scylla/loader.rb +8 -1
  60. data/scylla-0.1.0.gem +0 -0
  61. data/scylla.gemspec +69 -3
  62. data/source_texts/kannada.txt +283 -0
  63. data/test/classifier_test.rb +7 -0
  64. data/test/fixtures/lms/13375p33k.lm +400 -0
  65. data/test/fixtures/lms/danish.lm +400 -0
  66. data/test/fixtures/lms/english.lm +400 -0
  67. data/test/fixtures/lms/french.lm +400 -0
  68. data/test/fixtures/lms/german.lm +400 -0
  69. data/test/fixtures/lms/japanese.lm +400 -0
  70. data/test/fixtures/lms/kannada.lm +400 -0
  71. data/test/fixtures/lms/spanish.lm +400 -0
  72. data/test/fixtures/source_texts/13375P33K.txt +199 -0
  73. data/test/fixtures/source_texts/japanese.txt +199 -0
  74. data/test/fixtures/source_texts/kannada.txt +283 -0
  75. data/test/generator_test.rb +10 -7
  76. data/test/helper.rb +5 -6
  77. data/test/loader_test.rb +1 -0
  78. data/test/scylla_test.rb +1 -0
  79. metadata +78 -14
  80. data/source_texts/armenian.txt +0 -86
data/lib/scylla/lms/hindi.lm ADDED
@@ -0,0 +1,400 @@
1
+ � 28567
2
+ � 24888
3
+ � 23140
4
+ _ 9054
5
+ � 5565
6
+ � 5427
7
+ _� 4527
8
+ _� 4495
9
+ �� 2729
10
+ ा 2729
11
+ � 2729
12
+ � 2281
13
+ ् 2281
14
+ �� 2281
15
+ �� 2265
16
+ ्� 2265
17
+ ��� 2265
18
+ �� 2251
19
+ ��� 2251
20
+ ्� 2251
21
+ �� 2023
22
+ ा� 2023
23
+ ��� 2023
24
+ र 2005
25
+ �� 2005
26
+ � 2005
27
+ �� 1970
28
+ ा� 1970
29
+ ��� 1970
30
+ �� 1764
31
+ ि 1764
32
+ � 1764
33
+ र� 1706
34
+ ��� 1706
35
+ �� 1706
36
+ �� 1665
37
+ क 1665
38
+ � 1665
39
+ �� 1596
40
+ त 1596
41
+ �� 1516
42
+ ��� 1516
43
+ �� 1516
44
+ ि� 1516
45
+ ��� 1516
46
+ ि� 1516
47
+ � 1391
48
+ ��� 1390
49
+ �� 1390
50
+ क� 1390
51
+ �� 1381
52
+ न 1381
53
+ �� 1345
54
+ स 1345
55
+ � 1345
56
+ ��� 1264
57
+ �� 1264
58
+ स� 1264
59
+ �� 1226
60
+ त� 1200
61
+ ��� 1200
62
+ � 1181
63
+ े 1172
64
+ �� 1172
65
+ � 1144
66
+ व 1144
67
+ �� 1144
68
+ व� 1083
69
+ �� 1083
70
+ ��� 1083
71
+ �� 1075
72
+ ��� 1065
73
+ न� 1065
74
+ �� 963
75
+ ��� 963
76
+ र� 963
77
+ �� 920
78
+ व� 908
79
+ ��� 908
80
+ �� 908
81
+ ��� 894
82
+ त� 894
83
+ म 858
84
+ �� 858
85
+ � 858
86
+ � 853
87
+ य 836
88
+ �� 836
89
+ � 836
90
+ �� 829
91
+ प 829
92
+ �� 809
93
+ �� 792
94
+ म� 792
95
+ ��� 792
96
+ ��� 786
97
+ स� 786
98
+ �� 786
99
+ प� 785
100
+ ��� 785
101
+ �� 752
102
+ क� 752
103
+ ��� 752
104
+ र� 743
105
+ ��� 743
106
+ �� 743
107
+ �_ 706
108
+ ा_ 706
109
+ ��_ 706
110
+ � 705
111
+ �� 683
112
+ द 683
113
+ �� 671
114
+ ��� 663
115
+ द� 663
116
+ � 655
117
+ ु� 655
118
+ ु� 655
119
+ �� 655
120
+ �� 655
121
+ ��� 655
122
+ ु 655
123
+ ��� 655
124
+ �� 655
125
+ �� 638
126
+ क� 638
127
+ ��� 638
128
+ _स 634
129
+ _स� 634
130
+ �� 627
131
+ य� 621
132
+ �� 621
133
+ ��� 621
134
+ ��� 618
135
+ े� 618
136
+ ��� 613
137
+ �� 613
138
+ न� 613
139
+ �� 604
140
+ े� 595
141
+ ��� 595
142
+ �� 568
143
+ � 568
144
+ ह 568
145
+ �� 538
146
+ म� 538
147
+ ��� 538
148
+ े_ 532
149
+ ��_ 532
150
+ �_ 532
151
+ �� 527
152
+ ल 527
153
+ � 527
154
+ _व� 526
155
+ _व 526
156
+ �� 524
157
+ ह� 524
158
+ ��� 524
159
+ �_ 522
160
+ � 485
161
+ �� 478
162
+ स� 478
163
+ ��� 478
164
+ �् 477
165
+ ��् 477
166
+ �्� 477
167
+ �� 473
168
+ �र 472
169
+ ��र 472
170
+ �� 462
171
+ �� 462
172
+ _प 461
173
+ _प� 461
174
+ �� 454
175
+ श 454
176
+ �� 454
177
+ � 454
178
+ य� 454
179
+ ��� 454
180
+ �� 453
181
+ � 453
182
+ ी 453
183
+ �ि� 452
184
+ �ि 452
185
+ न� 452
186
+ ��ि 452
187
+ ��� 452
188
+ ब 442
189
+ �� 442
190
+ ब� 442
191
+ ��� 442
192
+ � 442
193
+ �� 442
194
+ �� 435
195
+ � 435
196
+ ो 435
197
+ �� 432
198
+ ल� 432
199
+ ��� 432
200
+ ��य 427
201
+ �य 427
202
+ �र� 424
203
+ �� 424
204
+ ं 424
205
+ ��ा 413
206
+ �ा 413
207
+ ं� 412
208
+ ज 412
209
+ �� 412
210
+ � 412
211
+ ��� 412
212
+ �� 407
213
+ �� 402
214
+ ��� 401
215
+ ं� 401
216
+ ��र 399
217
+ �र 399
218
+ ��� 397
219
+ प� 397
220
+ ��_ 396
221
+ त_ 396
222
+ ��� 388
223
+ प� 388
224
+ _क� 382
225
+ _क 382
226
+ ज� 377
227
+ ��� 377
228
+ �� 377
229
+ �� 371
230
+ ��� 371
231
+ द� 371
232
+ �क 362
233
+ ��क 362
234
+ � 361
235
+ ष 361
236
+ �� 361
237
+ ��� 354
238
+ �� 354
239
+ श� 354
240
+ �र� 349
241
+ ��� 348
242
+ ब� 348
243
+ �� 348
244
+ ��� 344
245
+ �� 344
246
+ ो� 344
247
+ �य� 343
248
+ �� 336
249
+ ध 336
250
+ � 336
251
+ �ा_ 323
252
+ �� 322
253
+ ��� 322
254
+ ो� 322
255
+ ��_ 316
256
+ न_ 316
257
+ �_ 316
258
+ �� 315
259
+ ग 315
260
+ � 315
261
+ ल� 313
262
+ �� 313
263
+ ��� 313
264
+ �� 306
265
+ ��� 306
266
+ त� 306
267
+ �� 300
268
+ �_ 299
269
+ ��_ 299
270
+ र_ 299
271
+ �� 298
272
+ � 298
273
+ ष� 296
274
+ �� 296
275
+ ��� 296
276
+ ��न 294
277
+ �न 294
278
+ द� 292
279
+ ��� 292
280
+ ह� 287
281
+ �� 287
282
+ ��� 287
283
+ _ब 286
284
+ _ब� 286
285
+ �� 283
286
+ ��� 283
287
+ भ� 283
288
+ भ 283
289
+ अ� 281
290
+ �� 281
291
+ ��� 281
292
+ अ� 281
293
+ �� 281
294
+ �� 281
295
+ अ 281
296
+ � 281
297
+ ��� 281
298
+ ध� 277
299
+ ��� 277
300
+ �� 277
301
+ क_ 275
302
+ �_ 275
303
+ ��_ 275
304
+ ��र 274
305
+ �र 274
306
+ �त 267
307
+ ��त 267
308
+ �� 266
309
+ ी� 266
310
+ �� 266
311
+ ��� 266
312
+ ी� 266
313
+ ��� 266
314
+ ज� 266
315
+ �� 266
316
+ ��� 266
317
+ ��� 263
318
+ ष� 263
319
+ �� 263
320
+ _अ 263
321
+ _अ� 263
322
+ ��् 261
323
+ �्� 261
324
+ �् 261
325
+ �्� 256
326
+ ��् 256
327
+ �् 256
328
+ �� 255
329
+ च 255
330
+ � 255
331
+ ग� 254
332
+ �� 254
333
+ ��� 254
334
+ म� 254
335
+ �� 254
336
+ ��� 254
337
+ �_ 248
338
+ ि_ 248
339
+ ��_ 248
340
+ �ा 246
341
+ ��ा 246
342
+ च� 245
343
+ �� 245
344
+ ��� 245
345
+ ��� 237
346
+ ��त 237
347
+ �� 237
348
+ ह� 237
349
+ �त 237
350
+ �� 233
351
+ �र� 233
352
+ ��� 233
353
+ श� 233
354
+ ण 230
355
+ �� 230
356
+ � 230
357
+ ��ा 225
358
+ �ा 225
359
+ �ा� 225
360
+ �ा 220
361
+ ��् 220
362
+ ��ा 220
363
+ �् 220
364
+ �् 219
365
+ ��् 219
366
+ �्� 219
367
+ �_ 215
368
+ ��_ 215
369
+ य_ 215
370
+ �� 211
371
+ �ा 206
372
+ ��ा 206
373
+ �्� 206
374
+ ग� 205
375
+ �� 205
376
+ ��� 205
377
+ �क� 204
378
+ भ� 204
379
+ ��� 204
380
+ �� 204
381
+ ��ि 204
382
+ _म 204
383
+ � 204
384
+ �ि 204
385
+ ए 204
386
+ _म� 204
387
+ �े 200
388
+ ��े 200
389
+ �त� 199
390
+ ��� 198
391
+ �� 198
392
+ च� 198
393
+ �त_ 195
394
+ �� 194
395
+ �� 194
396
+ आ� 194
397
+ �� 194
398
+ � 194
399
+ आ 194
400
+ ��� 194