scylla 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. data/Gemfile +0 -1
  2. data/Gemfile.lock +0 -10
  3. data/README.rdoc +22 -0
  4. data/VERSION +1 -1
  5. data/bin/scylla +13 -0
  6. data/lib/scylla/classifier.rb +2 -2
  7. data/lib/scylla/generator.rb +1 -1
  8. data/lib/scylla/lms/13375P33K.lm +400 -0
  9. data/lib/scylla/lms/afrikaans.lm +400 -0
  10. data/lib/scylla/lms/arabic.lm +400 -0
  11. data/lib/scylla/lms/bulgarian.lm +400 -0
  12. data/lib/scylla/lms/catalan.lm +400 -0
  13. data/lib/scylla/lms/chinese.lm +400 -0
  14. data/lib/scylla/lms/danish.lm +400 -0
  15. data/lib/scylla/lms/english.lm +400 -0
  16. data/lib/scylla/lms/esperanto.lm +400 -0
  17. data/lib/scylla/lms/finnish.lm +400 -0
  18. data/lib/scylla/lms/french.lm +400 -0
  19. data/lib/scylla/lms/german.lm +400 -0
  20. data/lib/scylla/lms/greek-iso8859-7.lm +400 -0
  21. data/lib/scylla/lms/hebrew.lm +400 -0
  22. data/lib/scylla/lms/hindi.lm +400 -0
  23. data/lib/scylla/lms/hungarian.lm +400 -0
  24. data/lib/scylla/lms/icelandic.lm +400 -0
  25. data/lib/scylla/lms/indonesian.lm +400 -0
  26. data/lib/scylla/lms/irish.lm +400 -0
  27. data/lib/scylla/lms/italian.lm +400 -0
  28. data/lib/scylla/lms/japanese.lm +400 -0
  29. data/lib/scylla/lms/kannada.lm +400 -0
  30. data/lib/scylla/lms/korean.lm +400 -0
  31. data/lib/scylla/lms/latin.lm +400 -0
  32. data/lib/scylla/lms/malay.lm +400 -0
  33. data/lib/scylla/lms/marathi.lm +400 -0
  34. data/lib/scylla/lms/mingo.lm +400 -0
  35. data/lib/scylla/lms/nepali.lm +400 -0
  36. data/lib/scylla/lms/norwegian.lm +400 -0
  37. data/lib/scylla/lms/polish.lm +400 -0
  38. data/lib/scylla/lms/portuguese.lm +400 -0
  39. data/lib/scylla/lms/quechua.lm +400 -0
  40. data/lib/scylla/lms/romanian.lm +400 -0
  41. data/lib/scylla/lms/rumantsch.lm +400 -0
  42. data/lib/scylla/lms/russian.lm +400 -0
  43. data/lib/scylla/lms/sanskrit.lm +400 -0
  44. data/lib/scylla/lms/scots_gaelic.lm +400 -0
  45. data/lib/scylla/lms/serbian-ascii.lm +400 -0
  46. data/lib/scylla/lms/slovak-ascii.lm +400 -0
  47. data/lib/scylla/lms/slovenian-ascii.lm +400 -0
  48. data/lib/scylla/lms/spanish.lm +400 -0
  49. data/lib/scylla/lms/swahili.lm +400 -0
  50. data/lib/scylla/lms/swedish.lm +400 -0
  51. data/lib/scylla/lms/tagalog.lm +400 -0
  52. data/lib/scylla/lms/tamil.lm +400 -0
  53. data/lib/scylla/lms/thai.lm +400 -0
  54. data/lib/scylla/lms/turkish.lm +400 -0
  55. data/lib/scylla/lms/ukrainian-koi8_u.lm +400 -0
  56. data/lib/scylla/lms/vietnamese.lm +400 -0
  57. data/lib/scylla/lms/welsh.lm +400 -0
  58. data/lib/scylla/lms/yiddish-utf.lm +400 -0
  59. data/lib/scylla/loader.rb +8 -1
  60. data/scylla-0.1.0.gem +0 -0
  61. data/scylla.gemspec +69 -3
  62. data/source_texts/kannada.txt +283 -0
  63. data/test/classifier_test.rb +7 -0
  64. data/test/fixtures/lms/13375p33k.lm +400 -0
  65. data/test/fixtures/lms/danish.lm +400 -0
  66. data/test/fixtures/lms/english.lm +400 -0
  67. data/test/fixtures/lms/french.lm +400 -0
  68. data/test/fixtures/lms/german.lm +400 -0
  69. data/test/fixtures/lms/japanese.lm +400 -0
  70. data/test/fixtures/lms/kannada.lm +400 -0
  71. data/test/fixtures/lms/spanish.lm +400 -0
  72. data/test/fixtures/source_texts/13375P33K.txt +199 -0
  73. data/test/fixtures/source_texts/japanese.txt +199 -0
  74. data/test/fixtures/source_texts/kannada.txt +283 -0
  75. data/test/generator_test.rb +10 -7
  76. data/test/helper.rb +5 -6
  77. data/test/loader_test.rb +1 -0
  78. data/test/scylla_test.rb +1 -0
  79. metadata +78 -14
  80. data/source_texts/armenian.txt +0 -86
data/lib/scylla/lms/japanese.lm ADDED
@@ -0,0 +1,400 @@
1
+ � 11219
2
+ _ 9082
3
+ � 5020
4
+ � 4899
5
+ � 4763
6
+ � 4568
7
+ � 3134
8
+ � 2631
9
+ � 2139
10
+ � 2121
11
+ _� 2052
12
+ � 2014
13
+ � 1886
14
+ � 1883
15
+ � 1681
16
+ � 1630
17
+ � 1618
18
+ � 1604
19
+ � 1415
20
+ � 1377
21
+ � 1255
22
+ � 1186
23
+ _� 1160
24
+ � 924
25
+ � 792
26
+ � 772
27
+ �� 736
28
+ �� 722
29
+ ゅ 722
30
+ � 722
31
+ �� 659
32
+ ょ 659
33
+ � 659
34
+ � 649
35
+ � 632
36
+ �� 626
37
+ ゃ 626
38
+ _� 611
39
+ � 597
40
+ _� 552
41
+ � 523
42
+ � 493
43
+ � 486
44
+ , 486
45
+ ,_ 486
46
+ �� 484
47
+ � 466
48
+ _� 457
49
+ �� 456
50
+ �� 452
51
+ _� 448
52
+ �� 445
53
+ � 441
54
+ _� 440
55
+ � 435
56
+ � 432
57
+ �� 430
58
+ � 414
59
+ �_ 398
60
+ � 395
61
+ � 379
62
+ � 368
63
+ � 349
64
+ � 349
65
+ �� 337
66
+ �� 328
67
+ � 322
68
+ �_ 319
69
+ � 317
70
+ �� 312
71
+ �� 310
72
+ �� 310
73
+ � 302
74
+ �� 296
75
+ � 294
76
+ �_ 291
77
+ � 291
78
+ �� 291
79
+ �� 291
80
+ _� 290
81
+ �_ 290
82
+ _� 289
83
+ _� 281
84
+ � 277
85
+ � 275
86
+ り 275
87
+ �� 275
88
+ �� 272
89
+ �� 271
90
+ � 271
91
+ � 270
92
+ � 269
93
+ � 268
94
+ � 266
95
+ �� 266
96
+ ぎ 266
97
+ じ 266
98
+ �� 266
99
+ � 263
100
+ �� 262
101
+ ち 262
102
+ ぢ 260
103
+ �� 260
104
+ � 258
105
+ �� 258
106
+ ち� 251
107
+ �� 251
108
+ ��� 251
109
+ � 248
110
+ �� 247
111
+ �� 247
112
+ �_ 241
113
+ �_ 238
114
+ �� 238
115
+ �� 233
116
+ �� 231
117
+ �� 228
118
+ � 227
119
+ �� 217
120
+ ��� 214
121
+ ぎ� 214
122
+ ��� 214
123
+ �� 214
124
+ ぎ� 214
125
+ �� 211
126
+ �� 209
127
+ �� 209
128
+ り� 209
129
+ ゃ� 209
130
+ ��� 209
131
+ り� 209
132
+ ��� 209
133
+ ��� 209
134
+ ょ� 208
135
+ �� 208
136
+ ��� 208
137
+ �� 208
138
+ び� 205
139
+ �� 205
140
+ び� 205
141
+ ��� 205
142
+ び 205
143
+ �� 205
144
+ ュ 205
145
+ � 205
146
+ �� 205
147
+ ��� 205
148
+ �� 205
149
+ � 203
150
+ �� 202
151
+ ��� 201
152
+ �� 201
153
+ ウ� 201
154
+ ��� 201
155
+ ��� 201
156
+ ち� 201
157
+ ウ 201
158
+ ウ� 201
159
+ � 198
160
+ �� 197
161
+ �� 197
162
+ �� 195
163
+ � 195
164
+ �� 194
165
+ �� 194
166
+ ��� 194
167
+ じ� 194
168
+ じ� 194
169
+ ��� 194
170
+ �� 194
171
+ �� 193
172
+ � 193
173
+ き 193
174
+ ��� 193
175
+ き� 193
176
+ �� 193
177
+ き� 193
178
+ ��� 193
179
+ ��� 192
180
+ ぢ� 192
181
+ � 192
182
+ ��� 192
183
+ �� 192
184
+ ぢ� 192
185
+ �� 192
186
+ �� 191
187
+ �� 190
188
+ �� 189
189
+ ゅ� 189
190
+ �� 189
191
+ ��� 189
192
+ ャ 188
193
+ �� 188
194
+ � 187
195
+ �_ 187
196
+ に� 185
197
+ に� 185
198
+ � 185
199
+ に 185
200
+ ��� 185
201
+ �� 185
202
+ ��� 185
203
+ �_ 184
204
+ み 184
205
+ �� 184
206
+ �� 183
207
+ ゥ 183
208
+ ��� 181
209
+ �� 181
210
+ ゃ� 181
211
+ �� 181
212
+ �� 180
213
+ �� 180
214
+ � 178
215
+ ひ 178
216
+ �� 178
217
+ � 177
218
+ � 175
219
+ ��� 172
220
+ � 172
221
+ �� 172
222
+ ゅ� 172
223
+ ��� 170
224
+ ぴ� 170
225
+ ぴ� 170
226
+ ��� 170
227
+ ぴ 170
228
+ �� 170
229
+ �� 161
230
+ �� 159
231
+ �� 159
232
+ ��� 158
233
+ �� 151
234
+ �� 150
235
+ ド 150
236
+ � 150
237
+ � 149
238
+ � 149
239
+ � 149
240
+ � 148
241
+ �� 147
242
+ � 146
243
+ �� 146
244
+ ゜ 146
245
+ ��� 144
246
+ �� 144
247
+ ド� 144
248
+ � 144
249
+ �� 144
250
+ � 143
251
+ � 142
252
+ � 142
253
+ �� 141
254
+ ィ 141
255
+ �� 139
256
+ � 139
257
+ �� 139
258
+ � 138
259
+ �� 138
260
+ � 137
261
+ �� 137
262
+ �� 137
263
+ � 137
264
+ �� 136
265
+ � 136
266
+ �� 136
267
+ �� 136
268
+ ��� 136
269
+ し 136
270
+ �� 136
271
+ � 136
272
+ し� 136
273
+ �� 136
274
+ � 136
275
+ �� 135
276
+ � 135
277
+ � 135
278
+ �� 135
279
+ � 134
280
+ � 134
281
+ �� 134
282
+ � 133
283
+ �� 133
284
+ � 133
285
+ � 133
286
+ �� 133
287
+ ァ 133
288
+ �� 132
289
+ ��� 132
290
+ �� 132
291
+ キ� 132
292
+ �� 132
293
+ �� 132
294
+ �� 132
295
+ キ� 132
296
+ �� 132
297
+ �� 132
298
+ キ 132
299
+ ��� 132
300
+ � 131
301
+ �� 131
302
+ _� 131
303
+ ��� 131
304
+ �� 131
305
+ �� 131
306
+ ゅ� 131
307
+ �� 131
308
+ � 130
309
+ � 130
310
+ �� 130
311
+ �� 130
312
+ �� 130
313
+ ��� 130
314
+ �� 130
315
+ � 130
316
+ �� 130
317
+ ひ� 130
318
+ ��� 130
319
+ ひ� 130
320
+ �� 129
321
+ � 129
322
+ �� 129
323
+ ��� 129
324
+ �� 129
325
+ �� 129
326
+ ��� 129
327
+ ��� 129
328
+ チ 129
329
+ ュ� 129
330
+ ュ� 129
331
+ �� 129
332
+ ��_ 129
333
+ ゅ_ 129
334
+ �_ 129
335
+ �� 128
336
+ �� 128
337
+ � 127
338
+ ョ 127
339
+ �� 127
340
+ �� 127
341
+ �� 126
342
+ � 126
343
+ �� 126
344
+ � 126
345
+ � 126
346
+ �_ 125
347
+ �� 125
348
+ �� 124
349
+ � 124
350
+ ��� 124
351
+ �� 124
352
+ ゃ� 124
353
+ �_ 124
354
+ ェ 123
355
+ ��� 123
356
+ �� 123
357
+ �� 123
358
+ み� 123
359
+ �� 123
360
+ �� 123
361
+ �� 123
362
+ �� 123
363
+ ��� 122
364
+ �� 122
365
+ �� 122
366
+ �き 122
367
+ �き� 122
368
+ ��� 122
369
+ み� 122
370
+ ヴ� 121
371
+ ��� 121
372
+ ��� 121
373
+ � 121
374
+ �� 121
375
+ ��� 121
376
+ ヴ 121
377
+ ヴ� 121
378
+ �� 121
379
+ � 121
380
+ �� 120
381
+ ��_ 119
382
+ �� 118
383
+ �� 117
384
+ �� 117
385
+ � 117
386
+ �� 117
387
+ _ち� 117
388
+ �� 117
389
+ _ち 117
390
+ �� 116
391
+ �� 114
392
+ ��� 114
393
+ �� 114
394
+ ょ� 114
395
+ �� 113
396
+ �� 113
397
+ �ヴ 113
398
+ �ヴ� 113
399
+ _� 112
400
+ ��� 110