scylla 0.4.3 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. data/Gemfile +1 -0
  2. data/Gemfile.lock +10 -0
  3. data/VERSION +1 -1
  4. data/lib/scylla/generator.rb +1 -1
  5. data/lib/scylla/lms/13375P33K.lm +156 -156
  6. data/lib/scylla/lms/arabic.lm +133 -133
  7. data/lib/scylla/lms/bulgarian.lm +122 -122
  8. data/lib/scylla/lms/catalan.lm +151 -151
  9. data/lib/scylla/lms/danish.lm +137 -137
  10. data/lib/scylla/lms/english.lm +207 -207
  11. data/lib/scylla/lms/french.lm +400 -400
  12. data/lib/scylla/lms/japanese.lm +400 -400
  13. data/lib/scylla/lms/korean.lm +233 -233
  14. data/lib/scylla/lms/norwegian.lm +398 -398
  15. data/lib/scylla/lms/spanish.lm +98 -98
  16. data/lib/scylla/lms/swedish.lm +123 -123
  17. data/lib/scylla/lms/tagalog.lm +223 -223
  18. data/lib/scylla/lms/welsh.lm +234 -234
  19. data/lib/scylla/resources.rb +10 -10
  20. data/scylla.gemspec +17 -40
  21. data/source_texts/catalan.txt +28 -28
  22. data/source_texts/danish.txt +62 -62
  23. data/source_texts/english.txt +10 -10
  24. data/source_texts/french.txt +470 -77
  25. data/source_texts/japanese.txt +453 -199
  26. data/source_texts/norwegian.txt +96 -63
  27. data/source_texts/spanish.txt +269 -269
  28. data/test/classifier_test.rb +2 -2
  29. data/test/fixtures/lms/13375p33k.lm +156 -156
  30. data/test/fixtures/lms/danish.lm +137 -137
  31. data/test/fixtures/lms/english.lm +207 -207
  32. data/test/fixtures/lms/french.lm +400 -400
  33. data/test/fixtures/lms/hindi.lm +400 -0
  34. data/test/fixtures/lms/italian.lm +400 -0
  35. data/test/fixtures/lms/japanese.lm +400 -400
  36. data/test/fixtures/lms/norwegian.lm +400 -0
  37. data/test/fixtures/lms/spanish.lm +98 -98
  38. data/test/fixtures/source_texts/danish.txt +62 -62
  39. data/test/fixtures/source_texts/english.txt +10 -10
  40. data/test/fixtures/source_texts/french.txt +470 -77
  41. data/test/fixtures/source_texts/hindi.txt +199 -0
  42. data/test/fixtures/source_texts/italian.txt +120 -0
  43. data/test/fixtures/source_texts/japanese.txt +453 -199
  44. data/test/fixtures/source_texts/norwegian.txt +190 -0
  45. data/test/fixtures/source_texts/spanish.txt +269 -269
  46. data/test/fixtures/test_languages/english +61 -0
  47. data/test/fixtures/test_languages/french +0 -0
  48. data/test/fixtures/test_languages/german +29 -0
  49. data/test/fixtures/test_languages/hindi +3 -0
  50. data/test/fixtures/test_languages/italian +6 -0
  51. data/test/fixtures/test_languages/japanese +79 -0
  52. data/test/fixtures/test_languages/norwegian +14 -0
  53. data/test/fixtures/test_languages/spanish +22 -0
  54. data/test/generator_test.rb +0 -1
  55. data/test/language_test.rb +28 -0
  56. metadata +20 -43
  57. data/lib/scylla/lms/esperanto.lm +0 -400
  58. data/lib/scylla/lms/hungarian.lm +0 -400
  59. data/lib/scylla/lms/irish.lm +0 -400
  60. data/lib/scylla/lms/kannada.lm +0 -400
  61. data/lib/scylla/lms/latin.lm +0 -400
  62. data/lib/scylla/lms/malay.lm +0 -400
  63. data/lib/scylla/lms/marathi.lm +0 -400
  64. data/lib/scylla/lms/mingo.lm +0 -400
  65. data/lib/scylla/lms/nepali.lm +0 -400
  66. data/lib/scylla/lms/quechua.lm +0 -400
  67. data/lib/scylla/lms/rumantsch.lm +0 -400
  68. data/lib/scylla/lms/sanskrit.lm +0 -400
  69. data/lib/scylla/lms/scots_gaelic.lm +0 -400
  70. data/lib/scylla/lms/serbian.lm +0 -400
  71. data/lib/scylla/lms/swahili.lm +0 -400
  72. data/lib/scylla/lms/tamil.lm +0 -400
  73. data/lib/scylla/lms/ukrainian.lm +0 -400
  74. data/lib/scylla/lms/yiddish.lm +0 -400
  75. data/source_texts/esperanto.txt +0 -199
  76. data/source_texts/hungarian.txt +0 -102
  77. data/source_texts/irish.txt +0 -209
  78. data/source_texts/kannada.txt +0 -283
  79. data/source_texts/latin.txt +0 -120
  80. data/source_texts/malay.txt +0 -108
  81. data/source_texts/marathi.txt +0 -100
  82. data/source_texts/mingo.txt +0 -146
  83. data/source_texts/nepali.txt +0 -131
  84. data/source_texts/quechua.txt +0 -108
  85. data/source_texts/rumantsch.txt +0 -110
  86. data/source_texts/sanskrit.txt +0 -135
  87. data/source_texts/scots_gaelic.txt +0 -93
  88. data/source_texts/serbian.txt +0 -121
  89. data/source_texts/swahili.txt +0 -120
  90. data/source_texts/tamil.txt +0 -167
  91. data/source_texts/ukrainian.txt +0 -214
  92. data/source_texts/yiddish-utf.txt +0 -83
  93. data/test/fixtures/lms/kannada.lm +0 -400
  94. data/test/fixtures/source_texts/kannada.txt +0 -283
@@ -1,4 +1,4 @@
1
- _ 2736
1
+ _ 2574
2
2
  e 773
3
3
  t 520
4
4
  i 515
@@ -19,31 +19,30 @@ s_ 150
19
19
  re 134
20
20
  _a 127
21
21
  t_ 126
22
- _t 125
23
22
  in 123
23
+ _t 123
24
24
  or 103
25
25
  f 99
26
26
  m_ 98
27
27
  b 91
28
28
  g 87
29
29
  _o 85
30
- n_ 84
31
30
  th 84
32
- d_ 83
31
+ n_ 84
33
32
  er 83
33
+ d_ 83
34
34
  _i 81
35
35
  he 81
36
36
  y 80
37
37
  an 79
38
- __ 78
39
- _e 78
40
38
  at 78
39
+ _e 78
41
40
  , 75
42
41
  ,_ 75
43
42
  on 73
44
- es 72
45
- _s 72
46
43
  te 72
44
+ _s 72
45
+ es 72
47
46
  _th 70
48
47
  is 69
49
48
  _p 65
@@ -55,346 +54,347 @@ se 62
55
54
  en 60
56
55
  w 60
57
56
  ti 59
58
- r_ 58
59
- it 58
60
57
  nd 58
58
+ it 58
59
+ r_ 58
61
60
  ur 52
62
61
  as 52
63
- ._ 52
64
- the 52
65
62
  _d 52
63
+ the 52
66
64
  nt 51
67
65
  _the 50
68
66
  a_ 50
69
67
  su 50
70
- qu 48
71
- q 48
72
68
  _c 48
73
69
  he_ 48
70
+ qu 48
71
+ q 48
74
72
  y_ 47
75
73
  _w 47
76
- us 46
77
74
  _b 46
75
+ us 46
78
76
  le 45
79
- of 44
77
+ ._ 44
80
78
  em 44
81
- si 43
79
+ of 44
82
80
  ta 43
81
+ si 43
83
82
  f_ 42
84
- ore 42
85
83
  um_ 42
86
- _of 42
87
84
  ed 42
85
+ _of 42
86
+ ore 42
88
87
  to 41
89
88
  k 41
90
89
  li 40
90
+ et 40
91
91
  ng 40
92
92
  re_ 40
93
- et 40
93
+ is_ 39
94
+ and 39
94
95
  the_ 39
95
- _of_ 39
96
- of_ 39
97
96
  ea 39
98
- and 39
99
- is_ 39
100
97
  _the_ 39
98
+ of_ 39
99
+ _of_ 39
101
100
  ol 38
102
101
  in_ 38
103
102
  st 38
104
103
  de 37
105
104
  _m 37
106
- _an 36
107
105
  _in 36
106
+ _an 36
107
+ _r 35
108
108
  io 35
109
109
  _h 35
110
- _r 35
111
- ri 34
112
110
  ve 34
111
+ ri 34
113
112
  ed_ 34
114
- ar 34
115
113
  la 34
116
- ce 33
114
+ ar 34
117
115
  _f 33
118
- ing 33
119
116
  ou 33
120
- I 32
117
+ ing 33
118
+ ce 33
121
119
  lo 32
120
+ I 32
122
121
  ec 32
123
122
  ro 32
123
+ ng_ 31
124
+ g_ 31
124
125
  nd_ 31
125
126
  _I 31
126
- g_ 31
127
- ng_ 31
128
- to_ 30
129
- _n 30
127
+ ing_ 30
130
128
  em_ 30
131
129
  al 30
132
- el 30
130
+ co 30
133
131
  ha 30
134
132
  _to 30
133
+ to_ 30
134
+ el 30
135
+ _n 30
135
136
  ni 30
136
- ing_ 30
137
- co 30
138
- es_ 29
139
- _l 29
140
- no 29
141
137
  di 29
142
- ut 29
138
+ no 29
143
139
  ra 29
140
+ es_ 29
144
141
  om 29
145
- _u 28
142
+ _l 29
143
+ ut 29
144
+ ai 28
146
145
  ic 28
146
+ ne 28
147
147
  ct 28
148
148
  am 28
149
- ne 28
150
- ai 28
151
- rem 27
149
+ _u 28
150
+ l_ 27
152
151
  ion 27
153
- _and 27
154
- _v 27
152
+ ns 27
155
153
  ll 27
156
154
  _and_ 27
157
155
  and_ 27
158
- ns 27
159
- l_ 27
156
+ _v 27
160
157
  _re 27
161
- ia 26
158
+ _and 27
159
+ rem 27
160
+ L 26
161
+ _a_ 26
162
162
  sum 26
163
163
  do 26
164
- _a_ 26
165
- L 26
164
+ ia 26
166
165
  _to_ 26
167
- _q 25
168
- _qu 25
169
- pa 25
170
- ru 25
171
166
  tio 25
172
- ci 25
173
- ps 25
167
+ pa 25
168
+ _q 25
174
169
  il 25
175
- ui 24
176
- ati 24
177
- un 24
170
+ ps 25
171
+ ci 25
172
+ ru 25
173
+ _qu 25
178
174
  me 24
179
175
  x 24
180
- psu 23
181
- rem_ 23
182
- _co 23
183
- orem 23
176
+ un 24
177
+ ui 24
178
+ ati 24
179
+ _in_ 23
184
180
  tion 23
185
- ss 23
186
- _L 23
181
+ hi 23
182
+ orem 23
187
183
  psum 23
184
+ psu 23
185
+ ss 23
186
+ _co 23
187
+ rem_ 23
188
188
  pl 23
189
- hi 23
190
- _in_ 23
191
- se_ 22
192
- ure 22
193
- ma 22
189
+ _L 23
194
190
  im 22
195
- mo 22
196
- orem_ 22
197
- tr 22
198
191
  _pa 22
199
- id 22
192
+ ma 22
193
+ mo 22
200
194
  _do 22
195
+ tr 22
196
+ ure 22
197
+ orem_ 22
201
198
  or_ 22
202
- sum_ 21
199
+ id 22
200
+ se_ 22
203
201
  lor 21
204
- nc 21
205
- ac 21
202
+ sum_ 21
206
203
  on_ 21
204
+ ac 21
205
+ nc 21
206
+ ex 20
207
207
  ge 20
208
+ " 20
209
+ tu 20
208
210
  it_ 20
209
- ex 20
210
211
  i_ 20
211
212
  _si 20
212
213
  ut_ 20
213
- " 20
214
214
  at_ 20
215
- tu 20
216
- Lore 19
215
+ qui 19
217
216
  et_ 19
218
- Lor 19
219
217
  Lo 19
220
- us_ 19
221
- qui 19
222
218
  Lorem 19
223
- pt 19
224
- psum_ 19
219
+ us_ 19
220
+ Lor 19
221
+ Lore 19
225
222
  ons 19
226
- _no 18
223
+ psum_ 19
224
+ pt 19
227
225
  be 18
228
- as_ 18
229
- ho 18
230
226
  ect 18
231
- pe 18
232
- er_ 18
233
- ain 18
227
+ ho 18
234
228
  ab 18
235
229
  T 18
230
+ ain 18
231
+ er_ 18
232
+ as_ 18
233
+ _no 18
234
+ pe 18
235
+ Ip 17
236
236
  bl 17
237
- h_ 17
238
- _Ip 17
239
- ple 17
240
- eas 17
241
- rs 17
242
237
  _Ipsu 17
243
- sa 17
244
- rum 17
238
+ _Lore 17
239
+ Ipsu 17
245
240
  Ips 17
246
- _Lo 17
247
- ca 17
248
- pr 17
249
- _is 17
250
- wh 17
241
+ sa 17
242
+ Ipsum 17
251
243
  up 17
252
- _Lore 17
253
- _Lor 17
244
+ wh 17
245
+ _is 17
246
+ h_ 17
254
247
  _et 17
255
- Ipsu 17
256
- _be 17
257
- vo 17
258
- Ip 17
248
+ _Lo 17
259
249
  _wh 17
260
- Ipsum 17
250
+ rs 17
251
+ eas 17
261
252
  _Ips 17
262
- du 16
263
- ep 16
264
- tur 16
265
- cu 16
253
+ rum 17
254
+ ple 17
255
+ vo 17
256
+ _Lor 17
257
+ _be 17
258
+ ca 17
259
+ _Ip 17
260
+ pr 17
266
261
  olo 16
267
- ent 16
268
- ch 16
269
- ate 16
270
262
  lea 16
271
263
  ere 16
272
- bu 16
273
- au 16
264
+ ep 16
274
265
  ul 16
275
- nt_ 16
276
- _qui 16
266
+ cu 16
277
267
  po 16
278
268
  ua 16
279
269
  _ma 16
270
+ du 16
271
+ au 16
272
+ tur 16
273
+ bu 16
274
+ ate 16
275
+ ch 16
280
276
  ag 16
277
+ _qui 16
278
+ ent 16
281
279
  lu 16
282
- na 15
283
- olor 15
284
- _. 15
285
- _is_ 15
286
- s,_ 15
287
- dol 15
288
- C 15
289
- os 15
280
+ nt_ 16
290
281
  _et_ 15
291
- ter 15
282
+ _ex 15
292
283
  tat 15
293
- ver 15
284
+ C 15
294
285
  _dol 15
295
- _dolo 15
296
286
  s, 15
287
+ s,_ 15
288
+ ter 15
289
+ ver 15
297
290
  leas 15
298
- _ex 15
299
- dolo 15
291
+ _dolo 15
292
+ _is_ 15
300
293
  dolor 15
301
- sur 14
294
+ na 15
295
+ dolo 15
296
+ olor 15
297
+ dol 15
298
+ os 15
299
+ _pl 14
302
300
  t,_ 14
303
- so 14
304
- _._ 14
305
- _con 14
306
- con 14
307
- _de 14
301
+ ke 14
308
302
  qua 14
309
- ip 14
310
- _pl 14
303
+ sur 14
304
+ _de 14
305
+ con 14
306
+ so 14
311
307
  wi 14
312
- ke 14
313
308
  ue 14
309
+ ip 14
314
310
  t, 14
315
- ure_ 13
316
- te_ 13
317
- her 13
311
+ _con 14
318
312
  ir 13
319
- plea 13
313
+ res 13
314
+ olu 13
315
+ _by_ 13
320
316
  atio 13
321
- by_ 13
322
- sure 13
323
- _by 13
317
+ _plea 13
324
318
  _ve 13
325
319
  en_ 13
326
- Th 13
327
- od 13
328
- pleas 13
320
+ by_ 13
321
+ vol 13
329
322
  _mo 13
330
- _ha 13
331
- by 13
323
+ plea 13
324
+ te_ 13
325
+ od 13
332
326
  _ple 13
333
- _by_ 13
327
+ Th 13
334
328
  upt 13
329
+ by 13
330
+ _ha 13
335
331
  ation 13
336
- olu 13
337
- res 13
338
- vol 13
339
- _plea 13
340
- ur_ 12
341
- lup 12
332
+ sure 13
333
+ pleas 13
334
+ her 13
335
+ ure_ 13
336
+ _by 13
337
+ lupta 12
338
+ volup 12
342
339
  leasu 12
343
340
  _fr 12
341
+ asu 12
344
342
  _vo 12
345
- lupta 12
346
- mp 12
347
- oo 12
343
+ ae 12
348
344
  upta 12
349
- fr 12
350
- bo 12
351
- ly 12
352
- ly_ 12
353
- lupt 12
354
- asur 12
355
- olupt 12
345
+ easur 12
356
346
  asure 12
357
- olup 12
358
- _volu 12
347
+ sin 12
359
348
  _vol 12
360
- _pr 12
361
- asu 12
362
- _wi 12
349
+ asur 12
350
+ lupt 12
351
+ nte 12
352
+ fr 12
363
353
  volu 12
364
- da 12
365
- easu 12
354
+ _pr 12
366
355
  age 12
367
- unt 12
368
- volup 12
369
- rum_ 12
370
- easur 12
371
- ae 12
356
+ easu 12
357
+ oo 12
372
358
  est 12
359
+ ly 12
360
+ ur_ 12
361
+ ly_ 12
362
+ lup 12
363
+ unt 12
364
+ _volu 12
365
+ da 12
366
+ _wi 12
367
+ olup 12
368
+ bo 12
373
369
  pta 12
374
- nte 12
375
- sin 12
376
- cons 11
377
- _cons 11
378
- here 11
370
+ olupt 12
371
+ rum_ 12
372
+ mp 12
373
+ oru 11
374
+ _pai 11
375
+ _te 11
376
+ _g 11
379
377
  _it 11
378
+ ain_ 11
379
+ here 11
380
380
  yo 11
381
- cc 11
382
- _pain 11
383
- equ 11
384
- _y 11
385
381
  _al 11
386
382
  fo 11
387
- The 11
388
- _T 11
389
- oi 11
390
- B 11
391
383
  orum 11
384
+ equ 11
392
385
  _se 11
393
- pain 11
394
- _te 11
395
- pai 11
396
- ain_ 11
386
+ The 11
387
+ cons 11
388
+ cc 11
397
389
  nce 11
398
- _g 11
399
- tin 11
390
+ _y 11
391
+ ib 11
400
392
  _us 11
393
+ pain 11
394
+ pai 11
395
+ _pain 11
396
+ _cons 11
397
+ _T 11
398
+ B 11
399
+ eq 11
400
+ oi 11