scylla 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -2
- data/Gemfile.lock +16 -1
- data/lib/scylla/classifier.rb +1 -1
- data/lib/scylla/generator.rb +16 -4
- data/lib/scylla/lms/afrikaans.lm +232 -232
- data/lib/scylla/lms/arabic.lm +175 -175
- data/lib/scylla/lms/bulgarian.lm +225 -225
- data/lib/scylla/lms/catalan.lm +309 -309
- data/lib/scylla/lms/danish.lm +167 -167
- data/lib/scylla/lms/english.lm +398 -398
- data/lib/scylla/lms/finnish.lm +237 -237
- data/lib/scylla/lms/french.lm +148 -148
- data/lib/scylla/lms/german.lm +258 -258
- data/lib/scylla/lms/greek.lm +236 -236
- data/lib/scylla/lms/hebrew.lm +154 -154
- data/lib/scylla/lms/hindi.lm +139 -139
- data/lib/scylla/lms/icelandic.lm +239 -239
- data/lib/scylla/lms/indonesian.lm +244 -244
- data/lib/scylla/lms/italian.lm +248 -248
- data/lib/scylla/lms/japanese.lm +90 -90
- data/lib/scylla/lms/korean.lm +306 -306
- data/lib/scylla/lms/norwegian.lm +193 -193
- data/lib/scylla/lms/polish.lm +241 -241
- data/lib/scylla/lms/portuguese.lm +232 -232
- data/lib/scylla/lms/romanian.lm +246 -246
- data/lib/scylla/lms/slovak.lm +242 -242
- data/lib/scylla/lms/slovenian.lm +229 -229
- data/lib/scylla/lms/spanish.lm +164 -164
- data/lib/scylla/lms/swedish.lm +157 -157
- data/lib/scylla/lms/tagalog.lm +247 -247
- data/lib/scylla/lms/thai.lm +252 -252
- data/lib/scylla/lms/turkish.lm +285 -285
- data/lib/scylla/lms/vietnamese.lm +250 -250
- data/lib/scylla/lms/welsh.lm +248 -248
- data/lib/scylla/resources.rb +1 -9
- data/lib/scylla.rb +4 -0
- data/scylla.gemspec +2 -120
- data/source_texts/english.txt +62 -27
- data/test/classifier_test.rb +1 -3
- data/test/fixtures/lms/danish.lm +173 -173
- data/test/fixtures/lms/english.lm +220 -220
- data/test/fixtures/lms/french.lm +175 -175
- data/test/fixtures/lms/german.lm +254 -254
- data/test/fixtures/lms/hindi.lm +139 -139
- data/test/fixtures/lms/italian.lm +236 -236
- data/test/fixtures/lms/japanese.lm +88 -88
- data/test/fixtures/lms/norwegian.lm +182 -182
- data/test/fixtures/lms/spanish.lm +164 -164
- data/test/fixtures/test_languages/spanish +0 -1
- data/test/generator_test.rb +13 -0
- data/test/helper.rb +2 -0
- metadata +18 -25
- data/.document +0 -5
- data/lib/scylla/lms/13375P33K.lm +0 -400
- data/scylla-0.1.0.gem +0 -0
- data/source_texts/13375P33K.txt +0 -199
- data/test/fixtures/lms/13375p33k.lm +0 -400
- data/test/fixtures/source_texts/13375P33K.txt +0 -199
data/lib/scylla/lms/japanese.lm
CHANGED
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
� 6187
|
|
6
6
|
� 4811
|
|
7
7
|
� 4488
|
|
8
|
-
_ 3674
|
|
9
8
|
� 3670
|
|
9
|
+
_ 3238
|
|
10
10
|
� 2935
|
|
11
|
-
� 2486
|
|
12
11
|
� 2486
|
|
12
|
+
� 2486
|
|
13
13
|
� 2136
|
|
14
14
|
� 2083
|
|
15
15
|
� 2033
|
|
@@ -37,7 +37,6 @@ _ 3674
|
|
|
37
37
|
�� 1144
|
|
38
38
|
� 1142
|
|
39
39
|
� 1094
|
|
40
|
-
__ 1058
|
|
41
40
|
� 1046
|
|
42
41
|
� 1039
|
|
43
42
|
� 1028
|
|
@@ -47,11 +46,12 @@ __ 1058
|
|
|
47
46
|
� 1000
|
|
48
47
|
� 991
|
|
49
48
|
� 981
|
|
50
|
-
る 966
|
|
51
49
|
�� 966
|
|
50
|
+
る 966
|
|
52
51
|
� 958
|
|
53
|
-
��
|
|
52
|
+
�� 944
|
|
54
53
|
� 901
|
|
54
|
+
__ 880
|
|
55
55
|
� 836
|
|
56
56
|
� 823
|
|
57
57
|
� 813
|
|
@@ -59,7 +59,7 @@ __ 1058
|
|
|
59
59
|
� 799
|
|
60
60
|
� 795
|
|
61
61
|
�� 791
|
|
62
|
-
��
|
|
62
|
+
�� 784
|
|
63
63
|
� 780
|
|
64
64
|
� 772
|
|
65
65
|
� 767
|
|
@@ -68,78 +68,78 @@ __ 1058
|
|
|
68
68
|
�� 757
|
|
69
69
|
� 743
|
|
70
70
|
� 716
|
|
71
|
-
��
|
|
71
|
+
�� 712
|
|
72
72
|
� 710
|
|
73
73
|
� 708
|
|
74
|
-
と 704
|
|
75
74
|
�� 704
|
|
75
|
+
と 704
|
|
76
76
|
� 702
|
|
77
77
|
� 696
|
|
78
78
|
� 695
|
|
79
79
|
� 695
|
|
80
|
-
。 681
|
|
81
80
|
�� 681
|
|
81
|
+
。 681
|
|
82
82
|
� 669
|
|
83
|
-
国 659
|
|
84
83
|
�� 659
|
|
84
|
+
国 659
|
|
85
85
|
� 656
|
|
86
86
|
�� 647
|
|
87
87
|
� 641
|
|
88
88
|
�� 639
|
|
89
|
-
を 637
|
|
90
89
|
�� 637
|
|
90
|
+
を 637
|
|
91
91
|
�� 634
|
|
92
92
|
�� 627
|
|
93
93
|
し 627
|
|
94
94
|
� 622
|
|
95
|
-
� 619
|
|
96
95
|
� 619
|
|
96
|
+
� 619
|
|
97
97
|
�� 616
|
|
98
98
|
た 616
|
|
99
|
-
て 613
|
|
100
99
|
�� 613
|
|
100
|
+
て 613
|
|
101
101
|
� 611
|
|
102
102
|
い 604
|
|
103
103
|
�� 604
|
|
104
|
-
��
|
|
105
|
-
��� 600
|
|
104
|
+
�� 602
|
|
106
105
|
し� 600
|
|
106
|
+
��� 600
|
|
107
107
|
� 596
|
|
108
108
|
�� 592
|
|
109
109
|
� 584
|
|
110
|
-
��� 582
|
|
111
110
|
る� 582
|
|
112
|
-
|
|
111
|
+
��� 582
|
|
113
112
|
日 580
|
|
113
|
+
�� 580
|
|
114
114
|
な 579
|
|
115
115
|
�� 579
|
|
116
116
|
� 576
|
|
117
117
|
�� 563
|
|
118
118
|
� 559
|
|
119
|
-
��
|
|
119
|
+
�� 554
|
|
120
120
|
は 552
|
|
121
121
|
� 547
|
|
122
122
|
�� 535
|
|
123
123
|
�� 531
|
|
124
|
-
|
|
125
|
-
|
|
124
|
+
い� 529
|
|
125
|
+
��� 529
|
|
126
126
|
�� 519
|
|
127
127
|
� 515
|
|
128
|
-
�� 513
|
|
129
|
-
で 513
|
|
130
|
-
�� 513
|
|
131
128
|
本 513
|
|
129
|
+
�� 513
|
|
130
|
+
で 513
|
|
131
|
+
�� 513
|
|
132
132
|
� 510
|
|
133
|
-
と� 509
|
|
134
133
|
��� 509
|
|
134
|
+
と� 509
|
|
135
135
|
�� 507
|
|
136
136
|
�� 504
|
|
137
137
|
� 492
|
|
138
138
|
れ 490
|
|
139
139
|
�� 490
|
|
140
140
|
� 484
|
|
141
|
-
��� 483
|
|
142
141
|
し� 483
|
|
142
|
+
��� 483
|
|
143
143
|
日� 480
|
|
144
144
|
��� 480
|
|
145
145
|
�� 471
|
|
@@ -147,8 +147,8 @@ __ 1058
|
|
|
147
147
|
��� 469
|
|
148
148
|
日� 469
|
|
149
149
|
�本 466
|
|
150
|
-
��本 466
|
|
151
150
|
� 466
|
|
151
|
+
��本 466
|
|
152
152
|
れ� 465
|
|
153
153
|
��� 465
|
|
154
154
|
�� 461
|
|
@@ -163,10 +163,10 @@ __ 1058
|
|
|
163
163
|
�� 434
|
|
164
164
|
��� 420
|
|
165
165
|
な� 420
|
|
166
|
+
�� 419
|
|
166
167
|
��� 419
|
|
167
168
|
と� 419
|
|
168
|
-
��
|
|
169
|
-
�� 416
|
|
169
|
+
�� 417
|
|
170
170
|
の� 407
|
|
171
171
|
��� 407
|
|
172
172
|
に� 402
|
|
@@ -180,8 +180,8 @@ __ 1058
|
|
|
180
180
|
�� 373
|
|
181
181
|
� 373
|
|
182
182
|
��� 373
|
|
183
|
-
� 372
|
|
184
183
|
は� 372
|
|
184
|
+
� 372
|
|
185
185
|
、� 369
|
|
186
186
|
�� 369
|
|
187
187
|
で� 367
|
|
@@ -199,68 +199,68 @@ __ 1058
|
|
|
199
199
|
�� 354
|
|
200
200
|
な� 349
|
|
201
201
|
��� 349
|
|
202
|
-
� 347
|
|
203
202
|
�� 347
|
|
203
|
+
� 347
|
|
204
204
|
�。 342
|
|
205
205
|
��。 342
|
|
206
206
|
�� 338
|
|
207
207
|
�� 336
|
|
208
|
+
、� 335
|
|
208
209
|
��� 335
|
|
209
210
|
�� 335
|
|
210
|
-
、� 335
|
|
211
211
|
� 334
|
|
212
212
|
�� 333
|
|
213
213
|
�� 331
|
|
214
214
|
�� 330
|
|
215
|
-
��� 329
|
|
216
|
-
（ 329
|
|
217
215
|
�� 329
|
|
216
|
+
��� 329
|
|
218
217
|
は� 329
|
|
218
|
+
（ 329
|
|
219
219
|
） 328
|
|
220
220
|
�� 327
|
|
221
221
|
�� 324
|
|
222
222
|
ら 323
|
|
223
|
-
�� 323
|
|
224
223
|
��� 323
|
|
224
|
+
�� 323
|
|
225
225
|
�� 321
|
|
226
|
-
あ 320
|
|
227
226
|
あ� 320
|
|
228
|
-
|
|
227
|
+
あ 320
|
|
228
|
+
�� 316
|
|
229
|
+
�� 316
|
|
229
230
|
て� 316
|
|
230
231
|
も 316
|
|
231
|
-
|
|
232
|
+
��� 316
|
|
232
233
|
�� 315
|
|
233
|
-
�� 315
|
|
234
|
-
��� 313
|
|
235
234
|
、� 313
|
|
235
|
+
��� 313
|
|
236
236
|
「 311
|
|
237
|
-
�� 311
|
|
238
237
|
」 311
|
|
238
|
+
�� 311
|
|
239
239
|
�� 309
|
|
240
240
|
�� 309
|
|
241
|
-
� 308
|
|
242
241
|
�� 308
|
|
242
|
+
� 308
|
|
243
243
|
国� 306
|
|
244
|
-
��� 306
|
|
245
244
|
�� 306
|
|
246
|
-
|
|
245
|
+
��� 306
|
|
247
246
|
��� 305
|
|
247
|
+
の� 305
|
|
248
248
|
�、 302
|
|
249
|
+
_� 302
|
|
249
250
|
��、 300
|
|
250
251
|
� 298
|
|
251
252
|
��� 297
|
|
252
253
|
で� 297
|
|
253
|
-
_� 296
|
|
254
254
|
あ� 284
|
|
255
255
|
��� 284
|
|
256
256
|
�� 281
|
|
257
257
|
�� 279
|
|
258
258
|
�� 277
|
|
259
259
|
�� 276
|
|
260
|
-
れ� 274
|
|
261
260
|
��� 274
|
|
262
|
-
|
|
261
|
+
れ� 274
|
|
263
262
|
�� 273
|
|
263
|
+
か 273
|
|
264
264
|
�� 272
|
|
265
265
|
す 272
|
|
266
266
|
年 270
|
|
@@ -270,131 +270,131 @@ _
|
|
|
270
270
|
��� 265
|
|
271
271
|
� 263
|
|
272
272
|
」� 262
|
|
273
|
-
�� 262
|
|
274
273
|
� 262
|
|
274
|
+
�� 262
|
|
275
275
|
す� 261
|
|
276
276
|
��� 261
|
|
277
|
+
�� 257
|
|
277
278
|
��� 256
|
|
278
279
|
��� 255
|
|
279
280
|
�� 254
|
|
280
281
|
）� 254
|
|
281
|
-
�� 254
|
|
282
282
|
�� 251
|
|
283
|
-
本� 250
|
|
284
283
|
��� 250
|
|
285
284
|
��� 250
|
|
285
|
+
本� 250
|
|
286
|
+
��て 246
|
|
286
287
|
す� 246
|
|
287
|
-
�て 246
|
|
288
|
-
��� 246
|
|
289
|
-
�� 246
|
|
290
288
|
�� 246
|
|
291
|
-
|
|
289
|
+
�� 246
|
|
290
|
+
��� 246
|
|
291
|
+
�て 246
|
|
292
292
|
�本� 245
|
|
293
293
|
� 245
|
|
294
294
|
� 245
|
|
295
295
|
�る 242
|
|
296
296
|
��る 242
|
|
297
297
|
�� 241
|
|
298
|
-
|
|
299
|
-
|
|
298
|
+
い� 240
|
|
299
|
+
��� 240
|
|
300
|
+
っ 238
|
|
300
301
|
� 238
|
|
301
302
|
�� 238
|
|
302
|
-
っ 238
|
|
303
|
-
っ� 237
|
|
304
303
|
�� 237
|
|
305
|
-
|
|
306
|
-
�� 237
|
|
304
|
+
っ� 237
|
|
307
305
|
�� 237
|
|
306
|
+
�� 237
|
|
307
|
+
ア 237
|
|
308
308
|
っ� 237
|
|
309
309
|
�� 236
|
|
310
|
-
�� 235
|
|
311
310
|
�� 235
|
|
312
311
|
・ 235
|
|
312
|
+
�� 235
|
|
313
313
|
� 233
|
|
314
314
|
� 233
|
|
315
315
|
�い 232
|
|
316
|
-
�い� 231
|
|
317
316
|
_年 231
|
|
318
317
|
_� 231
|
|
319
|
-
|
|
320
|
-
��� 230
|
|
318
|
+
�い� 231
|
|
321
319
|
。� 230
|
|
320
|
+
��� 230
|
|
322
321
|
� 230
|
|
322
|
+
� 230
|
|
323
323
|
た� 229
|
|
324
|
-
��� 229
|
|
325
324
|
�� 229
|
|
325
|
+
��� 229
|
|
326
326
|
��い 228
|
|
327
327
|
�� 225
|
|
328
328
|
��� 224
|
|
329
|
-
�� 223
|
|
329
|
+
�� 223
|
|
330
330
|
�� 223
|
|
331
|
+
�� 223
|
|
331
332
|
�� 223
|
|
332
|
-
�� 223
|
|
333
333
|
�� 222
|
|
334
334
|
�� 220
|
|
335
335
|
に� 220
|
|
336
336
|
�� 219
|
|
337
337
|
��� 218
|
|
338
338
|
さ 217
|
|
339
|
-
国� 215
|
|
340
339
|
� 215
|
|
341
|
-
��� 215
|
|
342
340
|
さ� 215
|
|
341
|
+
国� 215
|
|
342
|
+
��� 215
|
|
343
343
|
�る 214
|
|
344
344
|
��る 213
|
|
345
|
-
�� 210
|
|
346
345
|
大 210
|
|
347
|
-
|
|
346
|
+
�� 210
|
|
348
347
|
��� 209
|
|
349
|
-
|
|
348
|
+
ら� 209
|
|
350
349
|
��� 208
|
|
350
|
+
い� 208
|
|
351
351
|
�� 207
|
|
352
352
|
� 205
|
|
353
353
|
� 205
|
|
354
354
|
��� 202
|
|
355
|
-
��� 201
|
|
356
355
|
に� 201
|
|
357
|
-
|
|
356
|
+
��� 201
|
|
358
357
|
�� 199
|
|
358
|
+
�� 199
|
|
359
359
|
�� 198
|
|
360
|
-
�� 197
|
|
361
360
|
り 197
|
|
361
|
+
�� 197
|
|
362
362
|
�� 196
|
|
363
|
-
��� 194
|
|
364
363
|
本� 194
|
|
365
|
-
|
|
364
|
+
��� 194
|
|
365
|
+
�� 193
|
|
366
366
|
� 193
|
|
367
|
-
|
|
367
|
+
さ� 193
|
|
368
|
+
�る 193
|
|
368
369
|
��� 193
|
|
370
|
+
��る 193
|
|
369
371
|
� 193
|
|
370
|
-
�� 193
|
|
371
|
-
�る 193
|
|
372
|
-
�� 192
|
|
373
372
|
�� 192
|
|
373
|
+
�� 192
|
|
374
374
|
��� 191
|
|
375
|
-
��� 191
|
|
376
|
-
人 191
|
|
377
375
|
�� 191
|
|
376
|
+
��� 191
|
|
378
377
|
ア� 191
|
|
379
378
|
が� 191
|
|
380
|
-
|
|
379
|
+
人 191
|
|
381
380
|
中 190
|
|
381
|
+
�� 190
|
|
382
382
|
�� 188
|
|
383
383
|
�� 188
|
|
384
384
|
か� 187
|
|
385
385
|
��� 187
|
|
386
|
+
�れ 185
|
|
386
387
|
�� 185
|
|
387
|
-
�� 185
|
|
388
|
-
�� 185
|
|
389
388
|
��れ 185
|
|
390
|
-
|
|
391
|
-
��
|
|
392
|
-
��
|
|
389
|
+
�� 185
|
|
390
|
+
�� 185
|
|
391
|
+
�� 184
|
|
393
392
|
�れ� 182
|
|
393
|
+
�� 182
|
|
394
394
|
�� 182
|
|
395
|
-
�あ� 181
|
|
396
395
|
��あ 181
|
|
397
396
|
� 181
|
|
398
397
|
�あ 181
|
|
399
|
-
��
|
|
398
|
+
�� 181
|
|
399
|
+
�あ� 181
|
|
400
400
|
ど 180
|