scylla 0.9.3 → 1.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7ae67cbe11a572ef9f3da4fb7040fe87cf48034bbc436f74c7b81a4c013b70f5
+  data.tar.gz: '056492ee87ccf4d8171642e4a5ad91375d8b88fbb27af042f826175313ee029c'
+SHA512:
+  metadata.gz: be438618a77b5c0339ba0dd56469de139e1060e19959de5095daa0ed9a79d2b4621ee133dba90e177d7651a06ccfa1d1dc49de1f6304623cb261abb75f85de95
+  data.tar.gz: 4f57607ca609cdfbc74d39e5d245a6e869a0256926048f019fa31032dd61163d2d1cc2cd04a1b786b755260001b826255d14d5bfabe267ad2aaea132aaca18b2

data/README.rdoc

@@ -21,25 +21,16 @@ Multiple results for other possible languages:
  "isso poderia ser confundido com espanhol, bem".language
  => "portuguese"
 
- "isso poderia ser confundido com espanhol, bem".guess
+ "isso poderia ser confundido com espanhol, bem".guess_language
  => ["portuguese", "spanish"]
 
 == Training
 
-You can train scylla in new languages by providing sample texts in different languages. The default set is located in the 'source_texts' folder in the gem directory. Add new .txt files to this directory named according to the language i.e. a text file full of Hebrew text should be called 'hebrew.txt'. At least 500 lines of text recommended. Then, in the gem folder, run this:
+Training is done by fetching data from wikipedia. To fetch latest articles (The country name in the language
+in question, eg. "England" for English or "日本" for Japanese), run
 
  rake scylla:train
 
-If you want to store texts in your own folder, you can specify that to the rake task.
-WARNING: specifying a different folder deletes all language support for files located in the default directory if they are not copied over.
-
- rake scylla:train[/Users/hash/mytextdir]
- "Creating language map for /Users/hash/mytextdir/english.txt"
- "Creating language map for /Users/hash/mytextdir/kannada.txt"
- .
- .
- etc
-
 == Contributing to scylla
  
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet

data/bin/scylla

@@ -9,5 +9,5 @@ while(phrase != "exit")
   puts "Phrase:"
   STDOUT.flush
   phrase = gets.chomp
-  puts phrase.guess.join(" or ")
+  puts phrase.guess_language.join(" or ")
 end

data/lib/scylla/classifier.rb

@@ -50,7 +50,8 @@ module Scylla
     # Gets the score of the text in question compared to a particular language
     def get_score(unknown, ngram)
       i, p = 0,0
-      while i < unknown.size
+      max_size = [unknown.size, 400].min
+      while i < max_size
         if (ngram[unknown[i]])
           p += (ngram[unknown[i]]-i).abs
         else
@@ -61,4 +62,4 @@ module Scylla
       return p
     end
   end
-end
+end

data/lib/scylla/generator.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'sanitize'
 require 'cgi'
 require 'unicode'
@@ -5,7 +6,7 @@ require 'unicode'
 module Scylla
   class Generator
     attr_accessor :dirtext, :dirlm, :minsize, :delimiter
-    NONLATIN = ["bg","ar","ru","zh","ja","he","kn","ko","mr","hi","th","fa","el"]
+    NONLATIN = ["bg","ar","ru","zh","ja","he","kn","ko","mr","hi","th","fa","el","uk"]
     # dirtext: The location of the source training text files
     # minsize: The minimum size of the ngrams that you would like to store
     def initialize(dirtext = DEFAULT_SOURCE_DIR, dirlm = DEFAULT_TARGET_DIR, minsize = 0, silent = false, delimiter = "[[classifier_delimiter]]")
@@ -50,7 +51,9 @@ module Scylla
       }
       p article
       page = Wikipedia.find( article )
-      value = page.content.gsub(/\{\{(.*?)\}\}/,"")
+      value = page.raw_data['query']['pages'].values.first['revisions'].first.fetch('*')
+      value = value.force_encoding("UTF-8").chars.select {|c| c.valid_encoding?}.join
+      value = value.gsub(/\{\{(.*?)\}\}/,"")
       value = value.gsub(/\[\[(.+?)\]\]/m,"")
       value = value.gsub(/\{\{(.+?)\}\}/m,"")
       value = value.gsub(/\{(.+?)\}/m,"")
@@ -82,9 +85,13 @@ module Scylla
       string = string[0, delimit] if delimit
       string = Sanitize.clean(string)
       string = CGI.unescapeHTML(string)
+      string = Unicode::downcase(string)
       string.gsub!(/(?:http|https):\/\/[a-z0-9]+(?:[\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(?:(?::[0-9]{1,5})?\/[^\s]*)?/, "")
+      string.gsub!(/[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,4}/, "")
       string.gsub!(/[\*\^><!\"#\$%&\'\(\)\*\+:;,._\/=\?@\{\}\[\]|\-\n\r0-9]/," ")
-      Unicode::downcase(string.strip.split(" ").join(" "))
+      latin, nonlatin = string.scan(/[a-z]/), string.scan(/[\p{L}&&[^a-z]]/)
+      string.gsub!(/[a-zA-Z]/, "") if !latin.empty? && !nonlatin.empty? && nonlatin.size/(latin.size*1.0) > 0.5
+      string.strip.split(" ").join(" ")
     end
 
     # Creates a language map for a given input string. 
@@ -115,4 +122,4 @@ module Scylla
       return sorted
     end
   end
-end 
+end 

data/lib/scylla/lms/arabic.lm

@@ -1,400 +1,400 @@
-�	13411
-�	11301
-_	9980
-ا	3641
-�	3641
-_�	3002
-ل	2925
-�	2925
-��	2278
-ا�	2278
-_�	1985
-ي	1827
-�	1827
-�ل	1820
-�	1736
-م	1736
-ل�	1566
-��	1566
-_ا	1506
-و	1420
-�	1420
-ر	1140
-�	1140
-��	1114
-ل�	1114
-ا�	1075
-��	1075
-ن	1065
-�	1065
-�	1061
-ت	1061
-�	994
-ة	993
-�_	955
-ة_	954
-ع	913
-�	913
-��	904
-ي�	904
-��	861
-و�	861
-ب	851
-�	851
-�	804
-د	804
-م�	750
-��	750
-م�	696
-��	696
-_و	651
-�	575
-س	575
-ف	547
-�	547
-_م	536
-ه	509
-�	509
-ر�	497
-��	497
-ي_	494
-�_	494
-��	492
-ع�	492
-�	480
-ق	480
-و�	472
-��	472
-ن_	469
-�_	469
-��	450
-ت�	450
-ب�	444
-��	444
-ك	434
-�	434
-ي�	429
-��	429
-ح	406
-�	406
-�	394
-أ	394
-�ة	391
-��	368
-ن�	368
-��	352
-ر�	352
-د�	345
-��	345
-��	342
-ت�	342
-�م	331
-��	313
-ع�	313
-ط	309
-�	309
-��	307
-ف�	307
-��	303
-ب�	303
-��	302
-س�	302
-ج	301
-�	301
-_ب	298
-�ا	295
-��	294
-ق�	294
-ر_	291
-�_	291
-�_	288
-ا_	288
-م_	286
-�_	286
-�ن	283
-ت_	269
-�_	269
-_ف	255
-�ي	252
-�	252
-ص	252
-�_	247
-د_	247
-ل_	245
-�_	245
-��	238
-ه�	238
-��	235
-س�	235
-��	230
-ك�	230
-��	228
-ن�	228
-_ع	228
-�	223
-إ	223
-�ت	220
-�	218
-ش	218
-��	215
-أ�	215
-د�	212
-��	212
-خ	204
-�	204
-ح�	201
-��	201
-_أ	199
-�ي	198
-�ا	190
-ى	190
-�	190
-ى_	189
-�	189
-�_	189
-ض	189
-�ر	188
-�	186
-_ت	186
-ز	182
-��	176
-أ�	176
-�	174
-،	174
-،_	172
-�_	172
-ج�	171
-��	171
-��	168
-ح�	168
-�ي	167
-�ا	165
-ف�	158
-��	158
-إ�	154
-��	154
-�ل	154
-��	152
-ه�	152
-�ع	152
-�ا	150
-��	148
-ك�	148
-ش�	145
-��	145
-_ل	145
-�ا	144
-�ا	141
-�م	140
-ط�	138
-��	138
-�س	137
-�ي	135
-��	134
-ص�	134
-�ك	134
-�د	133
-ث	132
-�	132
-�ا	129
-��	128
-�ن	128
-خ�	128
-��	127
-ط�	127
-ق�	126
-��	126
-�ى	125
-�أ	124
-_إ	124
-�ي	122
-�ا	121
-�_	119
-ه_	119
-�ن	117
-�ي	116
-�	112
-ذ	112
-�ت	108
-�_	108
-ع_	108
-ص�	107
-��	107
-ج�	106
-��	106
-�_	104
-ب_	104
-_ي	103
-�ي	100
-�د	99
-غ	98
-�	98
-�ل	93
-�ع	93
-�و	93
-�ب	91
-ض�	89
-��	89
-و_	87
-�_	87
-�إ	85
-�	85
-ئ	85
-�د	85
-�ب	85
-�ل	83
-�ر	83
-�ر	83
-�_	82
-ف_	82
-�ا	81
-_ه	80
-�ل	80
-�ر	79
-_ك	79
-�ي	78
-�ت	78
-�ع	77
-�ة	77
-�م	77
-�ر	76
-�ر	76
-�ل	75
-�ق	75
-�ت	75
-�ش	73
-�ل	72
-�ة	72
-��	71
-�ا	71
-ز�	70
-إ�	69
-�ح	69
-�	69
-��	69
-ء	69
-�ع	68
-�ي	67
-_ن	67
-_س	67
-��	67
-�م	67
-ش�	67
-�س	67
-خ�	66
-��	66
-��	65
-ذ�	65
-�ا	65
-_ق	65
-�م	65
-_ح	64
-ء_	63
-�ا	63
-�_	63
-��	63
-ز�	63
-�ص	62
-�ء	61
-�ن	61
-�م	60
-�م	60
-ق_	60
-�ا	60
-�_	60
-�ج	59
-ً	59
-�	59
-�ن	58
-ئ�	57
-�ئ	57
-��	57
-�غ	56
-ك_	56
-�_	56
-�و	55
-�و	55
-�ي	54
-�و	54
-��	54
-�ر	54
-�ة	54
-ض�	54
-�ة	54
-�ع	54
-��	53
-�ح	53
-�ع	53
-�ب	53
-�ت	53
-ث�	53
-�ا	53
-�د	52
-�ت	52
-�_	52
-�ر	52
-�ن	51
-�ه	51
-�ي	51
-�ر	50
-�ة	50
-�و	50
-��	49
-ث�	49
-�ا	49
-ز_	49
-�س	49
-�د	49
-�خ	48
-�ل	48
-�ا	48
-_ش	48
-�و	48
-�ر	48
-��	47
-غ�	47
-�ف	47
-�و	46
-�_	46
-ض_	46
-�ع	45
-�	45
-ّ	45
-ط_	44
-�د	44
-�_	44
-�م	44
-�ف	44
-_ج	44
-�ب	44
-�ي	43
-�ي	43
-�ل	43
-�و	42
-�ر	42
-�ط	42
-�ق	42
-�ا	41
-�ف	41
-�و	41
-�ق	40
-�ص	40
-�ن	40
-�،	39
-�د	39
-�ر	39
-��	39
-�ي	39
-�و	39
-ة�	39
-�ح	39
-�_	38
-�د	38
-�ه	38
-س_	38
-�م	38
-�ة	37
-ح_	37
-�م	37
-�_	37
-�ت	37
+_	13714
+ا	4707
+ل	3908
+ي	2408
+م	2346
+ال	2310
+و	1917
+_ا	1909
+_ال	1867
+ر	1519
+ن	1505
+ت	1443
+ة	1291
+ع	1264
+ة_	1251
+د	1164
+ب	1148
+_و	867
+س	784
+ن_	747
+_م	742
+ف	730
+ه	670
+ق	644
+ي_	632
+ك	629
+ح	558
+أ	527
+ية	511
+ية_	487
+لم	443
+م_	435
+ا_	433
+_ب	411
+ط	402
+ج	391
+من	389
+الم	384
+ر_	382
+إ	380
+د_	365
+ت_	363
+ل_	359
+_ع	359
+وا	355
+ص	351
+_ف	351
+في	326
+_من	297
+_ت	288
+ات	287
+_في	286
+_أ	284
+ش	283
+من_	278
+في_	276
+خ	269
+لي	265
+ى	265
+ى_	264
+لا	243
+ات_	242
+ار	236
+ض	227
+_ل	226
+دي	224
+ز	221
+ما	219
+_وا	217
+عا	207
+ها	204
+وال	203
+را	198
+_إ	198
+ود	197
+ام	196
+مل	195
+لى_	189
+لك	189
+لى	189
+يا	183
+ين	183
+لس	182
+،	181
+،_	180
+ري	177
+ها_	173
+لع	170
+بي	168
+الس	166
+با	163
+عو	162
+عل	161
+الع	158
+سع	158
+عود	150
+ه_	149
+سعو	147
+ث	146
+و_	145
+مي	144
+ع_	143
+غ	142
+لت	141
+لأ	141
+ام_	140
+_ي	139
+ول	139
+اد	139
+ذ	137
+ب_	135
+ان	133
+لإ	133
+ين_	130
+الأ	129
+رة	125
+الإ	124
+رة_	123
+ودي	122
+ئ	122
+ف_	121
+_ك	120
+رب	119
+ير	118
+لب	116
+دية	116
+_س	115
+وي	115
+عد	114
+الت	112
+_عل	110
+وت	108
+لسع	107
+لل	105
+إل	105
+عر	104
+لح	102
+_إل	101
+_ن	101
+على	101
+_ق	100
+اع	100
+كا	100
+مع	99
+لق	99
+أن	99
+_وت	99
+الب	98
+در	97
+ء	96
+ملك	96
+مر	94
+بل	93
+_ه	93
+نا	93
+ار_	92
+كة	92
+سي	92
+عام	92
+ك_	90
+الح	90
+لن	90
+كة_	90
+سم	89
+نه	89
+لر	89
+مم	88
+ائ	87
+_عا	87
+شر	86
+ء_	86
+نت	85
+اء	84
+وم	84
+_لل	83
+طا	83
+الر	83
+_ح	83
+الق	83
+ون	82
+دو	82
+يم	81
+_با	81
+لص	79
+تم	78
+تر	78
+لج	77
+لة	77
+إلى	77
+ني	77
+مت	77
+دا	77
+دة	77
+اء_	76
+بر	76
+دة_	75
+لش	75
+لة_	75
+بال	75
+اح	75
+ممل	74
+الج	73
+ق_	73
+سا	72
+نو	72
+لد	72
+تو	72
+لغ	72
+_أن	72
+مة	71
+صا	71
+تع	70
+مة_	70
+لمم	70
+الص	70
+لعر	69
+تص	69
+عي	69
+ست	69
+اس	69
+ير_	68
+ور	68
+لي_	68
+لكة	68
+حد	68
+إن	68
+يو	68
+_و_	68
+ربي	68
+لو	67
+جا	67
+الش	67
+قا	66
+ال_	66
+ما_	66
+كم	66
+وب	66
+قر	65
+تي	65
+الد	65
+ز_	65
+اف	65
+مس	64
+الن	64
+الي	63
+خل	63
+ون_	63
+لك_	63
+يد	63
+لخ	63
+بن	63
+الخ	62
+هم	62
+ط_	62
+لف	62
+_مل	61
+قي	61
+_ج	61
+بع	60
+تق	59
+اد_	59
+مد	59
+كان	59
+عرب	59
+_ش	58
+ود_	58
+ان_	58
+بية	58
+مو	58
+نها	58
+_وي	57
+قد	57
+ته	57
+س_	56
+_مع	56
+اب	55
+أن_	55
+_ر	55
+لية	54
+خا	54
+يل	54
+رو	54
+جد	54
+ض_	54
+لا_	53
+زي	53
+أس	52
+حا	52
+ئي	51
+له	51
+تح	51
+ً	50
+تب	50
+يت	50
+بين	50
+تل	49
+ضا	49
+ملي	49
+وع	48
+ارة	48
+مط	48
+دم	48
+حل	48
+يع	48
+أو	47
+نف	47
+قة	47
+حر	47
+بة_	46
+در_	46
+حي	46
+رق	46
+إم	46
+قة_	46
+بة	46
+رات	46
+عة	45
+أم	45
+صد	45
+يس	45
+وه	45
+حو	45
+وف	45
+مال	45
+كل	44
+بد	44
+ول_	44
+قل	44
+اق	44
+هو	44
+رك	44
+_د	44
+الو	44
+طق	44
+عة_	44
+لم_	44
+يا_	44
+_بن	43
+لعا	43
+صل	43
+_بي	43
+ح_	42
+كو	42
+مح	42
+وج	42
+_وم	42
+لذ	42
+طار	41
+إما	41
+تا	41
+تي_	41
+_خ	41
+هد	41
+لمن	41
+رت	40
+ة،_	40
+مطا	40
+الا	40
+ة،	40
+ّ	40
+دول	40
+رية	40
+عن	40
+هم_	40
+بن_	40
+يد_	40
+اض	40
+سب	39
+يق	39
+_سع	39
+ظ	39
+طو	39
+رم	39
+نية	39
+سل	39
+تت	39
+وأ	39
+عب	38
+لإن	38
+_وه	38
+صاد	38
+الذ	38
+الغ	38
+_وأ	38
+مية	38
+اي	37
+بت	37
+اه	37
+يه	37
+فا	37
+هي	37
+_عد	37
+_كا	36
+ج_	36
+_كم	36
+_مس	36
+_عن	36
+نج	36
+كل_	36
+لط	36