scylla 0.5.0 → 0.6.0

data/Gemfile

@@ -8,10 +8,12 @@ source "http://rubygems.org"
 group :development do
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.6.4"
+  gem "mail"
+  gem "sanitize"
 end
 
 group :test do
   gem "shoulda", ">= 0"
-  gem "mocha", "~> 0.9.12", :require => nil
-  gem "ruby-debug",   "~> 0.10.4"
+  gem "mocha"
+  gem "ruby-debug", "~> 0.10.4"
 end

data/Gemfile.lock

@@ -3,13 +3,21 @@ GEM
   specs:
     columnize (0.3.4)
     git (1.2.5)
+    i18n (0.6.0)
     jeweler (1.6.4)
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
     linecache (0.46)
       rbx-require-relative (> 0.0.4)
+    mail (2.3.0)
+      i18n (>= 0.4.0)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.16)
     mocha (0.9.12)
+    nokogiri (1.4.7)
+    polyglot (0.3.2)
     rake (0.9.2)
     rbx-require-relative (0.0.5)
     ruby-debug (0.10.4)
@@ -17,7 +25,12 @@ GEM
       ruby-debug-base (~> 0.10.4.0)
     ruby-debug-base (0.10.4)
       linecache (>= 0.3)
+    sanitize (2.0.3)
+      nokogiri (>= 1.4.4, < 1.6)
     shoulda (2.11.3)
+    treetop (1.4.10)
+      polyglot
+      polyglot (>= 0.3.1)
 
 PLATFORMS
   ruby
@@ -25,6 +38,8 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.0.0)
   jeweler (~> 1.6.4)
-  mocha (~> 0.9.12)
+  mail
+  mocha
   ruby-debug (~> 0.10.4)
+  sanitize
   shoulda

data/lib/scylla/classifier.rb

@@ -5,7 +5,7 @@ module Scylla
     # limit    : Up to how many matching language results should be displayed
     # ngrams   : The total number of ngrams that are stored for each language
     # threshold: The threshold score for matches
-    def initialize(limit = 10, ngrams = 400, threshold = 1.06)
+    def initialize(limit = 10, ngrams = 400, threshold = 1.05)
       @limit = limit
       @ngrams = ngrams
       @threshold = threshold

data/lib/scylla/generator.rb

@@ -1,3 +1,6 @@
+require 'sanitize'
+require 'cgi'
+
 module Scylla
   class Generator
     attr_accessor :dirtext, :dirlm, :minsize
@@ -10,7 +13,7 @@ module Scylla
       @minsize = minsize
     end
 
-    # This loads all the .txt files in the specified source training text folder  
+    # Loads all the .txt files in the specified source training text folder  
     # and creates language maps using ngram frequencies. The maps are stored in
     # lib/scylla/lms as .lm files
     def train
@@ -26,7 +29,7 @@ module Scylla
     # lib/scylla/lms
     def write_lm(path)
       text = ""
-      File.open(path).each { |line| text += line.strip }
+      File.open(path).each { |line| text += clean(line) }
       p "Creating language map for " + path
       lm = create_lm(text, true)
       lmname = File.join(@dirlm, File.basename(path, ".txt") + ".lm")
@@ -41,12 +44,21 @@ module Scylla
       end
     end
 
-    # This creates a language map for a given input string. 
+    def clean(string)
+      string.strip!
+      string = Sanitize.clean(string)
+      string = CGI.unescapeHTML(string)
+      string.gsub!(/(?:http|https):\/\/[a-z0-9]+(?:[\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(?:(?::[0-9]{1,5})?\/[^\s]*)?/, "")
+      string.gsub!(/[\*\^><!\"#\$%&\'\(\)\*\+:;=\?@\{\}\[\]|-]/,"")
+      string.strip
+    end
+
+    # Creates a language map for a given input string. 
     # The frequencies boolean specifies whether or not the method should
     # return the freqencies of the ngrams, or simply an array in sorted order
     def create_lm(input, frequencies = false)
       text = ""
-      input.each_line {|line| text += line.strip }
+      input.each_line {|line| text += clean(line) }
       input = text
       ngram = Hash.new
       input.split(/[\d\s\[\]]/).each do |word|

data/lib/scylla/lms/afrikaans.lm

@@ -8,26 +8,27 @@ r	224
 t	206
 s	202
 d	190
-e_	141
+e_	144
 l	126
 g	125
 m	124
 ie	123
 k	122
-n_	111
+n_	112
 t_	92
 h	89
-an	83
 er	83
+an	83
 v	82
 y	81
 _d	76
+ie_	76
 aa	75
-ie_	74
-_h	73
+_h	74
 w	73
 di	73
 s_	71
+_n	69
 _v	66
 en	65
 .	62
@@ -38,7 +39,6 @@ u	59
 _di	58
 _s	57
 et	56
-_n	56
 _m	53
 die	51
 b	50
@@ -47,354 +47,354 @@ ee	47
 ni	46
 _w	46
 in	46
-_die	45
 die_	45
+_die	45
 an_	45
 et_	44
-_e	43
 p	43
+_e	43
 de	42
 _die_	41
 _o	40
-._	36
+._	38
 nd	36
-_g	35
-el	35
 ar	35
 at	35
-nie	34
+_g	35
+el	35
 oe	34
 k_	34
 wa	34
-_ni	33
+nie	34
 ,	33
+g_	33
 _a	33
+_ni	33
 he	32
 oo	32
-g_	32
 _nie	32
+,_	31
 ek	30
-,_	30
-om	29
 _i	29
-en_	28
+om	29
+_wa	28
 me	28
-va	28
-_k	28
+en_	28
 _he	28
+_k	28
+va	28
 aan	28
-_wa	28
 is	27
-_en	26
 f	26
+_en	26
 at_	26
 _ge	26
-es	25
 or	25
-it	24
-as	24
-_t	24
-ma	24
+es	25
 er_	24
+it	24
 on	24
+het	24
 _b	24
-m_	24
+ma	24
+_t	24
 l_	24
-het	24
+m_	24
 aar	24
+as	24
 d_	23
 van	23
-li	22
-_het	22
-ou	22
 ra	22
+ou	22
+_het	22
+li	22
 _va	21
-be	21
 het_	21
-"	21
-_het_	20
-rd	20
-ng	20
+be	21
+da	20
 om_	20
 al	20
-da	20
-_van	19
+ng	20
+_het_	20
+rd	20
+te_	19
 st	19
-ke	19
-'	19
 _en_	19
+_van	19
+ke	19
+van_	18
+is_	18
 sy	18
 ag	18
-is_	18
 sy_	18
-van_	18
-te_	18
-re	17
-and	17
-aar_	17
-se	17
+_van_	17
 ro	17
 -	17
-_van_	17
-hy	17
-le	17
-der	17
+aar_	17
+_ma	17
 ar_	17
+re	17
+nie_	17
 nde	17
-_ma	17
+se	17
+le	17
+and	17
+der	17
+hy	17
+as_	16
 _hy	16
-_me	16
 H	16
-we	16
-as_	16
 ei	16
+_me	16
+_H	16
 aan_	16
-ns	15
-ik	15
+_nie_	16
+we	16
 ho	15
-em	15
-so	15
-nie_	15
 _sy_	15
-_sy	15
-_'	15
-ko	15
 it_	15
-_H	15
+ko	15
+em	15
+ns	15
 ig	15
-'n	14
+ik	15
+so	15
+_sy	15
+rs	14
+in_	14
 hy_	14
-mo	14
 ha	14
+si	14
+vo	14
+nt	14
 vi	14
+mo	14
 ve	14
-nt	14
-_nie_	14
-vo	14
-rs	14
-si	14
-in_	14
-ou_	13
-'n_	13
-_in	13
-_be	13
-_'n	13
-_'n_	13
 no	13
-_vi	13
-_hy_	13
 rt	13
+_n_	13
+ou_	13
 u_	13
+_in	13
+eer	13
+_hy_	13
+wo	13
+e,	13
 ag_	13
+_be	13
+e,_	13
+_vi	13
 op	13
-e,	13
-wo	13
-eer	13
-ed	12
-ing	12
-ne	12
-e,_	12
-ver	12
-ew	12
-_te_	12
 ek_	12
 _vo	12
-_is	12
-was	12
-_te	12
-_om	12
+_te_	12
+ew	12
+ver	12
 _om_	12
-wat	12
+ne	12
+was	12
+_is	12
+e.	12
 al_	12
+wat	12
+ing	12
+_om	12
+_te	12
+ed	12
+_so	11
 _is_	11
+vir_	11
+ord	11
 bl	11
+vir	11
+ka	11
+_ko	11
 ta	11
-_mo	11
-_no	11
-_so	11
-moe	11
-ord	11
 met	11
-maa	11
+wat_	11
 _was	11
-ka	11
-vir	11
-ir	11
 _on	11
-vir_	11
-_ko	11
-ir_	11
-_p	11
-wat_	11
-e.	11
+_mo	11
+ir	11
+moe	11
+maa	11
 S	11
+_p	11
+_ha	11
+_no	11
+ir_	11
 met_	10
-sie	10
-p_	10
-nge	10
-was_	10
-vr	10
-_aa	10
+sta	10
+_aan	10
 to	10
-_ha	10
-ers	10
-raa	10
+f_	10
+nder	10
+sie	10
 _in_	10
 oor	10
-ri	10
-nder	10
-eg	10
-lik	10
+_vir	10
 E	10
+_vir_	10
 ui	10
 _da	10
+was_	10
+ers	10
+vr	10
+ri	10
+_was_	10
+lik	10
+raa	10
+_aa	10
 _ve	10
-sta	10
 nk	10
-f_	10
-_vir	10
-_was_	10
-_aan	10
-_vir_	10
-_ver	9
+nge	10
+eg	10
+p_	10
+A	9
+de_	9
 ges	9
+_wat	9
+pr	9
 ie.	9
 ll	9
-man	9
-_met	9
-de_	9
+_ver	9
 _ho	9
-_vr	9
-_moe	9
-pr	9
 el_	9
+_vr	9
+_Hy	9
+_met	9
+man	9
 _ek	9
+_met_	9
+_se	9
 _maa	9
-Hy	9
-_wat	9
 og	9
-."	9
-_se	9
-A	9
-_met_	9
+Hy	9
+_moe	9
+wor	8
 rte	8
-_l	8
-_my	8
 aat	8
-M	8
-wor	8
-_Hy	8
-_my_	8
-ga	8
 my_	8
-een	8
-my	8
+ak	8
+j	8
 es_	8
-Hy_	8
+_al	8
 _op	8
-din	8
-kl	8
-_Hy_	8
-taa	8
 _ek_	8
-ewe	8
+nie.	8
+_Hy_	8
+D	8
+_l	8
 _to	8
-j	8
-eu	8
-ak	8
-sk	8
+my	8
+ga	8
 _nie.	8
+_my_	8
+taa	8
+Hy_	8
 _r	8
-word	8
-nie.	8
+din	8
+M	8
 na	8
-_al	8
-_wat_	8
 pa	8
-D	8
-le_	7
-erd	7
-uit	7
-br	7
-staa	7
-a_	7
-op_	7
-_pr	7
+_wat_	8
+word	8
+sk	8
+ewe	8
+eu	8
+_my	8
+kl	8
+een	8
 ier	7
+ke_	7
+_dit	7
+op_	7
+og_	7
+gaa	7
 er.	7
-lin	7
+dit	7
+gaan	7
 _maar	7
-oe_	7
-nd_	7
-id	7
-do	7
-ders	7
-ke_	7
 _ges	7
 r.	7
-gew	7
-ie,_	7
+ur	7
+wee	7
 end	7
+oe_	7
+staa	7
+_pr	7
+e._	7
+ste	7
 _st	7
-O	7
+ie,_	7
+la	7
+gew	7
+ie,	7
+maar_	7
+id	7
+nd_	7
+do	7
+a_	7
+lin	7
 tee	7
-ur	7
-dit	7
-gaan	7
-_ne	7
-wee	7
-ger	7
-oet	7
+ti	7
+le_	7
 s.	7
+ders	7
+ger	7
+br	7
+erd	7
 maar	7
-ie,	7
-maar_	7
-gaa	7
-_dit	7
-og_	7
-la	7
-ste	7
+O	7
+uit	7
+ng_	7
+_ne	7
+oet	7
+rs_	6
+ns_	6
+ent	6
 nog	6
+dat	6
+_wi	6
+nig	6
+rm	6
+oet_	6
 eer_	6
-_bl	6
-_hier	6
-_hi	6
-ns_	6
-gaan_	6
-toe	6
-yd	6
-lle	6
 ens	6
-eur	6
-_u	6
-nders	6
-dat_	6
-os	6
-hi	6
-eid	6
+inde	6
 _wo	6
-t.	6
-_uit	6
-ert	6
-_as	6
+_hie	6
+_hi	6
+eke	6
+il	6
+_ui	6
+ie._	6
+wi	6
 nda	6
+_hier	6
+ing_	6
+lle	6
+hie	6
+toe	6
+os	6
+eni	6
 aak	6
-ng_	6
-ind	6
-dat	6
+dat_	6
+yd	6
+hi	6
 mer	6
-oet_	6
-nie,	6
-_nie,	6
-rs_	6
-nig	6
+kom	6
+hier	6
+_uit	6
 lo	6
-e._	6
-r,_	6
-ie._	6
+_nog	6
 ad	6
-hier	6
-am	6
-eke	6
-ige	6
+s._	6
+eid	6
 eli	6
+_nie,	6
+r,	6
+_bl	6
+gaan_	6
+ige	6
+am	6