scylla 0.7.0 → 0.7.5

data/lib/scylla/classifier.rb

@@ -5,7 +5,7 @@ module Scylla
     # limit    : Up to how many matching language results should be displayed
     # ngrams   : The total number of ngrams that are stored for each language
     # threshold: The threshold score for matches
-    def initialize(limit = 10, ngrams = 400, threshold = 1.05)
+    def initialize(limit = 10, ngrams = 400, threshold = 1.01)
       @limit = limit
       @ngrams = ngrams
       @threshold = threshold
@@ -13,15 +13,14 @@ module Scylla
 
     # Classifies a string to a list of languages in order of best match
     def classify_string(text)
-      @input = ""
-      text.each_line { |line| @input += line.strip }
+      @input = text
       classify
     end
 
     # Classifies a file to a list of languages in order of best match
     def classify_file(path)
       @input = ""
-      File.readlines(path).each { |line| @input += line.strip }
+      File.readlines(path).each { |line| @input += " " + line.strip }
       classify
     end
 

data/lib/scylla/generator.rb

@@ -29,7 +29,7 @@ module Scylla
     # lib/scylla/lms
     def write_lm(path)
       text = ""
-      File.open(path).each { |line| text += clean(line) }
+      File.open(path).each { |line| text += " " + line }
       p "Creating language map for " + path
       lm = create_lm(text, true)
       lmname = File.join(@dirlm, File.basename(path, ".txt") + ".lm")
@@ -45,21 +45,18 @@ module Scylla
     end
 
     def clean(string)
-      string.strip!
       string = Sanitize.clean(string)
       string = CGI.unescapeHTML(string)
       string.gsub!(/(?:http|https):\/\/[a-z0-9]+(?:[\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(?:(?::[0-9]{1,5})?\/[^\s]*)?/, "")
-      string.gsub!(/[\*\^><!\"#\$%&\'\(\)\*\+:;=\?@\{\}\[\]|-]/,"")
-      string.strip
+      string.gsub!(/[\*\^><!\"#\$%&\'\(\)\*\+:;=\?@\{\}\[\]|\-\n\r0-9]/," ")
+      string.strip.split(" ").join(" ")
     end
 
     # Creates a language map for a given input string. 
     # The frequencies boolean specifies whether or not the method should
     # return the freqencies of the ngrams, or simply an array in sorted order
     def create_lm(input, frequencies = false)
-      text = ""
-      input.each_line {|line| text += clean(line) }
-      input = text
+      input = clean(input)
       ngram = Hash.new
       input.split(/[\d\s\[\]]/).each do |word|
         word = "_" + word + "_";

data/lib/scylla/lms/afrikaans.lm

@@ -1,4 +1,4 @@
-_	1596
+_	1772
 e	609
 a	326
 n	308
@@ -8,393 +8,393 @@ r	224
 t	206
 s	202
 d	190
-e_	144
+e_	158
 l	126
 g	125
 m	124
 ie	123
 k	122
-n_	112
-t_	92
+n_	120
+t_	99
 h	89
+_d	84
 er	83
 an	83
+ie_	82
 v	82
 y	81
-_d	76
-ie_	76
+_h	79
+s_	76
 aa	75
-_h	74
 w	73
-di	73
-s_	71
-_n	69
-_v	66
-en	65
+di	72
+_n	71
+_v	71
+en	64
+r_	63
+y_	63
+_s	62
 .	62
+_di	61
 ge	60
-r_	60
-y_	59
 u	59
-_di	58
-_s	57
-et	56
-_m	53
+._	59
+_m	56
+et	54
 die	51
+_w	51
 b	50
+die_	49
+_die	48
 te	48
+_die_	47
 ee	47
+an_	46
 ni	46
-_w	46
+et_	46
+_e	46
 in	46
-die_	45
-_die	45
-an_	45
-et_	44
+_o	44
 p	43
-_e	43
 de	42
-_die_	41
-_o	40
-._	38
-nd	36
-ar	35
+_g	40
+k_	38
+_a	37
 at	35
-_g	35
 el	35
-oe	34
-k_	34
+g_	35
+ar	35
+nd	34
 wa	34
 nie	34
+oe	34
 ,	33
-g_	33
-_a	33
 _ni	33
+_i	33
+,_	33
 he	32
-oo	32
 _nie	32
-,_	31
+_wa	32
+oo	32
+_ge	30
+d_	30
 ek	30
-_i	29
+_he	30
+en_	30
+_k	29
 om	29
-_wa	28
-me	28
-en_	28
-_he	28
-_k	28
+_b	29
 va	28
 aan	28
+me	28
 is	27
+er_	27
+m_	27
+_en	27
+_t	27
 f	26
-_en	26
 at_	26
-_ge	26
+l_	25
 or	25
-es	25
-er_	24
-it	24
-on	24
 het	24
-_b	24
+as	24
+es	24
 ma	24
-_t	24
-l_	24
-m_	24
+_het	24
+on	24
+it	24
 aar	24
-as	24
-d_	23
 van	23
+li	22
+_het_	22
+het_	22
 ra	22
 ou	22
-_het	22
-li	22
+te_	22
+_en_	21
 _va	21
-het_	21
 be	21
-da	20
 om_	20
-al	20
 ng	20
-_het_	20
+da	20
+al	20
 rd	20
-te_	19
 st	19
-_en_	19
-_van	19
 ke	19
+nie_	19
+_van	19
+_sy	18
+sy_	18
 van_	18
-is_	18
 sy	18
+it_	18
+is_	18
+_nie_	18
 ag	18
-sy_	18
+_sy_	18
+der	17
+_hy_	17
+_me	17
+in_	17
 _van_	17
-ro	17
--	17
-aar_	17
-_ma	17
+_hy	17
 ar_	17
+aan_	17
+hy	17
+ro	17
+and	17
 re	17
-nie_	17
-nde	17
+_ma	17
+aar_	17
 se	17
 le	17
-and	17
-der	17
-hy	17
-as_	16
-_hy	16
+as_	17
+hy_	17
 H	16
-ei	16
-_me	16
 _H	16
-aan_	16
-_nie_	16
+ei	16
 we	16
-ho	15
-_sy_	15
-it_	15
-ko	15
-em	15
-ns	15
-ig	15
+_be	16
+nde	16
 ik	15
+_in	15
+ns	15
+ek_	15
+em	15
 so	15
-_sy	15
-rs	14
-in_	14
-hy_	14
+_te	15
+ig	15
+ko	15
+ho	15
+mo	14
+_te_	14
+_in_	14
+_n_	14
 ha	14
+ve	14
+ou_	14
+_is	14
+u_	14
+_vi	14
 si	14
-vo	14
-nt	14
 vi	14
-mo	14
-ve	14
-no	13
+nt	14
+vo	14
 rt	13
-_n_	13
-ou_	13
-u_	13
-_in	13
-eer	13
-_hy_	13
-wo	13
+_is_	13
+rs	13
+op	13
 e,	13
-ag_	13
-_be	13
+_ve	13
 e,_	13
-_vi	13
-op	13
-ek_	12
-_vo	12
-_te_	12
-ew	12
-ver	12
+_mo	13
+ag_	13
+eer	13
+_da	13
+_vo	13
+no	13
+wo	13
+_ha	13
 _om_	12
+ver	12
+_aan	12
+_aa	12
 ne	12
-was	12
-_is	12
-e.	12
-al_	12
+_wat	12
 wat	12
-ing	12
+_so	12
+_p	12
+al_	12
 _om	12
-_te	12
-ed	12
-_so	11
-_is_	11
-vir_	11
-ord	11
-bl	11
-vir	11
+ing	12
+_was	12
+was	12
+f_	12
 ka	11
-_ko	11
-ta	11
-met	11
-wat_	11
-_was	11
-_on	11
-_mo	11
-ir	11
-moe	11
+_was_	11
 maa	11
-S	11
-_p	11
-_ha	11
 _no	11
 ir_	11
-met_	10
-sta	10
-_aan	10
-to	10
-f_	10
-nder	10
-sie	10
-_in_	10
-oor	10
-_vir	10
-E	10
-_vir_	10
-ui	10
-_da	10
-was_	10
-ers	10
+moe	11
+was_	11
+S	11
+_vir_	11
+ta	11
+_ko	11
+p_	11
+wat_	11
+ir	11
+e.	11
+ed	11
+bl	11
+vir_	11
+_vir	11
+_ver	11
+_wat_	11
+_ek	11
+ew	11
+ord	11
+met	11
+vir	11
+_ek_	11
+_on	11
 vr	10
 ri	10
-_was_	10
-lik	10
+de_	10
+_met_	10
+_moe	10
 raa	10
-_aa	10
-_ve	10
-nk	10
+ui	10
+nder	10
+E	10
+e._	10
+sie	10
+sta	10
+met_	10
+_al	10
 nge	10
-eg	10
-p_	10
-A	9
-de_	9
-ges	9
-_wat	9
-pr	9
-ie.	9
-ll	9
-_ver	9
-_ho	9
-el_	9
+to	10
+_met	10
+lik	10
+oor	10
+_r	9
+ers	9
+og	9
 _vr	9
-_Hy	9
-_met	9
-man	9
-_ek	9
-_met_	9
-_se	9
+nk	9
+ll	9
 _maa	9
-og	9
+_se	9
 Hy	9
-_moe	9
-wor	8
-rte	8
+el_	9
+ie.	9
+man	9
+Hy_	9
+_ho	9
+A	9
+pr	9
+_Hy_	9
+_S	9
+nd_	9
+_Hy	9
+ges	9
+word	8
+_l	8
+ga	8
+M	8
+_pr	8
+pa	8
+_op	8
+nie.	8
 aat	8
-my_	8
+taa	8
+my	8
 ak	8
-j	8
+na	8
+my_	8
+_M	8
 es_	8
-_al	8
-_op	8
-_ek_	8
-nie.	8
-_Hy_	8
-D	8
-_l	8
+og_	8
+kl	8
+op_	8
 _to	8
-my	8
-ga	8
-_nie.	8
-_my_	8
-taa	8
-Hy_	8
-_r	8
 din	8
-M	8
-na	8
-pa	8
-_wat_	8
-word	8
-sk	8
-ewe	8
+_my_	8
+rte	8
+_E	8
+een	8
+ke_	8
 eu	8
+ie._	8
+ewe	8
+sk	8
+_nie.	8
+wor	8
+eg	8
 _my	8
-kl	8
-een	8
-ier	7
-ke_	7
-_dit	7
-op_	7
-og_	7
+_ne	8
+D	8
+j	8
+la	7
+le_	7
 gaa	7
-er.	7
-dit	7
-gaan	7
-_maar	7
-_ges	7
-r.	7
-ur	7
-wee	7
-end	7
+s._	7
+_dit_	7
 oe_	7
+ie,	7
+er._	7
 staa	7
-_pr	7
-e._	7
-ste	7
+tee	7
+gaan_	7
 _st	7
-ie,_	7
-la	7
-gew	7
-ie,	7
-maar_	7
-id	7
-nd_	7
-do	7
-a_	7
+maar	7
 lin	7
-tee	7
-ti	7
-le_	7
-s.	7
-ders	7
-ger	7
 br	7
+eer_	7
+ger	7
+r.	7
+ie,_	7
+maar_	7
 erd	7
-maar	7
+r._	7
 O	7
-uit	7
+end	7
+dit_	7
+s.	7
+id	7
 ng_	7
-_ne	7
 oet	7
-rs_	6
-ns_	6
+dit	7
+ur	7
+_maar	7
+ge_	7
+_ges	7
+der_	7
+a_	7
+er.	7
+_wo	7
+ns_	7
+nie._	7
+ier	7
+_dit	7
+ste	7
+wee	7
+gaan	7
+uit	7
+_hier	6
+_word	6
+eur	6
+ens	6
+r,	6
+_as	6
+_nog_	6
 ent	6
-nog	6
+_ui	6
+ig_	6
 dat	6
+eel	6
+nie,_	6
 _wi	6
-nig	6
-rm	6
-oet_	6
-eer_	6
-ens	6
-inde	6
-_wo	6
-_hie	6
-_hi	6
-eke	6
-il	6
-_ui	6
-ie._	6
-wi	6
-nda	6
-_hier	6
-ing_	6
-lle	6
-hie	6
-toe	6
-os	6
-eni	6
-aak	6
-dat_	6
-yd	6
-hi	6
-mer	6
-kom	6
 hier	6
-_uit	6
-lo	6
+ti	6
+hi	6
+nog_	6
+nders	6
+dat_	6
+_de	6
+gew	6
 _nog	6
+rs_	6
 ad	6
-s._	6
-eid	6
-eli	6
-_nie,	6
-r,	6
-_bl	6
-gaan_	6
 ige	6
+toe	6
+_nie,	6
+r,_	6
+nig	6
+eli	6
 am	6
+os	6
+_kl	6
+lo	6
+do	6
+nog	6