scylla 0.1.0 → 0.2.0

data/Gemfile

@@ -13,5 +13,4 @@ end
 group :test do
   gem "shoulda", ">= 0"
   gem "mocha", "~> 0.9.12", :require => nil
-  gem "ruby-debug"
 end

data/Gemfile.lock

@@ -1,22 +1,13 @@
 GEM
   remote: http://rubygems.org/
   specs:
-    columnize (0.3.4)
     git (1.2.5)
     jeweler (1.6.4)
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
-    linecache (0.46)
-      rbx-require-relative (> 0.0.4)
     mocha (0.9.12)
     rake (0.9.2)
-    rbx-require-relative (0.0.5)
-    ruby-debug (0.10.4)
-      columnize (>= 0.1)
-      ruby-debug-base (~> 0.10.4.0)
-    ruby-debug-base (0.10.4)
-      linecache (>= 0.3)
     shoulda (2.11.3)
 
 PLATFORMS
@@ -26,5 +17,4 @@ DEPENDENCIES
   bundler (~> 1.0.0)
   jeweler (~> 1.6.4)
   mocha (~> 0.9.12)
-  ruby-debug
   shoulda

data/README.rdoc

@@ -2,6 +2,28 @@
 
 Scylla is a language categorizing gem that allows you to guess the language of a given text. Scylla is a Ruby port of TextCat (http://www.let.rug.nl/~vannoord/TextCat) and is based on the text categorization algorithm presented in Cavnar, W. B. and J. M. Trenkle, ``N-Gram-Based Text Categorization'' In Proceedings of Third Annual Symposium on Document Analysis and Information Retrieval, Las Vegas, NV, UNLV Publications/Reprographics, pp. 161-175, 11-13 April 1994.
 
+Installation:
+
+gem install scylla
+
+Usage:
+
+require 'scylla'
+
+"this is english text".language
+ => "english" 
+
+"Este es un texto español".language
+=> "spanish"
+
+Multiple results for other possible languages:
+
+"isso poderia ser confundido com espanhol, bem".language
+=> "portuguese"
+
+"isso poderia ser confundido com espanhol, bem".guess
+=> ["portuguese", "spanish"]
+
 == Contributing to scylla
  
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/bin/scylla

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'scylla'
+phrase = ""
+puts "Welcome to Scylla Language Guesser"
+puts "Enter a phrase which you would like to identify"
+puts "Type exit to quit"
+while(phrase != "exit")
+  puts "Phrase:"
+  STDOUT.flush
+  phrase = gets.chomp
+  puts phrase.guess.join(" or ")
+end

data/lib/scylla/classifier.rb

@@ -1,6 +1,6 @@
 module Scylla
   class Classifier
-    attr_accessor :limit, :dir, :ngrams, :threshold, :input
+    attr_accessor :limit, :ngrams, :threshold, :input
 
     # limit    : Up to how many matching language results should be displayed
     # ngrams   : The total number of ngrams that are stored for each language
@@ -30,7 +30,7 @@ module Scylla
       results = Hash.new
       languages = Scylla::Loader.languages
       if languages.empty?
-        p "No languages (.lm files) found. Please run rake scylla:train after placing your training texts in the source_texts directory."
+        p "No languages (.lm files) found in + " + Scylla::Loader.dir + ". Please run rake scylla:train after placing your training texts in the source_texts directory."
         return
       end
       sg = Scylla::Generator.new

data/lib/scylla/generator.rb

@@ -14,7 +14,7 @@ module Scylla
     # and creates language maps using ngram frequencies. The maps are stored in
     # lib/scylla/lms as .lm files
     def train
-      languages = Dir.glob("**/*.lm")
+      languages = Dir.glob(@dirlm + "/*.lm")
       textpaths = Dir.glob(@dirtext + "/*.txt")
       languages.each {|l| File.delete(l) }
       textpaths.each do |path|

data/lib/scylla/lms/13375P33K.lm

@@ -0,0 +1,400 @@
+_	23034
+__	3952
+|	1114
+|_	748
+n	708
+r	683
+,	659
+,_	614
+_|	605
+.	575
+_n	563
+h	516
+_r	510
+._	505
+d	493
+z	485
+@	447
+r_	443
+t	428
+p	425
+_|_	395
+h_	365
+_d	363
+z_	350
+n_	350
+c	345
+v	339
+N	331
+_r_	326
+d_	319
+_p	319
+f	302
+l	297
+u	295
+_n_	285
++	273
+R	271
+_z	264
+m	258
+t_	251
+_t	250
+y	248
+_d_	241
+g	240
+_f	239
+k	229
+f_	221
+_c	220
+H	208
+U	203
+_h	203
+w	202
+_w	202
+m_	199
+_,	195
+_u	193
+\	190
+_f_	190
+_z_	189
+_+	187
+T	187
+L	187
+v_	184
+_N	179
+_,_	175
+D	174
+I	174
+_m	173
+p_	170
+_g	170
+_y	169
+(	169
+_v	168
+y_	165
+b	163
+_@	162
+e	158
+_h_	156
+_l	153
+g_	153
+_b	152
+_t_	151
+_|\	149
+_.	149
+|\	149
+_(	148
+R_	143
+l_	140
+_k	138
+k_	136
+c_	134
+_._	134
+_w_	133
+w_	133
+T_	130
+N_	125
+-	121
+_m_	120
++_	119
+E	118
+_y_	118
+O	117
+_g_	114
+_p_	112
+a	111
+x	107
+o	107
+W	106
+i	105
+|\|	104
+_|\|	104
+\|	104
+_c_	104
+M	102
+u_	102
+$	102
+vv	101
+H_	97
+L_	94
+D_	91
+_I	90
+F	88
+/	86
+\/	86
+_R	86
+_T	84
+s	84
+_W	84
+_U	83
+b_	83
+_b_	83
+_l_	81
+x_	81
+_v_	79
+_D	78
+_u_	78
+(_	77
+_vv	77
+_s	76
+$_	72
+P	70
+_L	69
+_M	68
+_(_	67
+e_	67
+@R	66
+\|_	66
+_k_	66
+_|\|_	66
+|\|_	66
+Y	66
+-|	65
+|-	65
+_T_	64
+_x	62
+_+_	62
+vv_	62
+@n	62
+_F	61
+F_	60
+IN	60
+_N_	60
+_H	59
+_P	59
+@_	57
+_i	56
+_a	56
+C	54
+_@R	54
+s_	50
+tz	50
+gh	50
+_R_	49
+_vv_	49
+_C	48
+!	48
+_F_	48
+_+h	47
+_O	47
++h	47
+cH	47
+|-|	47
+_x_	46
+_|\/	45
+\/|	45
+/|	45
+|\/|	45
+_|\/|	45
+|\/	45
+u|_	44
+u|	44
+tz_	44
+E_	44
+_IN	43
+Ul	43
+_gh	43
+Wh	43
+)	43
+_s_	42
+||	42
+A	41
+Wh_	41
+@R_	40
+Or	40
+_L_	40
+p|	40
+nd	40
+z,_	39
+_@R_	39
+p@	39
+_Wh	39
+G	39
+_p@	39
+_wh	39
+wh	39
+z,	39
+d,	38
+rz	38
+_rz	38
+nD	38
+aR	37
+df	37
+_D_	37
+(h	37
+M_	37
+_Wh_	37
+c|	37
+HE	36
+(h_	36
+_d,	36
+Up	36
+_nT	35
++H	35
+_@n	35
+nT	35
+LL	35
+_d@	35
+cH_	35
+d@	35
+B	34
+_u|	34
+_u|_	34
+j	34
+_j	34
+n.	34
+_Up	34
+y,	34
+pdf	34
+pd	34
+_@$	33
+_||	33
+d,_	33
+@$	33
+lt	33
+Y_	32
+_aR	32
+_zUl	32
+Rc	32
+q	32
+p|_	32
+_$	32
+_)	32
+_p|	32
+_zU	32
+z.	32
+zUl	32
+_p|_	32
+y,_	32
+_aRc	32
+aRc	32
+M@	32
+zU	32
+@r	32
+_B	31
+_d,_	31
+|-|_	31
+o_	31
+nd_	31
+-|_	31
+|\/|_	30
+_M_	30
+/|_	30
+\/|_	30
+@g	29
+_b|_	29
+_+H	29
+|__	29
+b|	29
+gh_	29
+r.	29
+_b|	29
+h@	29
+O_	29
+b|_	29
+)_	29
+PH	28
+||_	28
+De	28
+G_	28
+_nT_	28
+nT_	28
+_H_	28
+ND	28
+n._	28
+ve	28
+z._	28
+K	28
+_nd	28
+D,	27
+W_	27
+I_	27
+_PH	27
+_K	27
+_n.	27
+ve_	27
+ph	26
+rE	26
+_+hO	26
+_tz	26
+_rE	26
+@n_	26
+hO	26
+Up_	26
+n,	26
+_(h	26
++hO	26
+LL_	26
+r._	26
+n,_	26
+_(h_	25
+=	25
+fO	25
+rz_	25
+!=	25
+_r.	25
++o	25
+_+HE	25
++HE	25
+P_	25
+_tz_	25
+_rz_	25
+fOr	25
+_pR_	24
+_wh_	24
+Ult	24
+_pR	24
+_Up_	24
+nt	24
+wh_	24
+Or_	24
+\/_	24
+'	24
+_nt	24
+zUltz	24
+iN	24
+_!	24
+_zUlt	24
+@g_	24
+Ultz	24
+/_	24
+pR_	24
+pR	24
+zUlt	24
+_+o	24
+_!=	24
+$.	24
+ltz	24
+In	24
+_iN	24
+RcH	23
+rv_	23
+rv	23
+_|\|d	23
+|d	23
+_pd	23
+|\|d	23
+aRcH	23
+_rv	23
+\|d	23
+g|_	23
+d.	23
+!_	23
+_aRcH	23
+_n,_	23
+_pdf	23
+rc	23
+,.	23
+D,_	23
+_be	23
+_rv_	23
+g|	23
+be	23
+_n,	23
+|_|	22
+\|d_	22
+BuT	22
+,,_	22
+Bu	22
+nc	22
+$._	22