scylla 0.8.0 → 0.8.29

data/Gemfile

@@ -10,6 +10,10 @@ group :development do
   gem "jeweler", "~> 1.6.4"
   gem "mail"
   gem "sanitize"
+  gem "character-encodings"
+  gem "wikipedia-client"
+  gem "json"
+  gem "unicode"
 end
 
 group :test do

data/Gemfile.lock

@@ -1,6 +1,7 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    character-encodings (0.4.1)
     columnize (0.3.4)
     git (1.2.5)
     i18n (0.6.0)
@@ -8,6 +9,7 @@ GEM
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
+    json (1.6.3)
     linecache (0.46)
       rbx-require-relative (> 0.0.4)
     mail (2.3.0)
@@ -18,7 +20,7 @@ GEM
     mocha (0.9.12)
     nokogiri (1.4.7)
     polyglot (0.3.2)
-    rake (0.9.2)
+    rake (0.9.2.2)
     rbx-require-relative (0.0.5)
     ruby-debug (0.10.4)
       columnize (>= 0.1)
@@ -31,15 +33,21 @@ GEM
     treetop (1.4.10)
       polyglot
       polyglot (>= 0.3.1)
+    unicode (0.4.0)
+    wikipedia-client (1.0.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.0.0)
+  character-encodings
   jeweler (~> 1.6.4)
+  json
   mail
   mocha
   ruby-debug (~> 0.10.4)
   sanitize
   shoulda
+  unicode
+  wikipedia-client

data/lib/scylla/generator.rb

@@ -1,16 +1,19 @@
 require 'sanitize'
 require 'cgi'
+require 'wikipedia'
+require 'unicode'
 
 module Scylla
   class Generator
-    attr_accessor :dirtext, :dirlm, :minsize
+    attr_accessor :dirtext, :dirlm, :minsize, :delimiter
 
     # dirtext: The location of the source training text files
     # minsize: The minimum size of the ngrams that you would like to store
-    def initialize(dirtext = DEFAULT_SOURCE_DIR, dirlm = DEFAULT_TARGET_DIR, minsize = 0, silent = false)
+    def initialize(dirtext = DEFAULT_SOURCE_DIR, dirlm = DEFAULT_TARGET_DIR, minsize = 0, silent = false, delimiter = "[[classifier_delimiter]]")
       @dirtext = dirtext
       @dirlm   = dirlm
       @minsize = minsize
+      @delimiter = delimiter
     end
 
     # Loads all the .txt files in the specified source training text folder  
@@ -18,21 +21,47 @@ module Scylla
     # lib/scylla/lms as .lm files
     def train
       languages = Dir.glob(@dirlm + "/*.lm")
-      textpaths = Dir.glob(@dirtext + "/*.txt")
       languages.each {|l| File.delete(l) }
-      textpaths.each do |path|
-        write_lm(path)
+      locales = Scylla::Resources.locales
+      locales.each do |key, value|
+        path = File.join(@dirtext, "#{key}.txt")
+        text = ""
+        File.open(path).each { |line| text += " " + line }
+        write_lm(text, key)
       end
     end
+    
+    def get_wikis
+      locales = Scylla::Resources.locales
+      locales.each do |key, value|
+        text = get_wiki(value[0],value[1])
+        textname = File.join(@dirtext, "#{key}.txt")
+        File.delete(textname) if File.exists?(textname)
+        File.open(textname, 'w') { |f| f.write(text) }
+      end
+    end
+
+    def get_wiki(locale,article)
+      Wikipedia.Configure {
+        domain "#{locale}.wikipedia.org"
+        path   'w/api.php'
+      }
+      p article
+      page = Wikipedia.find( article )
+      value = page.content.gsub(/\{\{(.*?)\}\}/,"")
+      value = value.gsub(/\[\[(.+?)\]\]/m,"")
+      value = value.gsub(/\{\{(.+?)\}\}/m,"")
+      value = value.gsub(/\{(.+?)\}/m,"")
+      value = value.gsub(/\[(.+?)\]/m,"")
+      Sanitize.clean(value)
+    end
 
     # Reads a single text file specified by a path and writes a .lm file in 
     # lib/scylla/lms
-    def write_lm(path)
-      text = ""
-      File.open(path).each { |line| text += " " + line }
-      p "Creating language map for " + path
+    def write_lm(text, language)
+      p "Creating language map for #{language}"
       lm = create_lm(text, true)
-      lmname = File.join(@dirlm, File.basename(path, ".txt") + ".lm")
+      lmname = File.join(@dirlm, "#{language}.lm")
       File.delete(lmname) if File.exists?(lmname)
       File.open(lmname, 'w') do |f|
         i = 0
@@ -45,11 +74,13 @@ module Scylla
     end
 
     def clean(string)
+      delimit = string.index(@delimiter)
+      string = string[0, delimit] if delimit
       string = Sanitize.clean(string)
       string = CGI.unescapeHTML(string)
       string.gsub!(/(?:http|https):\/\/[a-z0-9]+(?:[\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(?:(?::[0-9]{1,5})?\/[^\s]*)?/, "")
-      string.gsub!(/[\*\^><!\"#\$%&\'\(\)\*\+:;=\?@\{\}\[\]|\-\n\r0-9]/," ")
-      string.strip.split(" ").join(" ")
+      string.gsub!(/[\*\^><!\"#\$%&\'\(\)\*\+:;,._\/=\?@\{\}\[\]|\-\n\r0-9]/," ")
+      Unicode::downcase(string.strip.split(" ").join(" "))
     end
 
     # Creates a language map for a given input string. 
@@ -57,12 +88,14 @@ module Scylla
     # return the freqencies of the ngrams, or simply an array in sorted order
     def create_lm(input, frequencies = false)
       input = clean(input)
+      debugger
       ngram = Hash.new
       input.split(/[\d\s\[\]]/).each do |word|
         word = "_" + word + "_";
         len = word.size
         for i in 0..word.size
-          (1..5).each do |j|
+          for j in (1..3)
+            next unless word[i,j]
             ngram[word[i,j]] ||= 0
             ngram[word[i,j]] += 1 if (len > (j - 1))
           end