treat 0.2.5 → 1.0.0

data/lib/treat/inflectors/ordinalizers/linguistics.rb

@@ -0,0 +1,17 @@
+# This class is a wrapper for the functions included
+# in the 'linguistics' gem that allow to describe a
+# number in words in ordinal form.
+#
+# Project website: http://deveiate.org/projects/Linguistics/
+class Treat::Inflectors::Ordinalizers::Linguistics
+
+  require 'treat/loaders/linguistics'
+  
+  # Desribe a number in words in ordinal form, using the
+  # 'linguistics' gem.
+  def self.ordinal(number, options = {})
+    klass = Treat::Loaders::Linguistics.load(number.language)
+    klass.ordinate(number.to_s)
+  end
+  
+end

data/lib/treat/inflectors/stemmers/porter.rb

@@ -0,0 +1,160 @@
+# Stem a word using a native Ruby implementation of the
+# Porter stemming algorithm, ported to Ruby from a
+# version coded up in Perl. This is a simplified
+# implementation; for a true and fast Porter stemmer,
+# see Treat::Inflectors::Stemmers::PorterC.
+#
+# Authored by Ray Pereda (raypereda@hotmail.com).
+# Unknown license.
+#
+# Original paper: Porter, 1980. An algorithm for suffix stripping,
+# Program, Vol. 14, no. 3, pp 130-137,
+# Original C implementation: http://www.tartarus.org/~martin/PorterStemmer.
+class Treat::Inflectors::Stemmers::Porter
+  
+  # Returns the stem of a word using a native Porter stemmer.
+  #
+  # Options: none.
+  def self.stem(word, options = {})
+    # Copy the word and convert it to a string.
+    w = word.to_s
+    return w if w.length < 3
+    # Map initial y to Y so that the patterns
+    # never treat it as vowel.
+    w[0] = 'Y' if w[0] == ?y
+    # Step 1a
+    if w =~ /(ss|i)es$/
+      w = $` + $1
+    elsif w =~ /([^s])s$/
+      w = $` + $1
+    end
+    # Step 1b
+    if w =~ /eed$/
+      w.chop! if $` =~ MGR0
+    elsif w =~ /(ed|ing)$/
+      stem = $`
+      if stem =~ VOWEL_IN_STEM
+        w = stem
+        case w
+        when /(at|bl|iz)$/             then w << "e"
+        when /([^aeiouylsz])\1$/       then w.chop!
+        when /^#{CC}#{V}[^aeiouwxy]$/o then w << "e"
+        end
+      end
+    end
+    if w =~ /y$/
+      stem = $`
+      w = stem + "i" if stem =~ VOWEL_IN_STEM
+    end
+    # Step 2
+    if w =~ SUFFIX_1_REGEXP
+      stem = $`
+      suffix = $1
+      if stem =~ MGR0
+        w = stem + STEP_2_LIST[suffix]
+      end
+    end
+    # Step 3
+    if w =~
+      /(icate|ative|alize|iciti|ical|ful|ness)$/
+      stem = $`
+      suffix = $1
+      if stem =~ MGR0
+        w = stem + STEP_3_LIST[suffix]
+      end
+    end
+    # Step 4
+    if w =~ SUFFIX_2_REGEXP
+      stem = $`
+      if stem =~ MGR1
+        w = stem
+      end
+    elsif w =~ /(s|t)(ion)$/
+      stem = $` + $1
+      if stem =~ MGR1
+        w = stem
+      end
+    end
+    #  Step 5
+    if w =~ /e$/
+      stem = $`
+      if (stem =~ MGR1) ||
+        (stem =~ MEQ1 && stem !~
+        /^#{CC}#{V}[^aeiouwxy]$/o)
+        w = stem
+      end
+    end
+    if w =~ /ll$/ && w =~ MGR1
+      w.chop!
+    end
+    # and turn initial Y back to y
+    w[0] = 'y' if w[0] == ?Y
+    w
+  end
+
+  STEP_2_LIST = {
+    'ational'=>'ate', 'tional'=>'tion', 'enci'=>'ence', 'anci'=>'ance',
+    'izer'=>'ize', 'bli'=>'ble',
+    'alli'=>'al', 'entli'=>'ent', 'eli'=>'e', 'ousli'=>'ous',
+    'ization'=>'ize', 'ation'=>'ate',
+    'ator'=>'ate', 'alism'=>'al', 'iveness'=>'ive', 'fulness'=>'ful',
+    'ousness'=>'ous', 'anati'=>'al',
+    'iviti'=>'ive', 'binati'=>'ble', 'logi'=>'log'
+  }
+  
+  STEP_3_LIST = {
+    'icate'=>'ic', 'ative'=>'', 'alize'=>'al', 'iciti'=>'ic',
+    'ical'=>'ic', 'ful'=>'', 'ness'=>''
+  }
+  
+  SUFFIX_1_REGEXP = /(
+  ational  |
+  tional   |
+  enci     |
+  anci     |
+  izer     |
+  bli      |
+  alli     |
+  entli    |
+  eli      |
+  ousli    |
+  ization  |
+  ation    |
+  ator     |
+  alism    |
+  iveness  |
+  fulness  |
+  ousness  |
+  anati    |
+  iviti    |
+  binati   |
+  logi)$/x
+  SUFFIX_2_REGEXP = /(
+  al       |
+  ance     |
+  ence     |
+  er       |
+  ic       |
+  able     |
+  ible     |
+  ant      |
+  ement    |
+  ment     |
+  ent      |
+  ou       |
+  ism      |
+  ate      |
+  iti      |
+  ous      |
+  ive      |
+  ize)$/x
+  C = "[^aeiou]"         # consonant
+  V = "[aeiouy]"         # vowel
+  CC = "#{C}(?>[^aeiouy]*)"  # consonant sequence
+  VV = "#{V}(?>[aeiou]*)"    # vowel sequence
+  MGR0 = /^(#{CC})?#{VV}#{CC}/o                # [cc]vvcc... is m>0
+  MEQ1 = /^(#{CC})?#{VV}#{CC}(#{VV})?$/o       # [cc]vvcc[vv] is m=1
+  MGR1 = /^(#{CC})?#{VV}#{CC}#{VV}#{CC}/o      # [cc]vvccvvcc... is m>1
+  VOWEL_IN_STEM   = /^(#{CC})?#{V}/o                      # vowel in stem
+  
+end

data/lib/treat/inflectors/stemmers/porter_c.rb

@@ -0,0 +1,24 @@
+# Stems words using the 'ruby-stemmer' gem, which
+# wraps a C version of the Porter stemming algorithm.
+#
+# Project website: https://github.com/aurelian/ruby-stemmer
+# Original paper: Porter, 1980. An algorithm for suffix stripping,
+# Program, Vol. 14, no. 3, pp 130-137,
+# Original C implementation: http://www.tartarus.org/~martin/PorterStemmer.
+module Treat::Inflectors::Stemmers::PorterC
+  
+  # Require the 'ruby-stemmer' gem.
+  silence_warnings { require 'lingua/stemmer' }
+  
+  # Remove a conflict between this gem and the 'engtagger' gem.
+  ::LinguaStemmer = ::Lingua
+  Object.instance_eval { remove_const :Lingua }
+  
+  # Stem the word using a full-blown Porter stemmer in C.
+  #
+  # Options: none.
+  def self.stem(word, options = {})
+    ::LinguaStemmer.stemmer(word.to_s)
+  end
+  
+end

data/lib/treat/inflectors/stemmers/uea.rb

@@ -0,0 +1,28 @@
+# Stems a word using the UEA algorithm, implemented
+# by the 'uea-stemmer' gem.
+#
+# "Similar to other stemmers, UEA-Lite operates on a
+# set of rules which are used as steps. There are two
+# groups of rules: the first to clean the tokens, and
+# the second to alter suffixes."
+#
+# Project website: https://github.com/ealdent/uea-stemmer
+# Original paper: Jenkins, Marie-Claire, Smith, Dan,
+# Conservative stemming for search and indexing, 2005.
+# http://www.uea.ac.uk/polopoly_fs/1.85493!stemmer25feb.pdf
+class Treat::Inflectors::Stemmers::UEA
+
+  # Require the 'uea-stemmer' gem.
+  silence_warnings { require 'uea-stemmer' }
+
+  # Keep only one copy of the stemmer.
+  @@stemmer = nil
+
+  # Stems a word using the UEA algorithm, implemented
+  # by the 'uea-stemmer' gem.
+  def self.stem(entity, options = {})
+    @@stemmer ||= ::UEAStemmer.new
+    @@stemmer.stem(entity.to_s).strip
+  end
+  
+end

data/lib/treat/installer.rb

@@ -0,0 +1,308 @@
+# Installer is a dependency manager for languages.
+#
+# It can be called by using Treat.install(language).
+module Treat::Installer
+
+  # Require the Rubygem dependency installer.
+  silence_warnings do
+    require 'rubygems/dependency_installer'
+  end
+  require 'treat/downloader'
+  require 'treat/dependencies'
+
+  # Package managers for each platforms.
+  PackageManagers = {
+    :mac => 'port',
+    :linux => 'apt-get',
+    :windows => 'win-get'
+  }
+
+  # Address of the server with the files.
+  Server = 'www.louismullie.com'
+
+  # Filenames for the Stanford packages.
+  StanfordPackages = {
+    :minimal => "stanford-core-nlp-minimal.zip",
+    :english => "stanford-core-nlp-english.zip",
+    :all => "stanford-core-nlp-all.zip"
+  }
+
+  # Absolute paths required for cp and mkdir.
+  Paths = {
+    :tmp => File.absolute_path(Treat.tmp),
+    :bin => File.absolute_path(Treat.bin),
+    :models => File.absolute_path(Treat.models)
+  }
+  
+  # Install required dependencies and optional
+  # dependencies for a specific language.
+  def self.install(language = :english)
+    
+    @@installer = Gem::DependencyInstaller.new
+    
+    if language == :travis
+      install_travis; return
+    end
+    
+    lang_class = Treat::Languages.get(language.to_s)
+    l = "#{language.to_s.capitalize} language"
+
+    puts
+    puts "Treat Installer, v. #{Treat::VERSION.to_s}\n"
+    puts
+    
+    begin
+
+      title "Install language-independent gem dependencies."
+
+      case prompt(
+        "1 - Install all default language-independent dependencies\n" +
+        "2 - Select dependencies to install manually\n" +
+        "3 - Skip this step", ['1', '2', '3'])
+      when '1' then install_dependencies(false)
+      when '2' then install_dependencies(true)
+      when '3' then puts 'Skipping this step.'
+      end
+
+      title "Install gem dependencies for the #{l}.\n"
+
+      dflt = lang_class::RequiredDependencies
+      all = dflt + lang_class::OptionalDependencies
+      case prompt("1 - Install default dependencies.\n" +
+        "2 - Select dependencies to install manually.\n" +
+        "3 - Skip this step.", ['1', '2', '3'])
+      when '1' then install_language_dependencies(dflt, false)
+      when '2' then install_language_dependencies(all, true)
+      when '3' then puts 'Skipping this step.'
+      end
+      
+      Treat::Downloader.show_progress = true
+      
+      # If gem is installed only, download models.
+      begin
+        Gem::Specification.find_by_name('punkt-segmenter')
+        title "Downloading model for the Punkt segmenter for the #{l}."
+        download_punkt_models(language)
+      rescue Gem::LoadError; end
+
+      # If stanford is installed, download models.
+      begin
+        Gem::Specification.find_by_name('stanford-core-nlp')
+        title "Download Stanford Core NLP JARs and " +
+        "model files for the the #{l}.\n\n"
+        package = (language == :english) ? :english : :all
+        download_stanford(package)
+      rescue Gem::LoadError; end
+
+      title "Install external binary libraries " +
+            "(requires port, apt-get or win-get).\n"
+      puts "Warning: this may take a long amount of time."
+
+      case prompt("1 - Select binaries to install manually.\n" +
+        "2 - Skip this step.", ['1', '2'])
+      when '1' then install_binaries
+      when '2' then puts 'Skipping this step.'
+      end
+
+      puts
+      puts "-----\nDone!"
+
+    rescue Errno::EACCES => e
+
+      raise Treat::Exception,
+      "Couldn't write to file - permission denied (#{e.message}). " +
+      "You may need to run Ruby or Rake on sudo."
+
+    end
+
+  end
+  
+  # Automated install for Travis CI.
+  def self.install_travis
+    dep = (Treat::Languages::English::RequiredDependencies + 
+          Treat::Languages::English::OptionalDependencies)
+    install_dependencies(false)
+    install_language_dependencies(dep, false)
+    download_stanford(:minimal)
+    download_punkt_models(:english)
+  end
+  
+  def self.install_dependencies(optionally)
+
+    Treat::Dependencies::Gem.each do |d|
+      dep, ver, pur = *d
+      install_gem(dep, ver, pur, optionally)
+    end
+
+  end
+
+  def self.install_language_dependencies(dependencies, optionally)
+
+    puts "No dependencies to install.\n" if dependencies.empty?
+    dependencies.each do |dependency|
+      install_gem(dependency, nil, nil, optionally)
+    end
+
+  end
+
+  def self.install_binaries
+
+    puts "Warning: this will require authentification."
+
+    p = detect_platform
+    man = PackageManagers[p]
+
+    if !man
+      puts "Cannot find a download manager "+
+      "for the #{p} platform.\n\n"
+    else
+      unless `hash #{man} 2>&1` == ''
+        puts "The '#{man}' command is required "+
+        "to install binaries on #{p}.\n\n"
+        man = nil
+      end
+    end
+
+    unless man
+      puts "Skipping installation of the "+
+      "following binaries:\n\n"
+      Binaries.each do |binary, purpose|
+        puts "- #{binary} to #{purpose}"
+      end
+      return
+    end
+
+    Treat::Dependencies::Binary.each do |binary, purpose|
+      if prompt("install #{binary} to " +
+        "#{purpose} (y/n)", ['y', 'n']) == 'y'
+        `sudo #{man} install #{binary}`
+      end
+    end
+
+  end
+
+  def self.download_stanford(package = :minimal)
+    
+    f = StanfordPackages[package]
+    loc = Treat::Downloader.download(
+    'http', Server, 'treat', f, Treat.tmp)
+    puts "- Unzipping package ..."
+    dest = File.join(Treat.tmp, 'stanford')
+    unzip_stanford(loc, dest)
+    
+    model_dir = File.join(Paths[:models], 'stanford')
+    bin_dir = File.join(Paths[:bin], 'stanford')
+    origin = File.join(Paths[:tmp], 'stanford')
+    
+    # Mac hidden files fix.
+    mac_remove = File.join(dest, '__MACOSX')
+    if File.readable?(mac_remove)
+      FileUtils.rm_rf(mac_remove)
+    end
+    
+    unless File.readable?(bin_dir)
+      puts "- Creating directory bin/stanford ..."
+      FileUtils.mkdir_p(bin_dir)
+    end
+    
+    unless File.readable?(model_dir)
+      puts "- Creating directory models/stanford ..."
+      FileUtils.mkdir_p(model_dir)
+    end
+
+    puts "- Copying JAR files to bin/stanford " +
+         "and model files to models/stanford ..."
+    Dir.glob(File.join(origin, '*')) do |f|
+      next if ['.', '..'].include?(f)
+      if f.index('jar')
+        FileUtils.cp(f, File.join(Paths[:bin], 
+        'stanford', File.basename(f)))
+      elsif FileTest.directory?(f)
+        FileUtils.cp_r(f, model_dir)
+      end
+    end
+    
+    puts "- Cleaning up..."
+    FileUtils.rm_rf(origin)
+    
+  end
+
+  def self.download_punkt_models(language)
+
+    f = "#{language}.yaml"
+    dest = "#{Treat.models}punkt/"
+    
+    loc = Treat::Downloader.download(
+    'http', Server, 'treat/punkt', f, Treat.tmp)
+
+    unless File.readable?(dest)
+      puts "- Creating directory models/punkt ..."
+      FileUtils.mkdir_p(File.absolute_path(dest))
+    end
+
+    puts "- Copying model file to models/punkt ..."
+    FileUtils.cp(loc, File.join(Paths[:models], 'punkt', f))
+    
+    puts "- Cleaning up..."
+    FileUtils.rm_rf(Paths[:tmp] + Server)
+
+  end
+
+  private
+
+  @@n = 1
+
+  # Print out a numbered title.
+  def self.title(string)
+    puts
+    puts "#{@@n}. #{string}"
+    puts
+    @@n += 1
+  end
+
+  # Install a dependency with a supplied purpose
+  # but ask the user if she wishes to do so first.
+  def self.install_gem(dependency, version = nil, 
+                       purpose = nil, optionally = false)
+
+    install = false
+
+    begin
+      purpose = purpose ? " to #{purpose}" : ''
+      if optionally
+        if prompt("install #{dependency}#{purpose}", 
+          ['y', 'n']) == 'y'
+          install = true
+        end
+      else
+        puts "\n- Installing #{dependency}#{purpose}."
+        install = true
+      end
+      silence_warnings do 
+        @@installer.install(dependency, version)
+      end if install
+    rescue Exception => error
+      raise
+      puts "Couldn't install gem '#{dependency}' " +
+           "(#{error.message})."
+    end
+    
+  end
+
+  # Unzip a file to the destination path.
+  def self.unzip_stanford(file, destination)
+
+    require 'zip/zip'
+    f_path = ''
+    
+    Zip::ZipFile.open(file) do |zip_file|
+      zip_file.each do |f|
+        f_path = File.join(destination, f.name)
+        FileUtils.mkdir_p(File.absolute_path(File.dirname(f_path)))
+        zip_file.extract(f, f_path) unless File.exist?(f_path)
+      end
+    end
+
+  end
+
+end