treat 0.1.1

data/lib/treat/inflectors/stemmers/porter_c.rb

@@ -0,0 +1,23 @@
+module Treat
+  module Inflectors
+    module Stemmers
+      # Stems words using the 'ruby-stemmer' gem, which
+      # wraps a C version of the Porter stemming algorithm.
+      # 
+      # Project website: https://github.com/aurelian/ruby-stemmer
+      # Original paper: Porter, 1980. An algorithm for suffix stripping, 
+      # Program, Vol. 14, no. 3, pp 130-137,
+      # Original C implementation: http://www.tartarus.org/~martin/PorterStemmer.
+      class PorterC
+        silently { require 'lingua/stemmer' }
+        ::LinguaStemmer = ::Lingua
+        Object.instance_eval { remove_const :Lingua }
+        # Stem the word using the Porter C algorithm.
+        # Options: none.
+        def self.stem(word, options = {})
+          silently { ::LinguaStemmer.stemmer(word.to_s) }
+        end
+      end
+    end
+  end
+end

data/lib/treat/inflectors/stemmers/uea.rb

@@ -0,0 +1,30 @@
+module Treat
+  module Inflectors
+    module Stemmers
+      # Stems a word using the UEA algorithm, implemented
+      # by the 'uea-stemmer' gem.
+      # 
+      # "Similar to other stemmers, UEA-Lite operates on a
+      # set of rules which are used as steps. There are two 
+      # groups of rules: the first to clean the tokens, and 
+      # the second to alter suffixes."
+      #
+      #   Project website: https://github.com/ealdent/uea-stemmer
+      #   Original paper: Jenkins, Marie-Claire, Smith, Dan, 
+      #   Conservative stemming for search and indexing, 2005.
+      #   http://www.uea.ac.uk/polopoly_fs/1.85493!stemmer25feb.pdf
+      class UEA
+        # Require the 'uea-stemmer' gem.
+        silently { require 'uea-stemmer' }
+        # Keep only one copy of the stemmer.
+        @@stemmer = nil
+        # Stems a word using the UEA algorithm, implemented
+        # by the 'uea-stemmer' gem.        
+        def self.stem(entity, options = {})
+          @@stemmer ||= silently { ::UEAStemmer.new }
+          @@stemmer.stem(entity.to_s).strip
+        end
+      end
+    end
+  end
+end

data/lib/treat/lexicalizers.rb

@@ -0,0 +1,49 @@
+module Treat
+  # Lexicalizers allow the retrieval of lexical information
+  # (part of speech tag, synsets, hypersets, hyposets, etc.)
+  # of an entity.
+  module Lexicalizers
+    # Taggers return the part of speech tag of a word.
+    module Tag
+      extend Group
+      self.type = :annotator
+      self.targets = [:phrase, :word]
+    end
+    module Category
+      extend Group
+      self.type = :annotator
+      self.targets = [:phrase, :word]
+      
+      def self.cat(entity, category); category; end # Remove
+    end
+    # Linkers allow to retrieve grammatical links
+    # between words.
+    module Linkages
+      extend Group
+      self.type = :annotator
+      self.targets = [:sentence, :word]
+    end
+    # Lexicons are dictionnaries of semantically linked
+    # word forms.
+    module Synsets
+      extend Group
+      self.type = :annotator
+      self.targets = [:word, :number]
+      
+      def self.synonyms(entity, synsets)
+        synsets.collect { |ss| ss.synonyms }.flatten - [entity.value]
+      end
+      def self.antonyms(entity, synsets)
+        synsets.collect { |ss| ss.antonyms }.flatten
+      end    
+      def self.hyponyms(entity, synsets)
+        synsets.collect { |ss| ss.hyponyms }.flatten
+      end
+      def self.hypernyms(entity, synsets)
+        synsets.collect { |ss| ss.hypernyms }.flatten
+      end
+      
+    end
+    extend Treat::Category
+  end
+end

data/lib/treat/lexicalizers/category/from_tag.rb

@@ -0,0 +1,30 @@
+module Treat
+  module Lexicalizers
+    module Category
+      # A class that detects the category of a word from its tag,
+      # using the default tagger for the language of the entity.
+      class FromTag
+        # Find the category of the current entity.
+        # Options:
+        # :tagger => (Symbol) force the use of a tagger.
+        # :tag_to_cat => (Hash) a list of categories for each possible tag.
+        def self.category(entity, options = {})
+          if options.empty?
+            options = {
+              tagger: nil,
+              tag_to_cat: Treat::Resources::Tags::PTBWordTagToCategory
+            }
+          end
+          tag = options[:tagger].nil? ? entity.tag : entity.tag(options[:tagger])
+          cat = options[:tag_to_cat][tag]
+          if cat.nil?
+            warn "Category not found for tag #{tag}."
+            :unknown
+          else
+            cat
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/lexicalizers/linkages/naive.rb

@@ -0,0 +1,63 @@
+module Treat
+  module Lexicalizers
+    module Linkages
+      class Naive
+        def self.linkages(entity, options = {})
+          linkage = options.delete(:linkage)
+          if linkage.nil?
+            raise Treat::Exception,
+            "You must supply the :linkage option."
+          end
+          if !respond_to?(linkage)
+            raise Treat::Exception,
+            "No handler to resolve linkage #{linkage}."
+          end
+          self.send(linkage, entity, options)
+        end
+        # %%%
+        def self.patient(entity, options)
+          # Not so simple here...                 Fix
+          if main_verb.has_feature?(:aux)
+            subject
+          elsif main_verb.voice == 'passive'
+            subject
+          elsif main_verb.voice == 'active'
+            # Each prepos.
+          end
+        end
+        # Return the subject of the sentence|verb.
+        def self.subject(entity, options)
+          verb = entity.category == :verb ? 
+          main_verb(entity) : entity.main_verb
+          args = []
+          main_verb.edges.each_pair do |id,edge|
+            args << find(id)
+          end
+          args[0]
+        end
+        # Return the object of the sentence|verb.
+        def self.object(entity, options)
+          verb = entity.category == :verb ? 
+          main_verb(entity) : entity.main_verb
+          if verb.voice == 'passive'
+            return
+          end
+          args = []
+          verb.edges.each_pair do |id,edge|
+            args << find(id)
+          end
+          args[1]
+        end
+        # Find the main verb (shallowest verb in the tree).
+        def self.main_verb(entity, options)
+          verbs = entity.words_with_cat(:verb)
+          if verbs.empty?
+            return
+          end
+          verbs.sort! { |a,b| a.depth <=> b.depth }
+          verbs[0]
+        end
+      end
+    end
+  end
+end

data/lib/treat/lexicalizers/synsets/rita_wn.rb

@@ -0,0 +1,23 @@
+module Treat
+  module Lexicalizers
+    module Synsets
+      # Currently not implemented.
+      class RitaWn
+        # Require the Ruby-Java bridge.
+        #silently do
+          require 'rjb'
+          # Load the RitaWN jars.
+          Rjb::load("#{Treat.bin}/jwnl/jwnl.jar", [])
+          JWNLException = Rjb::import('net.didion.jwnl.JWNLException')
+          Rjb::load("#{Treat.bin}/ritaWN/library/ritaWN.jar", [])
+          Rjb::add_jar("#{Treat.bin}/ritaWN/library/supportWN.jar")
+          Rjb::add_jar("#{Treat.bin}/ritaWNcore1.0.jar")
+          RiWordnet = ::Rjb::import('rita.wordnet.RiWordnet')
+        #end
+        def self.synsets(word, options = nil)
+
+        end
+      end
+    end
+  end
+end

data/lib/treat/lexicalizers/synsets/wordnet.rb

@@ -0,0 +1,72 @@
+module Treat
+  module Lexicalizers
+    module Synsets
+      # Obtain lexical information about a word using the
+      # ruby 'wordnet' gem.
+      class Wordnet
+        # Require the 'wordnet' gem.
+        require 'wordnet'
+        # Obtain lexical information about a word using the
+        # ruby 'wordnet' gem.
+        def self.synsets(word, options = nil)
+          unless [:noun, :adjective, :verb].include?(word.category)
+            return []
+          end
+          cat = word.category.to_s.capitalize
+          index = ::WordNet.const_get(cat + 'Index').instance
+          lemma = index.find(word.value.downcase)
+          return [] if lemma.nil?
+          synsets = []
+          lemma.synsets.each { |synset| synsets << Synset.new(synset) }
+          synsets
+        end
+      end
+    end
+    # An adaptor for synsets used by the Wordnet gem.
+    class Synset
+      # The POS tag of the word.
+      attr_accessor :pos
+      # The definition of the synset.
+      attr_accessor :definition
+      # The examples in the synset.
+      attr_accessor :examples
+      def initialize(synset)
+        @original_synset = synset
+        @pos, @definition, @examples = 
+        parse_synset(synset.to_s.split(')'))
+      end
+      def parse_synset(res) 
+        pos = res[0][1..-1].strip
+        res2 = res[1].split('(')
+        res3 = res2[1].split(';')
+        1.upto(res3.size-1) do |i|
+          res3[i] = res3[i].strip[1..-2]
+        end
+        definition = res3[0]
+        examples = res3[1..-1]
+        return pos, definition, examples
+      end
+      # The words in the synset.
+      def words; @original_synset.words; end
+      def synonyms; @original_synset.words; end
+      # A gloss (short definition with examples)
+      # for the synset.
+      def gloss; @original_synset.gloss; end
+      # The antonym sets of the synset.
+      def antonyms; antonym.collect { |a| a.words }; end
+      # The hypernym sets of the synset.
+      def hypernyms; hypernym.words; end
+      # The hyponym sets of the synset.
+      def hyponyms; hyponym.collect { |h| h.words }; end
+      # Respond to the missing method event.
+      def method_missing(sym, *args, &block)
+        ret = @original_synset.send(sym)
+        if ret.is_a?(::WordNet::Synset)
+          Synset.new(ret)
+        else
+          ret
+        end
+      end
+    end
+  end
+end

data/lib/treat/lexicalizers/tag/brill.rb

@@ -0,0 +1,101 @@
+module Treat
+  module Lexicalizers
+    module Tag
+      # Adapter class for the 'rbtagger' gem, a port
+      # of the Perl Lingua::BrillTagger class, based
+      # on the rule-based tagger developped by Eric Brill.
+      # 
+      # The Brill tagger is a simple rule-based part of
+      # speech tagger. The main advantages over stochastic
+      # taggers is a vast reduction in information required
+      # and better portability from one tag set, corpus genre
+      # or language to another.
+      # 
+      # Original paper: 
+      # Eric Brill. 1992. A simple rule-based part of speech tagger. 
+      # In Proceedings of the third conference on Applied natural 
+      # language processing (ANLC '92). Association for Computational 
+      # Linguistics, Stroudsburg, PA, USA, 152-155. 
+      # DOI=10.3115/974499.974526 http://dx.doi.org/10.3115/974499.974526
+      # Project website: 
+      # http://rbtagger.rubyforge.org/
+      # Original Perl module site: 
+      # http://search.cpan.org/~kwilliams/Lingua-BrillTagger-0.02/lib/Lingua/BrillTagger.pm
+      class Brill
+        patch = false
+        # Require the 'rbtagger' gem.
+        begin
+          silently { require 'rbtagger' }
+        # This whole mess is required to deal with
+        # the fact that the 'rbtagger' gem defines
+        # a top-level module called 'Word', which
+        # will clash with the top-level class 'Word' 
+        # we define when syntactic sugar is enabled.
+        rescue TypeError
+          if Treat.edulcorated?
+            patch = true
+            # Unset the class Word for the duration
+            # of loading the tagger.
+            Object.const_unset(:Word); retry
+          else
+            raise Treat::Exception,
+            'Something went wrong due to a name clash with the "rbtagger" gem.' + 
+            'Turn off syntactic sugar to resolve this problem.'
+          end
+        ensure
+          # Reset the class Word if using syntactic sugar.
+          if Treat.edulcorated? && patch
+            Object.const_set(:Word, Treat::Entities::Word)
+          end
+        end
+        # Hold the tagger.
+        @@tagger = nil
+        # Hold the user-set options
+        @@options = {}
+        # Hold the default options.
+        DefaultOptions =  {
+          lexicon: nil,
+          lexical_rules: nil, 
+          contextual_rules: nil
+        }
+        # Tag words using a native Brill tagger.
+        #
+        # Available options:
+        # :lexicon => String (Lexicon file to use)
+        # :lexical_rules => String (Lexical rule file to use)
+        # :contextual_rules => String (Contextual rules file to use)
+        def self.tag(entity, options = {})
+          # Reinitialize the tagger if the options have changed.
+          if options != @@options
+            @@options = DefaultOptions.merge(options)
+            @@tagger = nil # Reset the tagger
+          end
+          # Create the tagger if necessary
+          @@tagger ||= ::Brill::Tagger.new(options[:lexicon],
+          options[:lexical_rules], options[:contextual_rules])
+          # Perform tagging.
+          if entity.type == :word
+            # Setup the context of the word
+            l = entity.left
+            r = entity.right
+            l = l.nil? ? '' : l.to_s
+            r = r.nil? ? '' : r.to_s
+            c = "#{l} #{entity.value} #{r}"
+          end
+          res = @@tagger.tag(c)
+          if l == ''
+            unless r == ''
+              entity.next_sibling.set(:tag, res[3][1])
+            end
+            return res[2][1]
+          else
+            unless r == ''
+              entity.next_sibling.set(:tag, res[2][1])
+            end
+            return res[1][1]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/lexicalizers/tag/lingua.rb

@@ -0,0 +1,114 @@
+module Treat
+  module Lexicalizers
+    module Tag
+      # An adapter for the 'engtagger' gem, which
+      # is a port of the Perl Lingua::EN::Tagger module.
+      #
+      # "This module uses part-of-speech statistics from 
+      # the Penn Treebank to assign POS tags to English text. 
+      # The tagger applies a bigram (two-word) Hidden Markov 
+      # Model to guess the appropriate POS tag for a word.  
+      # That means that the tagger will try to assign a POS 
+      # tag based on the known POS tags for a given word and 
+      # the POS tag assigned to its predecessor.
+      # 
+      # Project website: http://engtagger.rubyforge.org/
+      # Original Perl module site:
+      # http://cpansearch.perl.org/src/ACOBURN/Lingua-EN-Tagger-0.15/
+      class Lingua
+        # Require the 'engtagger' gem.
+        silently { require 'engtagger' }
+        # Hold the tagger.
+        @@tagger = nil
+        # Hold the user-set options
+        @@options = {}
+        # Hold the default options.
+        DefaultOptions =  {
+          unknown_word_tag: '?',
+          relax: false,
+          debug: false
+        }
+        # Tag the word using a probabilistic model taking
+        # into account known words found in a lexicon and
+        # the tag of the previous word.
+        #
+        # Options:
+        # 
+        #   :relax => (Boolean) Relax the Hidden Markov Model: 
+        #   this may improve accuracy for uncommon words, 
+        #   particularly words used polysemously.
+        #   :debug => (Boolean) Print debug messages.
+        #   :unknown_word_tag => (String) Tag for unknown words.
+        def self.tag(entity, options = {})
+          # Reinitialize the tagger if the options have changed.
+          if options != @@options
+            @@options = DefaultOptions.merge(options)
+            @@tagger = nil # Reset the tagger
+          end
+          @@tagger ||= ::EngTagger.new(@@options)
+          left = entity.left
+          if left.nil? || left.type != :word
+            left_tag = 'pp'
+          else
+            left_tag = left.tag.downcase
+            left_tag = 'pp' if left_tag == ''
+          end
+          w = @@tagger.clean_word(entity.to_s)
+          t = @@tagger.conf[:current_tag] = 
+          @@tagger.assign_tag(left_tag, w)
+          t.upcase
+        end
+      end
+    end
+  end
+end
+
+=begin
+
+CC      Conjunction, coordinating               and, or
+CD      Adjective, cardinal number              3, fifteen
+DET     Determiner                              this, each, some
+EX      Pronoun, existential there              there
+FW      Foreign words
+IN      Preposition / Conjunction               for, of, although, that
+JJ      Adjective                               happy, bad
+JJR     Adjective, comparative                  happier, worse
+JJS     Adjective, superlative                  happiest, worst
+LS      Symbol, list item                       A, A.
+MD      Verb, modal                             can, could, 'll
+NN      Noun                                    aircraft, data
+NNP     Noun, proper                            London, Michael
+NNPS    Noun, proper, plural                    Australians, Methodists
+NNS     Noun, plural                            women, books
+PDT     Determiner, prequalifier                quite, all, half
+POS     Possessive                              's, '
+PRP     Determiner, possessive second           mine, yours
+PRPS    Determiner, possessive                  their, your
+RB      Adverb                                  often, not, very, here
+RBR     Adverb, comparative                     faster
+RBS     Adverb, superlative                     fastest
+RP      Adverb, particle                        up, off, out
+SYM     Symbol                                  *
+TO      Preposition                             to
+UH      Interjection                            oh, yes, mmm
+VB      Verb, infinitive                        take, live
+VBD     Verb, past tense                        took, lived
+VBG     Verb, gerund                            taking, living
+VBN     Verb, past/passive participle           taken, lived
+VBP     Verb, base present form                 take, live
+VBZ     Verb, present 3SG -s form               takes, lives
+WDT     Determiner, question                    which, whatever
+WP      Pronoun, question                       who, whoever
+WPS     Determiner, possessive & question       whose
+WRB     Adverb, question                        when, how, however
+
+PP      Punctuation, sentence ender             ., !, ?
+PPC     Punctuation, comma                      ,
+PPD     Punctuation, dollar sign                $
+PPL     Punctuation, quotation mark left        ``
+PPR     Punctuation, quotation mark right       ''
+PPS     Punctuation, colon, semicolon, elipsis  :, ..., -
+LRB     Punctuation, left bracket               (, {, [
+RRB     Punctuation, right bracket              ), }, ]
+
+=end