treat 0.1.4 → 0.2.0

data/lib/treat/lexicalizers.rb

@@ -6,42 +6,53 @@ module Treat
     # Taggers return the part of speech tag of a word.
     module Tag
       extend Group
+      require 'treat/lexicalizers/tag/tagger'
       self.type = :annotator
       self.targets = [:word]
     end
+    
+    # Return the general category of a word.
     module Category
       extend Group
       self.type = :annotator
-      self.targets = [:phrase, :word]
-      
-      def self.cat(entity, category); category; end # Remove
-    end
-    # Linkers allow to retrieve grammatical links
-    # between words.
-    module Linkages
-      extend Group
-      self.type = :annotator
-      self.targets = [:sentence, :word]
+      self.targets = [:word]
+      self.default = :from_tag
     end
+    
     # Lexicons are dictionnaries of semantically linked
     # word forms.
     module Synsets
       extend Group
       self.type = :annotator
-      self.targets = [:word, :number]
-      def self.synonyms(entity, synsets)
-        synsets.collect { |ss| ss.synonyms }.flatten - [entity.value]
-      end
-      def self.antonyms(entity, synsets)
-        synsets.collect { |ss| ss.antonyms }.flatten
-      end    
-      def self.hyponyms(entity, synsets)
-        synsets.collect { |ss| ss.hyponyms }.flatten
-      end
-      def self.hypernyms(entity, synsets)
-        synsets.collect { |ss| ss.hypernyms }.flatten
-      end
+      self.targets = [:word]
+      self.postprocessors = {
+        :synonyms => lambda do |entity, synsets|
+          synsets.collect { |ss| ss.synonyms }.flatten - 
+          [entity.value]
+        end,
+        :antonyms => lambda do |entity, synsets|
+          synsets.collect { |ss| ss.antonyms }.flatten
+        end,
+        :hyponyms => lambda do |entity, synsets|
+          synsets.collect { |ss| ss.hyponyms }.flatten
+        end,
+        :hypernyms => lambda do |entity, synsets|
+          synsets.collect { |ss| ss.hypernyms }.flatten
+        end
+      }
+    end
+    
+    module Linkages
+      extend Group
+      self.type = :annotator
+      self.targets = [:zone]
+      self.presets = {
+        :is_a => {:linkage => :is_a},
+        :synonym_of => {:linkage => :synonym_of},
+        :antonym_of => {:linkage => :antonym_of}
+      }
     end
+    
     extend Treat::Category
   end
 end

data/lib/treat/lexicalizers/category/from_tag.rb

@@ -5,22 +5,29 @@ module Treat
       # using the default tagger for the language of the entity.
       class FromTag
         # Find the category of the current entity.
-        # 
+        #
         # Options:
-        # 
+        #
         # - (Symbol) :tagger => force the use of a tagger.
         def self.category(entity, options = {})
-          tag = options[:tagger].nil? ? entity.tag : entity.tag(options[:tagger])
-          lang = Treat::Languages.get(entity.language)
-          cat = lang::WordTagToCategory[tag]
-          if cat.nil?
-            warn "Category not found for tag #{tag}."
-            :unknown
+          tag = entity.tag(options[:tagger])
+          return :unknown if tag.nil? || tag == ''
+          return :sentence if tag == 'S'
+          if entity.is_a?(Treat::Entities::Phrase)
+            cat = Treat::Languages::Tags::PhraseTagToCategory[tag]
+            unless cat
+              cat = Treat::Languages::Tags::WordTagToCategory[tag]
+            end
+          elsif entity.is_a?(Treat::Entities::Word)
+            cat = Treat::Languages::Tags::WordTagToCategory[tag]
+          end
+          if cat == nil
+            warn "Category not found for tag '#{tag}'."
+            return :unknown
           else
             if cat.size == 1
-              return cat[0]
+              return cat[entity.tag_set]
             else
-              entity.set :tag_set, :penn
               if entity.has?(:tag_set)
                 if cat[entity.tag_set]
                   return cat[entity.tag_set]

data/lib/treat/lexicalizers/linkages/naive.rb

@@ -2,60 +2,60 @@ module Treat
   module Lexicalizers
     module Linkages
       class Naive
+        # Fix - add options for sentences.
         def self.linkages(entity, options = {})
-          linkage = options.delete(:linkage)
-          if linkage.nil?
+          if options[:linkage] == :is_a ||
+            options[:linkage] == :hypernym_of
+            
+            entity.each_word do |w1|
+              hypernyms = []
+              entity.each_word do |w2|
+                next if w1 == w2
+                if w2.hypernyms.include?(w1.value) ||
+                  w1.hyponyms.include?(w2.value)
+                  hypernyms << w1
+                  w2.link(w1, :is_a)
+                  w1.link(w2, :hypernym_of)
+                end
+              end
+              w1.set :hypernyms, hypernyms
+            end
+            
+          elsif options[:linkage] == :synonym_of
+            
+            entity.each_word do |w1|
+              synonyms = []
+              entity.each_word do |w2|
+                next if w1 == w2
+                if w2.synonyms.include?(w1.value)
+                  synonyms << w1
+                  w2.link(w1, :synonym_of)
+                  w1.link(w2, :synonym_of)
+                end
+              end
+              w1.set :synonyms, synonyms
+            end
+  
+          elsif options[:linkage] == :antonym_of
+            
+            entity.each_word do |w1|
+              antonyms = []
+              entity.each_word do |w2|
+                next if w1 == w2
+                if w2.antonyms.include?(w1.value)
+                  antonyms << w1
+                  w2.link(w1, :antonym_of)
+                  w1.link(w2, :antonym_of)
+                end
+              end
+              w1.set :antonyms, antonyms
+            end
+            
+          else
             raise Treat::Exception,
-            "You must supply the :linkage option."
+            "Invalid linkage option '#{options[:linkage]}'."
           end
-          if !respond_to?(linkage)
-            raise Treat::Exception,
-            "No handler to resolve linkage #{linkage}."
-          end
-          self.send(linkage, entity, options)
-        end
-        # %%%
-        def self.patient(entity, options)
-          # Not so simple here...                 Fix
-          if main_verb.has_feature?(:aux)
-            subject
-          elsif main_verb.voice == 'passive'
-            subject
-          elsif main_verb.voice == 'active'
-            # Each prepos.
-          end
-        end
-        # Return the subject of the sentence|verb.
-        def self.subject(entity, options)
-          verb = (entity.has?(:category) && entity.category == :verb) ? 
-          main_verb(entity) : entity.main_verb
-          args = []
-          main_verb.edges.each_pair do |id,edge|
-            args << find(id)
-          end
-          args[0]
-        end
-        # Return the object of the sentence|verb.
-        def self.object(entity, options)
-          verb = (entity.has?(:category) && entity.category == :verb) ? 
-          main_verb(entity) : entity.main_verb
-          if verb.voice == 'passive'
-            return
-          end
-          args = []
-          verb.edges.each_pair do |id,edge|
-            args << find(id)
-          end
-          args[1]
-        end
-        # Find the main verb (shallowest verb in the tree).
-        def self.main_verb(entity, options)
-          verbs = entity.verbs
-          if verbs.empty?
-            return
-          end
-          verbs.sort! { |a,b| a.depth <=> b.depth }
-          verbs[0]
+          
         end
       end
     end

data/lib/treat/lexicalizers/synsets/wordnet.rb

@@ -55,7 +55,11 @@ module Treat
       # The antonym sets of the synset.
       def antonyms; antonym.collect { |a| a.words }; end
       # The hypernym sets of the synset.
-      def hypernyms; hypernym.words; end
+      def hypernyms; 
+        h = hypernym
+        return [] unless h
+        h.words
+      end
       # The hyponym sets of the synset.
       def hyponyms; hyponym.collect { |h| h.words }; end
       # Respond to the missing method event.

data/lib/treat/lexicalizers/tag/brill.rb

@@ -4,47 +4,47 @@ module Treat
       # Adapter class for the 'rbtagger' gem, a port
       # of the Perl Lingua::BrillTagger class, based
       # on the rule-based tagger developped by Eric Brill.
-      # 
+      #
       # The Brill tagger is a simple rule-based part of
       # speech tagger. The main advantages over stochastic
       # taggers is a vast reduction in information required
       # and better portability from one tag set, corpus genre
       # or language to another.
-      # 
-      # Original paper: 
-      # Eric Brill. 1992. A simple rule-based part of speech tagger. 
-      # In Proceedings of the third conference on Applied natural 
-      # language processing (ANLC '92). Association for Computational 
-      # Linguistics, Stroudsburg, PA, USA, 152-155. 
+      #
+      # Original paper:
+      # Eric Brill. 1992. A simple rule-based part of speech tagger.
+      # In Proceedings of the third conference on Applied natural
+      # language processing (ANLC '92). Association for Computational
+      # Linguistics, Stroudsburg, PA, USA, 152-155.
       # DOI=10.3115/974499.974526 http://dx.doi.org/10.3115/974499.974526
-      # Project website: 
+      # Project website:
       # http://rbtagger.rubyforge.org/
-      # Original Perl module site: 
+      # Original Perl module site:
       # http://search.cpan.org/~kwilliams/Lingua-BrillTagger-0.02/lib/Lingua/BrillTagger.pm
-      class Brill
+      class Brill < Tagger
         patch = false
         # Require the 'rbtagger' gem.
+        require 'rbtagger'
         begin
-          silence_warnings { require 'rbtagger' }
-        # This whole mess is required to deal with
-        # the fact that the 'rbtagger' gem defines
-        # a top-level module called 'Word', which
-        # will clash with the top-level class 'Word' 
-        # we define when syntactic sugar is enabled.
+          # This whole mess is required to deal with
+          # the fact that the 'rbtagger' gem defines
+          # a top-level module called 'Word', which
+          # will clash with the top-level class 'Word'
+          # we define when syntactic sugar is enabled.
         rescue TypeError
-          if Treat.edulcorated?
+          if Treat.sweetened?
             patch = true
             # Unset the class Word for the duration
             # of loading the tagger.
             Object.const_unset(:Word); retry
           else
             raise Treat::Exception,
-            'Something went wrong due to a name clash with the "rbtagger" gem.' + 
+            'Something went wrong due to a name clash with the "rbtagger" gem.' +
             'Turn off syntactic sugar to resolve this problem.'
           end
         ensure
           # Reset the class Word if using syntactic sugar.
-          if Treat.edulcorated? && patch
+          if Treat.sweetened? && patch
             Object.const_set(:Word, Treat::Entities::Word)
           end
         end
@@ -55,38 +55,33 @@ module Treat
         # Tag words using a native Brill tagger.
         #
         # Options:
-        # 
+        #
         # :lexicon => String (Lexicon file to use)
         # :lexical_rules => String (Lexical rule file to use)
         # :contextual_rules => String (Contextual rules file to use)
         def self.tag(entity, options = {})
+          r = super(entity, options)
+          return r if r && r != :isolated_word
           # Reinitialize the tagger if the options have changed.
           @@tagger = nil if options != @@options
           # Create the tagger if necessary
           @@tagger ||= ::Brill::Tagger.new(options[:lexicon],
           options[:lexical_rules], options[:contextual_rules])
-          entity.set :tag_set, :penn
-          # Perform tagging.
-          if entity.type == :word
-            # Setup the context of the word
-            l = entity.left
-            r = entity.right
-            l = l.nil? ? '' : l.to_s
-            r = r.nil? ? '' : r.to_s
-            c = "#{l} #{entity.value} #{r}"
-          end
-          res = @@tagger.tag(c)
-          if l == ''
-            unless r == ''
-              entity.next_sibling.set(:tag, res[3][1])
+          words = (r == :isolated_word) ? [entity] : entity.tokens
+          res = @@tagger.tag(words.join(' '))[1..-1]
+          res ||= []
+          res.each do |info|
+            words.each do |word|
+              if word.value == info[0]
+                word.set :tag_set, :penn
+                word.set :tag, info[1]
+                return info[1] if r == :isolated_word
+              end
             end
-            return res[2][1]
-          else
-            unless r == ''
-              entity.next_sibling.set(:tag, res[2][1])
-            end
-            return res[1][1]
           end
+          entity.set :tag_set, :penn
+          return 'P' if entity.is_a?(Treat::Entities::Phrase)
+          return 'S' if entity.is_a?(Treat::Entities::Sentence)
         end
       end
     end

data/lib/treat/lexicalizers/tag/lingua.rb

@@ -15,7 +15,7 @@ module Treat
       # Project website: http://engtagger.rubyforge.org/
       # Original Perl module site:
       # http://cpansearch.perl.org/src/ACOBURN/Lingua-EN-Tagger-0.15/
-      class Lingua
+      class Lingua < Tagger
         # Require the 'engtagger' gem.
         silence_warnings { require 'engtagger' }
         # Hold the tagger.
@@ -24,8 +24,8 @@ module Treat
         @@options = {}
         # Hold the default options.
         DefaultOptions =  {
-          unknown_word_tag: 'FW',
-          relax: false
+          :unknown_word_tag => 'pp',  # Fix unknown word tag
+          :relax => false
         }
         # Tag the word using a probabilistic model taking
         # into account known words found in a lexicon and
@@ -38,24 +38,29 @@ module Treat
         #   particularly words used polysemously.
         # - (String) :unknown_word_tag => Tag for unknown words.
         def self.tag(entity, options = {})
+          options = DefaultOptions.merge(options)
+          r = super(entity, options)
+          return r if r && r != :isolated_word
           # Reinitialize the tagger if the options have changed.
           if options != @@options
             @@options = DefaultOptions.merge(options)
             @@tagger = nil # Reset the tagger
           end
           @@tagger ||= ::EngTagger.new(@@options)
-          entity.set :tag_set, :penn
-          left = entity.left
-          if left.nil? || left.type != :word
-            left_tag = 'pp'
-          else
-            left_tag = left.tag.downcase
-            left_tag = 'pp' if left_tag == ''
+          left_tag = @@tagger.conf[:current_tag] = 'pp'
+          tokens = (r == :isolated_word) ? [entity] : entity.tokens
+          tokens.each do |token|
+            w = @@tagger.clean_word(token.to_s)
+            t = @@tagger.assign_tag(left_tag, w)
+            t = options[:unknown_word_tag] if t.nil? || t == ''
+            @@tagger.conf[:current_tag] = left_tag = t
+            token.set :tag, t.upcase
+            token.set :tag_set, :penn
+            return t.upcase if r == :isolated_word
           end
-          w = @@tagger.clean_word(entity.to_s)
-          t = @@tagger.conf[:current_tag] = 
-          @@tagger.assign_tag(left_tag, w)
-          t.upcase
+          entity.set :tag_set, :penn
+          return 'P' if entity.is_a?(Treat::Entities::Phrase)
+          return 'S' if entity.is_a?(Treat::Entities::Sentence)
         end
       end
     end

data/lib/treat/lexicalizers/tag/stanford.rb

@@ -1,85 +1,76 @@
 module Treat
   module Lexicalizers
     module Tag
-      class Stanford
-        # Require the Ruby-Java bridge.
-        silence_warnings do
-          require 'rjb'
-          jar = "#{Treat.bin}/stanford-tagger*/stanford-postagger*.jar"
-          jars = Dir.glob(jar)
-          if jars.empty? || !File.readable?(jars[0])
-            raise "Could not find stanford tagger JAR file (looking in #{jar})."+
-            " You may need to manually download the JAR files and/or set Treat.bin."
-          end
-          Rjb::load(jars[0], ['-Xms256M', '-Xmx512M'])
-          MaxentTagger = ::Rjb::import('edu.stanford.nlp.tagger.maxent.MaxentTagger')
-          Word = ::Rjb::import('edu.stanford.nlp.ling.Word')
-          List = ::Rjb::import('java.util.ArrayList')
-        end
-        # A list of models to use by language.
-        # Other models are available; see the models/ folder
-        # in the Stanford Tagger distribution files.
-        LanguageToModel = {
-          eng: 'english-left3words-distsim.tagger',
-          ger: 'german-fast.tagger',
-          fra: 'french.tagger',
-          ara: 'arabic-fast.tagger',
-          chi: 'chinese.tagger'
-        }
+      class Stanford < Tagger
+        require 'stanford-core-nlp'
         # Hold one tagger per language.
         @@taggers = {}
-        # Hold the user-set options for each language.
-        @@options = {}
         # Hold the default options.
-        DefaultOptions =  {}
+        DefaultOptions =  {
+          :tagger_model => nil,
+          :silence => false, 
+          :log_to_file => nil
+        }
+        LanguageToTagSet = {
+          :eng => :penn,
+          :ger => :negra,
+          :chi => :penn_chinese,
+          :fre => :simple
+        }
         # Tag the word using one of the Stanford taggers.
         def self.tag(entity, options = {})
+          # Handle options and set models.
+          options = DefaultOptions.merge(options)
+          r = super(entity, options)
+          return r if r && r != :isolated_word
+          # Arrange options.
           lang = entity.language
-          # Find the model.
-          if options[:model]
-            model = options[:model]
-          else
-            model = LanguageToModel[lang]
-            if model.nil?
-              raise Treat::Exception, "There exists no Stanford tagger model for " +
-              "the #{Treat::Languages.describe(lang)} language ."
-            end
+          @@tag_set = LanguageToTagSet[lang]
+          unless @@tag_set
+            warn "The tag set for the tagger you are requiring is not supported."
           end
-          # Reinitialize the tagger if the options have changed.
-          if options != @@options[lang]
-            @@options[lang] = DefaultOptions.merge(options)
-            @@taggers[lang] = nil # Reset the tagger
+          
+          if options[:tagger_model]
+            ::StanfordCoreNLP.set_model(
+              'pos.model', options[:tagger_model]
+            )
           end
-          if @@taggers[lang].nil?
-            model = "#{Treat.bin}/stanford-tagger*/models/#{model}"
-            models = Dir.glob(model)
-            if models.empty? || !File.readable?(models[0])
-              raise "Could not find a tagger model for the " +
-              "#{Treat::Languages.describe(lang)}: looking in #{model}."
-            end
-            silence_streams(STDOUT, STDERR) do
-              @@taggers[lang] =
-              MaxentTagger.new(models[0])
-            end
+          if options[:silence]
+            options[:log_to_file] = '/dev/null'
           end
-          entity.set :tag_set, :penn
-          list = List.new
-          id_list = {}
-          i = 0
-          [entity].each do |word|    # Fix...
-            list.add(Word.new(word.to_s))
-            id_list[i] = word
-            i += 1
+          if options[:log_to_file]
+            ::StanfordCoreNLP.log_file = 
+              options[:log_to_file] 
           end
-          it = nil
-          it = @@taggers[lang].apply(list).iterator
-          i = 0
-          while it.has_next
-            w = it.next
-            id_list[i].set :tag, w.tag
-            i += 1
+          
+          # Load the tagger.
+          StanfordCoreNLP.use(lang)
+          @@taggers[lang] ||= ::StanfordCoreNLP.load(:tokenize, :ssplit, :pos)
+          # Tag the text.
+          text = ::StanfordCoreNLP::Text.new(entity.to_s)
+          @@taggers[lang].annotate(text)
+          # Realign the tags.
+          entity.each_token do |t1|
+            text.get(:sentences).each do |sentence|
+              sentence.get(:tokens).each do |t2|
+                if t2.value == t1.value
+                  tag = t2.get(:part_of_speech).to_s
+                  tag_s, tag_opt = *tag.split('-')
+                  tag_s ||= ''
+                  t1.set :tag, tag_s
+                  t1.set :tag_opt, tag_opt
+                  t1.set :tag_set, @@tag_set if @@tag_set
+                  return tag_s if r == :isolated_word
+                  break
+                end
+              end
+            end
           end
-          w.tag
+          
+          # Handle tags for sentences and phrases.
+          entity.set :tag_set, @@tag_set if @@tag_set
+          return 'P' if entity.is_a?(Treat::Entities::Phrase)
+          return 'S' if entity.is_a?(Treat::Entities::Sentence)
         end
       end
     end