treat 0.2.5 → 1.0.0

data/lib/treat/lexicalizers/category/from_tag.rb

@@ -1,49 +0,0 @@
-module Treat
-  module Lexicalizers
-    module Category
-      # A class that detects the category of a word from its tag,
-      # using the default tagger for the language of the entity.
-      class FromTag
-        # Find the category of the current entity.
-        #
-        # Options:
-        #
-        # - (Symbol) :tagger => force the use of a tagger.
-        def self.category(entity, options = {})
-          tag = entity.tag(options[:tagger])
-          return :unknown if tag.nil? || tag == ''
-          return :sentence if tag == 'S'
-          if entity.is_a?(Treat::Entities::Phrase)
-            cat = Treat::Languages::Tags::PhraseTagToCategory[tag]
-            unless cat
-              cat = Treat::Languages::Tags::WordTagToCategory[tag]
-            end
-          elsif entity.is_a?(Treat::Entities::Word)
-            cat = Treat::Languages::Tags::WordTagToCategory[tag]
-          end
-          if cat == nil
-            warn "Category not found for tag '#{tag}'."
-            return :unknown
-          else
-            if cat.size == 1
-              return cat[entity.tag_set]
-            else
-              if entity.has?(:tag_set)
-                if cat[entity.tag_set]
-                  return cat[entity.tag_set]
-                else
-                  raise Treat::Exception,
-                  "The specified tag set (#{entity.tag_set})" +
-                  " does not contain the tag #{tag}."
-                end
-              else
-                raise Treat::Exception,
-                "No information can be found regarding which tag set to use."
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/treat/lexicalizers/linkages/naive.rb

@@ -1,63 +0,0 @@
-module Treat
-  module Lexicalizers
-    module Linkages
-      class Naive
-        # Fix - add options for sentences.
-        def self.linkages(entity, options = {})
-          if options[:linkage] == :is_a ||
-            options[:linkage] == :hypernym_of
-            
-            entity.each_word do |w1|
-              hypernyms = []
-              entity.each_word do |w2|
-                next if w1 == w2
-                if w2.hypernyms.include?(w1.value) ||
-                  w1.hyponyms.include?(w2.value)
-                  hypernyms << w1
-                  w2.link(w1, :is_a)
-                  w1.link(w2, :hypernym_of)
-                end
-              end
-              w1.set :hypernyms, hypernyms
-            end
-            
-          elsif options[:linkage] == :synonym_of
-            
-            entity.each_word do |w1|
-              synonyms = []
-              entity.each_word do |w2|
-                next if w1 == w2
-                if w2.synonyms.include?(w1.value)
-                  synonyms << w1
-                  w2.link(w1, :synonym_of)
-                  w1.link(w2, :synonym_of)
-                end
-              end
-              w1.set :synonyms, synonyms
-            end
-  
-          elsif options[:linkage] == :antonym_of
-            
-            entity.each_word do |w1|
-              antonyms = []
-              entity.each_word do |w2|
-                next if w1 == w2
-                if w2.antonyms.include?(w1.value)
-                  antonyms << w1
-                  w2.link(w1, :antonym_of)
-                  w1.link(w2, :antonym_of)
-                end
-              end
-              w1.set :antonyms, antonyms
-            end
-            
-          else
-            raise Treat::Exception,
-            "Invalid linkage option '#{options[:linkage]}'."
-          end
-          
-        end
-      end
-    end
-  end
-end

data/lib/treat/lexicalizers/synsets/wordnet.rb

@@ -1,76 +0,0 @@
-module Treat
-  module Lexicalizers
-    module Synsets
-      # Obtain lexical information about a word using the
-      # ruby 'wordnet' gem.
-      class Wordnet
-        # Require the 'wordnet' gem.
-        require 'wordnet'
-        # Obtain lexical information about a word using the
-        # ruby 'wordnet' gem.
-        def self.synsets(word, options = nil)
-          unless [:noun, :adjective, :verb].include?(word.category)
-            return []
-          end
-          cat = word.category.to_s.capitalize
-          index = ::WordNet.const_get(cat + 'Index').instance
-          lemma = index.find(word.value.downcase)
-          return [] if lemma.nil?
-          synsets = []
-          lemma.synsets.each { |synset| synsets << Synset.new(synset) }
-          synsets
-        end
-      end
-    end
-    # An adaptor for synsets used by the Wordnet gem.
-    class Synset
-      # The POS tag of the word.
-      attr_accessor :pos
-      # The definition of the synset.
-      attr_accessor :definition
-      # The examples in the synset.
-      attr_accessor :examples
-      def initialize(synset)
-        @original_synset = synset
-        @pos, @definition, @examples = 
-        parse_synset(synset.to_s.split(')'))
-      end
-      def parse_synset(res) 
-        pos = res[0][1..-1].strip
-        res2 = res[1].split('(')
-        res3 = res2[1].split(';')
-        1.upto(res3.size-1) do |i|
-          res3[i] = res3[i].strip[1..-2]
-        end
-        definition = res3[0]
-        examples = res3[1..-1]
-        return pos, definition, examples
-      end
-      # The words in the synset.
-      def words; @original_synset.words; end
-      def synonyms; @original_synset.words; end
-      # A gloss (short definition with examples)
-      # for the synset.
-      def gloss; @original_synset.gloss; end
-      # The antonym sets of the synset.
-      def antonyms; antonym.collect { |a| a.words }; end
-      # The hypernym sets of the synset.
-      def hypernyms; 
-        h = hypernym
-        return [] unless h
-        h.words
-      end
-      # The hyponym sets of the synset.
-      def hyponyms; hyponym.collect { |h| h.words }; end
-      # Respond to the missing method event.
-      def method_missing(sym, *args, &block)
-        ret = @original_synset.send(sym)
-        if ret.is_a?(::WordNet::Synset)
-          Synset.new(ret)
-        else
-          ret
-        end
-      end
-    end
-  end
-end

data/lib/treat/lexicalizers/tag/brill.rb

@@ -1,91 +0,0 @@
-module Treat
-  module Lexicalizers
-    module Tag
-      # Adapter class for the 'rbtagger' gem, a port
-      # of the Perl Lingua::BrillTagger class, based
-      # on the rule-based tagger developped by Eric Brill.
-      #
-      # The Brill tagger is a simple rule-based part of
-      # speech tagger. The main advantages over stochastic
-      # taggers is a vast reduction in information required
-      # and better portability from one tag set, corpus genre
-      # or language to another.
-      #
-      # Original paper:
-      # Eric Brill. 1992. A simple rule-based part of speech tagger.
-      # In Proceedings of the third conference on Applied natural
-      # language processing (ANLC '92). Association for Computational
-      # Linguistics, Stroudsburg, PA, USA, 152-155.
-      # DOI=10.3115/974499.974526 http://dx.doi.org/10.3115/974499.974526
-      # Project website:
-      # http://rbtagger.rubyforge.org/
-      # Original Perl module site:
-      # http://search.cpan.org/~kwilliams/Lingua-BrillTagger-0.02/lib/Lingua/BrillTagger.pm
-      class Brill
-        patch = false
-        # Require the 'rbtagger' gem.
-        require 'rbtagger'
-        begin
-          # This whole mess is required to deal with
-          # the fact that the 'rbtagger' gem defines
-          # a top-level module called 'Word', which
-          # will clash with the top-level class 'Word'
-          # we define when syntactic sugar is enabled.
-        rescue TypeError
-          if Treat.sweetened?
-            patch = true
-            # Unset the class Word for the duration
-            # of loading the tagger.
-            Object.const_unset(:Word); retry
-          else
-            raise Treat::Exception,
-            'Something went wrong due to a name clash with the "rbtagger" gem.' +
-            'Turn off syntactic sugar to resolve this problem.'
-          end
-        ensure
-          # Reset the class Word if using syntactic sugar.
-          if Treat.sweetened? && patch
-            Object.const_set(:Word, Treat::Entities::Word)
-          end
-        end
-        # Hold the tagger.
-        @@tagger = nil
-        # Tag words using a native Brill tagger.
-        # Performs own tokenization.
-        #
-        # Options:
-        #
-        # :lexicon => String (Lexicon file to use)
-        # :lexical_rules => String (Lexical rule file to use)
-        # :contextual_rules => String (Contextual rules file to use)
-        def self.tag(entity, options = {})
-          if entity.has_children?
-            warn "The Brill tagger performs its own tokenization. " +
-                 "Removing all children of #{entity.type} with value #{entity.short_value}."
-            entity.remove_all!
-          end
-          # Create the tagger if necessary
-          @@tagger ||= ::Brill::Tagger.new(options[:lexicon],
-          options[:lexical_rules], options[:contextual_rules])
-          res = @@tagger.tag(entity.to_s)
-          res ||= []
-          isolated_word = entity.is_a?(Treat::Entities::Token)
-          res.each do |info|
-            next if info[1] == ')'
-            token = Treat::Entities::Token.from_string(info[0])
-            token.set :tag_set, :penn
-            token.set :tag, info[1]
-            if isolated_word
-              entity.set :tag_set, :penn
-              return info[1]
-            end
-            entity << token
-          end
-          entity.set :tag_set, :penn
-          return 'P' if entity.is_a?(Treat::Entities::Phrase)
-          return 'S' if entity.is_a?(Treat::Entities::Sentence)
-        end
-      end
-    end
-  end
-end

data/lib/treat/lexicalizers/tag/lingua.rb

@@ -1,123 +0,0 @@
-module Treat
-  module Lexicalizers
-    module Tag
-      # An adapter for the 'engtagger' gem, which
-      # is a port of the Perl Lingua::EN::Tagger module.
-      #
-      # "This module uses part-of-speech statistics from 
-      # the Penn Treebank to assign POS tags to English text. 
-      # The tagger applies a bigram (two-word) Hidden Markov 
-      # Model to guess the appropriate POS tag for a word.  
-      # That means that the tagger will try to assign a POS 
-      # tag based on the known POS tags for a given word and 
-      # the POS tag assigned to its predecessor.
-      # 
-      # Project website: http://engtagger.rubyforge.org/
-      # Original Perl module site:
-      # http://cpansearch.perl.org/src/ACOBURN/Lingua-EN-Tagger-0.15/
-      class Lingua
-        # Require the 'engtagger' gem.
-        silence_warnings { require 'engtagger' }
-        # Hold the tagger.
-        @@tagger = nil
-        # Hold the user-set options
-        @@options = {}
-        # Hold the default options.
-        DefaultOptions =  {
-          :unknown_word_tag => 'pp',  # Fix unknown word tag
-          :relax => false
-        }
-        # Tag the word using a probabilistic model taking
-        # into account known words found in a lexicon and
-        # the tag of the previous word.
-        #
-        # Options:
-        # 
-        # - (Boolean) :relax => Relax the Hidden Markov Model: 
-        #   this may improve accuracy for uncommon words, 
-        #   particularly words used polysemously.
-        # - (String) :unknown_word_tag => Tag for unknown words.
-        def self.tag(entity, options = {})
-          if !entity.has_children?
-            warn "The Lingua tagger requires prior tokenization."
-            warn "Tokenizing the entity #{entity.short_value}."
-          end
-          options = DefaultOptions.merge(options)
-          # Reinitialize the tagger if the options have changed.
-          if options != @@options
-            @@options = DefaultOptions.merge(options)
-            @@tagger = nil # Reset the tagger
-          end
-          @@tagger ||= ::EngTagger.new(@@options)
-          left_tag = @@tagger.conf[:current_tag] = 'pp'
-          isolated_word = entity.is_a?(Treat::Entities::Token)
-          entity.tokens.each do |token|
-            w = @@tagger.clean_word(token.to_s)
-            t = @@tagger.assign_tag(left_tag, w)
-            t = options[:unknown_word_tag] if t.nil? || t == ''
-            @@tagger.conf[:current_tag] = left_tag = t
-            token.set :tag, t.upcase
-            token.set :tag_set, :penn
-            if isolated_word
-              entity.set :tag_set, :penn
-              return t.upcase
-            end
-          end
-          entity.set :tag_set, :penn
-          return 'P' if entity.is_a?(Treat::Entities::Phrase)
-          return 'S' if entity.is_a?(Treat::Entities::Sentence)
-        end
-      end
-    end
-  end
-end
-
-=begin
-
-CC      Conjunction, coordinating               and, or
-CD      Adjective, cardinal number              3, fifteen
-DET     Determiner                              this, each, some
-EX      Pronoun, existential there              there
-FW      Foreign words
-IN      Preposition / Conjunction               for, of, although, that
-JJ      Adjective                               happy, bad
-JJR     Adjective, comparative                  happier, worse
-JJS     Adjective, superlative                  happiest, worst
-LS      Symbol, list item                       A, A.
-MD      Verb, modal                             can, could, 'll
-NN      Noun                                    aircraft, data
-NNP     Noun, proper                            London, Michael
-NNPS    Noun, proper, plural                    Australians, Methodists
-NNS     Noun, plural                            women, books
-PDT     Determiner, prequalifier                quite, all, half
-POS     Possessive                              's, '
-PRP     Determiner, possessive second           mine, yours
-PRPS    Determiner, possessive                  their, your
-RB      Adverb                                  often, not, very, here
-RBR     Adverb, comparative                     faster
-RBS     Adverb, superlative                     fastest
-RP      Adverb, particle                        up, off, out
-SYM     Symbol                                  *
-TO      Preposition                             to
-UH      Interjection                            oh, yes, mmm
-VB      Verb, infinitive                        take, live
-VBD     Verb, past tense                        took, lived
-VBG     Verb, gerund                            taking, living
-VBN     Verb, past/passive participle           taken, lived
-VBP     Verb, base present form                 take, live
-VBZ     Verb, present 3SG -s form               takes, lives
-WDT     Determiner, question                    which, whatever
-WP      Pronoun, question                       who, whoever
-WPS     Determiner, possessive & question       whose
-WRB     Adverb, question                        when, how, however
-
-PP      Punctuation, sentence ender             ., !, ?
-PPC     Punctuation, comma                      ,
-PPD     Punctuation, dollar sign                $
-PPL     Punctuation, quotation mark left        ``
-PPR     Punctuation, quotation mark right       ''
-PPS     Punctuation, colon, semicolon, elipsis  :, ..., -
-LRB     Punctuation, left bracket               (, {, [
-RRB     Punctuation, right bracket              ), }, ]
-
-=end

data/lib/treat/lexicalizers/tag/stanford.rb

@@ -1,70 +0,0 @@
-module Treat
-  module Lexicalizers
-    module Tag
-      class Stanford
-        require 'stanford-core-nlp'
-        # Hold one tagger per language.
-        @@taggers = {}
-        # Hold the default options.
-        DefaultOptions =  {
-          :tagger_model => nil,
-          :silence => false, 
-          :log_to_file => nil
-        }
-        LanguageToTagSet = {
-          :eng => :penn,
-          :ger => :negra,
-          :chi => :penn_chinese,
-          :fre => :simple
-        }
-        # Tag the word using one of the Stanford taggers.
-        def self.tag(entity, options = {})
-          # Handle options and set models.
-          options = DefaultOptions.merge(options)
-          if entity.has_children?
-            warn "The Stanford tagger performs its own tokenization." +
-                 "Removing all children of #{entity.type} with value #{entity.short_value}."
-            entity.remove_all!
-          end
-          # Arrange options.
-          lang = entity.language
-          tag_set = LanguageToTagSet[lang]
-          warn "The tag set for the Stanford tagger you are requiring is not supported." unless tag_set
-          ::StanfordCoreNLP.set_model('pos.model', options[:tagger_model]) if options[:tagger_model]
-          options[:log_to_file] = '/dev/null' if options[:silence]
-          ::StanfordCoreNLP.log_file = options[:log_to_file]  if options[:log_to_file]
-          
-          # Load the tagger.
-          StanfordCoreNLP.use(lang)
-          @@taggers[lang] ||= ::StanfordCoreNLP.load(:tokenize, :ssplit, :pos)
-          
-          # Tag the text.
-          text = ::StanfordCoreNLP::Text.new(entity.to_s)
-          isolated_word = entity.is_a?(Treat::Entities::Token)
-          @@taggers[lang].annotate(text)
-          
-          text.get(:tokens).each do |token|
-            val = token.get(:value).to_s
-            tok = Treat::Entities::Token.from_string(val)
-            tag = token.get(:part_of_speech).to_s
-            tag_s, tag_opt = *tag.split('-')
-            tag_s ||= ''
-            tok.set :tag, tag_s
-            tok.set :tag_opt, tag_opt
-            tok.set :tag_set, tag_set if tag_set
-            if isolated_word
-              entity.set :tag_set, :penn
-              return tag_s
-            end
-            entity << tok
-          end
-          
-          # Handle tags for sentences and phrases.
-          entity.set :tag_set, tag_set if tag_set
-          return 'P' if entity.is_a?(Treat::Entities::Phrase)
-          return 'S' if entity.is_a?(Treat::Entities::Sentence)
-        end
-      end
-    end
-  end
-end