treat 0.2.5 → 1.0.0

data/lib/treat/languages/swedish.rb

@@ -1,16 +1,16 @@
-module Treat
-  module Languages
-    class Swedish
-      RequiredDependencies = []
-      OptionalDependencies = []
-      Processors = {
-        :chunkers => [:txt],
-        :segmenters => [:punkt],
-        :tokenizers => [:tactful]
-      }
-      Extractors = {}
-      Inflectors = {}
-      Lexicalizers = {}
-    end
-  end
-end
+class Treat::Languages::Swedish
+  
+  RequiredDependencies = []
+  OptionalDependencies = []
+  
+  Extractors = {}
+  Inflectors = {}
+  Lexicalizers = {}
+  Processors = {
+    :chunkers => [:txt],
+    :segmenters => [:punkt],
+    :tokenizers => [:perl, :tactful]
+  }
+  Retrievers = {}
+  
+end

data/lib/treat/lexicalizers.rb

@@ -1,57 +1,36 @@
-module Treat
-  # Lexicalizers allow the retrieval of lexical information
-  # (part of speech tag, synsets, hypersets, hyposets, etc.)
-  # of an entity.
-  module Lexicalizers
-    # Taggers return the part of speech tag of a word.
-    module Tag
-      extend Group
-      self.type = :annotator
-      self.targets = [:sentence, :phrase, :token]
-    end
-    
-    # Return the general category of a word.
-    module Category
-      extend Group
-      self.type = :annotator
-      self.targets = [:word]
-      self.default = :from_tag
-    end
-    
-    # Lexicons are dictionnaries of semantically linked
-    # word forms.
-    module Synsets
-      extend Group
-      self.type = :annotator
-      self.targets = [:word]
-      self.postprocessors = {
-        :synonyms => lambda do |entity, synsets|
-          synsets.collect { |ss| ss.synonyms }.flatten - 
-          [entity.value]
-        end,
-        :antonyms => lambda do |entity, synsets|
-          synsets.collect { |ss| ss.antonyms }.flatten
-        end,
-        :hyponyms => lambda do |entity, synsets|
-          synsets.collect { |ss| ss.hyponyms }.flatten
-        end,
-        :hypernyms => lambda do |entity, synsets|
-          synsets.collect { |ss| ss.hypernyms }.flatten
-        end
-      }
-    end
-    
-    module Linkages
-      extend Group
-      self.type = :annotator
-      self.targets = [:zone]
-      self.presets = {
-        :is_a => {:linkage => :is_a},
-        :synonym_of => {:linkage => :synonym_of},
-        :antonym_of => {:linkage => :antonym_of}
-      }
-    end
-    
-    extend Treat::Category
+# Lexicalizers allow to retrieve lexical information
+# (part of speech tag, general word category, synsets,
+# synonyms, antonyms, hyponyms, hypernyms, lexical
+# relations, grammatical links).
+# of an entity.
+module Treat::Lexicalizers
+
+  # Taggers return the part of speech tag of a word.
+  module Taggers
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:sentence, :phrase, :token]
   end
+
+  # Return the general category of a word.
+  module Categorizers
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:token]
+    self.default = :from_tag
+  end
+
+  # Find the synsets of a word in a lexicon.
+  module Sensers
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:word]
+    self.preset_option = :nym
+    self.presets = [:synonyms, :antonyms, 
+                    :hyponyms, :hypernyms]
+  end
+  
+  # Make Lexicalizers categorizable.
+  extend Treat::Categorizable
+
 end

data/lib/treat/lexicalizers/categorizers/from_tag.rb

@@ -0,0 +1,54 @@
+# Finds the general part of speech of an entity
+# (:sentence, :noun_phrase, :verb, :adverb, etc.)
+# from its tag (e.g. 'S', 'NP', 'VBZ', 'ADV', etc.).
+class Treat::Lexicalizers::Categorizers::FromTag
+
+  Pttc = Treat::Linguistics::Tags::PhraseTagToCategory
+  Wttc = Treat::Linguistics::Tags::WordTagToCategory
+  Ptc = Treat::Linguistics::Tags::PunctuationToCategory
+  
+  # Find the category of the entity from its tag.
+  def self.category(entity, options = {})
+
+    tag = entity.check_has(:tag)
+    return :unknown if tag.nil? || tag == '' || entity.type == :symbol
+    return :sentence if tag == 'S' || entity.type == :sentence
+    return :number if entity.type == :number
+    return Ptc[entity.to_s] if entity.type == :punctuation
+    
+    if entity.is_a?(Treat::Entities::Phrase)
+      cat = Pttc[tag]
+      cat = Wttc[tag] unless cat
+    else
+      cat = Wttc[tag]
+    end
+
+    return :unknown if cat == nil
+    
+    ts = nil
+    
+    if entity.has?(:tag_set)
+      ts = entity.get(:tag_set)
+    elsif entity.parent_phrase && 
+      entity.parent_phrase.has?(:tag_set)
+      ts = entity.parent_phrase.get(:tag_set)
+    else
+      raise Treat::Exception,
+      "No information can be found regarding "+
+      "which tag set to use."
+    end
+
+    if cat[ts]
+      return cat[ts]
+    else
+      raise Treat::Exception,
+      "The specified tag set (#{ts})" +
+      " does not contain the tag #{tag} " +
+      "for token #{entity.to_s}."
+    end
+
+    :unknown
+
+  end
+
+end

data/lib/treat/lexicalizers/sensers/wordnet.rb

@@ -0,0 +1,57 @@
+# Obtain lexical information about a word using the
+# ruby 'wordnet' gem.
+class Treat::Lexicalizers::Sensers::Wordnet
+
+  # Require the 'wordnet' gem.
+  require 'wordnet'
+  
+  # Patch for bug.
+  ::WordNet.module_eval do
+    remove_const(:SynsetType)
+    const_set(:SynsetType, 
+    {"n" => "noun", "v" => "verb", "a" => "adj"})
+  end
+  
+  # Require an adaptor for Wordnet synsets.
+  require 'treat/lexicalizers/sensers/wordnet/synset'
+  
+  # Noun, adjective and verb indexes.
+  @@indexes = {}
+  
+  # Obtain lexical information about a word using the
+  # ruby 'wordnet' gem.
+  def self.sense(word, options = nil)
+    
+    category = word.check_has(:category)
+    
+    unless options[:nym]
+      raise Treat::Exception, "You must supply " +
+      "the :nym option (:synonym, :hypernym, etc.)"
+    end
+    
+    unless [:noun, :adjective, :verb].
+      include?(word.category)
+      return []
+    end
+    
+    cat = category.to_s.capitalize
+    
+    @@indexes[cat] ||= 
+    ::WordNet.const_get(cat + 'Index').instance
+    lemma = @@indexes[cat].find(word.value.downcase)
+
+    return [] if lemma.nil?
+    synsets = []
+    
+    lemma.synsets.each do |synset|
+      synsets << 
+      Treat::Lexicalizers::Sensers::Wordnet::Synset.new(synset)
+    end
+    
+    ((synsets.collect do |ss|
+      ss.send(options[:nym])
+    end - [word.value]).flatten).uniq
+    
+  end
+
+end

data/lib/treat/lexicalizers/sensers/wordnet/synset.rb

@@ -0,0 +1,71 @@
+# An adaptor for synsets used by the Wordnet gem.
+class Treat::Lexicalizers::Sensers::Wordnet::Synset
+
+  # The POS tag of the word.
+  attr_accessor :pos
+  # The definition of the synset.
+  attr_accessor :definition
+  # The examples in the synset.
+  attr_accessor :examples
+  
+  def initialize(synset)
+    @original_synset = synset
+    @pos, @definition, @examples =
+    parse_synset(synset.to_s.split(')'))
+  end
+  
+  def parse_synset(res)
+    pos = res[0][1..-1].strip
+    res2 = res[1].split('(')
+    res3 = res2[1].split(';')
+    1.upto(res3.size-1) do |i|
+      res3[i] = res3[i].strip[1..-2]
+    end
+    definition = res3[0]
+    examples = res3[1..-1]
+    return pos, definition, examples
+  end
+  
+  # The words in the synset.
+  def words
+    @original_synset.words
+  end
+  
+  def synonyms
+    @original_synset.words
+  end
+  
+  # A gloss (short definition with examples)
+  # for the synset.
+  def gloss
+    @original_synset.gloss
+  end
+  
+  # The antonym sets of the synset.
+  def antonyms
+    antonym.collect { |a| a.words }
+  end
+  
+  # The hypernym sets of the synset.
+  def hypernyms
+    h = hypernym
+    return [] unless h
+    h.words
+  end
+  
+  # The hyponym sets of the synset.
+  def hyponyms
+    hyponym.collect { |h| h.words }
+  end
+  
+  # Respond to the missing method event.
+  def method_missing(sym, *args, &block)
+    ret = @original_synset.send(sym)
+    if ret.is_a?(Treat::Lexicalizers::Sensers::Wordnet::Synset)
+      self.new(ret)
+    else
+      ret
+    end
+  end
+
+end

data/lib/treat/lexicalizers/taggers/brill.rb

@@ -0,0 +1,70 @@
+# Adapter class for the 'rbtagger' gem, a port
+# of the Perl Lingua::BrillTagger class, based
+# on the rule-based tagger developped by Eric Brill.
+#
+# Original paper:
+#
+# Eric Brill. 1992. A simple rule-based part of speech tagger.
+# In Proceedings of the third conference on Applied natural
+# language processing (ANLC '92). Association for Computational
+# Linguistics, Stroudsburg, PA, USA, 152-155.
+# DOI=10.3115/974499.974526 http://dx.doi.org/10.3115/974499.974526
+#
+# Project website:
+#
+# http://rbtagger.rubyforge.org/
+module Treat::Lexicalizers::Taggers::Brill
+
+  require 'rbtagger'
+
+  require 'treat/lexicalizers/taggers/brill/patch'
+
+  # Hold one instance of the tagger.
+  @@tagger = nil
+
+  # Tag words using a native Brill tagger.
+  # Performs own tokenization.
+  #
+  # Options (see the rbtagger gem for more info):
+  #
+  # :lexicon => String (Lexicon file to use)
+  # :lexical_rules => String (Lexical rule file to use)
+  # :contextual_rules => String (Contextual rules file to use)
+  def self.tag(entity, options = {})
+
+    # Tokenize the sentence/phrase.
+    if !entity.has_children? &&
+      !entity.is_a?(Treat::Entities::Token)
+      entity.tokenize(:perl, options)
+    end
+    
+    # Create the tagger if necessary
+    @@tagger ||= ::Brill::Tagger.new(options[:lexicon],
+    options[:lexical_rules], options[:contextual_rules])
+    
+    isolated_token = entity.is_a?(Treat::Entities::Token)
+    tokens = isolated_token ? [entity] : entity.tokens
+    tokens_s = tokens.map { |t| t.value }
+    
+    tags = @@tagger.tag_tokens( tokens_s )
+
+    pairs = tokens.zip(tags)
+
+    pairs.each do |pair|
+      pair[0].set :tag, pair[1]
+      pair[0].set :tag_set, :penn if isolated_token
+      return pair[1] if isolated_token
+    end
+    
+    if entity.is_a?(Treat::Entities::Sentence) ||
+      (entity.is_a?(Treat::Entities::Phrase) && 
+      !entity.parent_sentence)
+        entity.set :tag_set, :penn
+    end
+    
+    return 'S' if entity.is_a?(Treat::Entities::Sentence)
+    return 'P' if entity.is_a?(Treat::Entities::Phrase)
+
+  end
+
+end

data/lib/treat/lexicalizers/taggers/brill/patch.rb

@@ -0,0 +1,61 @@
+patch = false
+
+begin
+  # This whole mess is required to deal with
+  # the fact that the 'rbtagger' gem defines
+  # a top-level module called 'Word', which
+  # will clash with the top-level class 'Word'
+  # we define when syntactic sugar is enabled.
+rescue TypeError
+  if Treat.sweetened?
+    patch = true
+    # Unset the class Word for the duration
+    # of loading the tagger.
+    Object.const_unset(:Word); retry
+  else
+    raise Treat::Exception,
+    'Something went wrong due to a name clash with the "rbtagger" gem.' +
+    'Turn off syntactic sugar to resolve this problem.'
+  end
+ensure
+  # Reset the class Word if using syntactic sugar.
+  if Treat.sweetened? && patch
+    Object.const_set(:Word, Treat::Entities::Word)
+  end
+end
+
+Brill::Tagger.class_eval do 
+
+  def tag_tokens(tokens)
+    
+    tags = Brill::Tagger.tag_start( tokens )
+
+    @tagger.apply_lexical_rules( tokens, tags, [], 0 )
+    @tagger.default_tag_finish( tokens, tags )
+
+    # Brill uses these fake "STAART" tags to delimit the start & end of sentence.
+    tokens << "STAART"
+    tokens << "STAART"
+    tokens.unshift "STAART"
+    tokens.unshift "STAART"
+    tags << "STAART"
+    tags << "STAART"
+    tags.unshift "STAART"
+    tags.unshift "STAART"
+
+    @tagger.apply_contextual_rules( tokens, tags, 1 )
+
+    tags.shift
+    tags.shift
+    tokens.shift
+    tokens.shift
+    tags.pop
+    tags.pop
+    tokens.pop
+    tokens.pop
+
+    tags
+    
+  end
+  
+end

data/lib/treat/lexicalizers/taggers/lingua.rb

@@ -0,0 +1,90 @@
+# An adapter for the 'engtagger' gem, which
+# is a port of the Perl Lingua::EN::Tagger module.
+#
+# "This module uses part-of-speech statistics from
+# the Penn Treebank to assign POS tags to English text.
+# The tagger applies a bigram (two-word) Hidden Markov
+# Model to guess the appropriate POS tag for a word.
+# That means that the tagger will try to assign a POS
+# tag based on the known POS tags for a given word and
+# the POS tag assigned to its predecessor.
+#
+# Project website: http://engtagger.rubyforge.org/
+# Original Perl module site:
+# http://cpansearch.perl.org/src/ACOBURN/Lingua-EN-Tagger-0.15/
+class Treat::Lexicalizers::Taggers::Lingua
+  
+  # Require the 'engtagger' gem.
+  silence_warnings { require 'engtagger' }
+  
+  # Undefine the porter stemming business.
+  String.class_eval { undef :stem }
+  
+  # Hold one instance of the tagger.
+  @@tagger = nil
+  
+  # Hold the default options.
+  DefaultOptions =  { :relax => false }
+  
+  # Replace punctuation tags used by this gem
+  # to the standard PTB tags.
+  Punctuation = {
+    'pp' => '.',
+    'pps' => ';',
+    'ppc' => ',',
+    'ppd' => '$',
+    'ppl' => 'lrb',
+    'ppr' => 'rrb'
+  }
+  
+  # Tag the word using a probabilistic model taking
+  # into account known words found in a lexicon and
+  # the tag of the previous word.
+  #
+  # Options:
+  #
+  # - (Boolean) :relax => Relax the HMM model -
+  #   this may improve accuracy for uncommon words,
+  #   particularly words used polysemously.
+  def self.tag(entity, options = {})
+    
+    if !entity.has_children? && 
+      !entity.is_a?(Treat::Entities::Token)
+      entity.tokenize
+    end
+    
+    options = DefaultOptions.merge(options)
+    
+    @@tagger ||= ::EngTagger.new(options)
+    left_tag = @@tagger.conf[:current_tag] = 'pp'
+    isolated_token = entity.is_a?(Treat::Entities::Token)
+    tokens = isolated_token ? [entity] : entity.tokens
+    
+    tokens.each do |token|
+      next if token.to_s == ''
+      w = @@tagger.clean_word(token.to_s)
+      t = @@tagger.assign_tag(left_tag, w)
+      t = 'fw' if t.nil? || t == ''
+      @@tagger.conf[:current_tag] = left_tag = t
+      t = 'prp$' if t == 'prps'
+      t = 'dt' if t == 'det'
+      t = Punctuation[t] if Punctuation[t]
+      token.set :tag, t.upcase
+      token.set :tag_set, :penn if isolated_token
+      return t.upcase if isolated_token
+      
+    end
+
+    
+    if entity.is_a?(Treat::Entities::Sentence) ||
+      (entity.is_a?(Treat::Entities::Phrase) && 
+      !entity.parent_sentence)
+        entity.set :tag_set, :penn
+    end
+
+    return 'S' if entity.is_a?(Treat::Entities::Sentence)
+    return 'P' if entity.is_a?(Treat::Entities::Phrase)
+    
+  end
+  
+end