treat 0.2.5 → 1.0.0

data/lib/treat/processors/parsers/stanford.rb

@@ -1,137 +1,136 @@
-module Treat
-  module Processors
-    module Parsers
-      # A wrapper class for the Stanford parser.
-      class Stanford
-        require 'stanford-core-nlp'
-        @@parser = {}
-        DefaultOptions = {
-          :silence => false, 
-          :log_to_file => nil, 
-          :parser_model => nil,
-          :tagger_model => nil
-        }
-        # Parse the entity using the Stanford parser.
-        #
-        # Options:
-        # - (String) :log_to_file => a filename to log output to
-        # instead of displaying it.
-        def self.parse(entity, options = {})
-          val = entity.to_s
-          entity.remove_all! if entity.has_children?
-          options = DefaultOptions.merge(options)
-          lang = entity.language
-          StanfordCoreNLP.use(lang)
-          if options[:tagger_model]
-            ::StanfordCoreNLP.set_model(
-              'pos.model', options[:tagger_model]
-            )
-          end
-          if options[:parser_model]
-            ::StanfordCoreNLP.set_model(
-              'parser.model', options[:parser_model]
-            )
-          end
-          if options[:silence]
-            options[:log_to_file] = '/dev/null'
-          end
-          if options[:log_to_file]
-            ::StanfordCoreNLP.log_file = 
-              options[:log_to_file] 
-          end
-          @@parser[lang] ||= 
-            ::StanfordCoreNLP.load(
-              :tokenize, :ssplit, :pos, :lemma, :parse
-            )
-          text = ::StanfordCoreNLP::Text.new(val)
-          @@parser[lang].annotate(text)
-          
-          text.get(:sentences).each do |s|
-            if entity.is_a?(Treat::Entities::Sentence) ||
-              entity.is_a?(Treat::Entities::Phrase)
-              tag = s.get(:category).to_s
-              tag_s, tag_opt = *tag.split('-')
-              tag_s ||= 'S'
-              entity.set :tag_set, :penn
-              entity.set :tag, tag_s
-              entity.set :tag_opt, tag_opt if tag_opt
-              recurse(s.get(:tree), entity)
-              break
-            else
-              recurse(s.get(:tree), entity)
-            end
-          end
+# A wrapper class for the Stanford parser.
+class Treat::Processors::Parsers::Stanford
+
+  require 'treat/loaders/stanford'
+
+  # Hold one instance of the pipeline per language.
+  @@parsers = {}
+
+  DefaultOptions = {
+    :parser_model => nil,
+    :tagger_model => nil
+  }
+
+  # Parse the entity using the Stanford parser.
+  #
+  # Options:
+  #
+  # - (Boolean) :silent => whether to silence the output
+  #   of the JVM.
+  # - (String) :log_file => a filename to log output to
+  # instead of displaying it.
+  def self.parse(entity, options = {})
+
+    entity.check_hasnt_children
+
+    val = entity.to_s
+    lang = entity.language
+    init(lang, options)
+
+    text = ::StanfordCoreNLP::Text.new(val)
+    @@parsers[lang].annotate(text)
+
+    text.get(:sentences).each do |s|
+      
+      if entity.is_a?(Treat::Entities::Sentence) ||
+        entity.is_a?(Treat::Entities::Phrase)
+        tag = s.get(:category).to_s
+        tag_s, tag_opt = *tag.split('-')
+        tag_s ||= 'S'
+        entity.set :tag_set, :penn
+        entity.set :tag, tag_s
+        entity.set :tag_opt, tag_opt if tag_opt
+        recurse(s.get(:tree).children[0], entity)
+        break
+      else
+        recurse(s.get(:tree), entity)
+      end
+      
+    end
+
+  end
+
+  def self.init(lang, options)
+    return if @@parsers[lang]
+    options = DefaultOptions.merge(options)
+    StanfordCoreNLP.use(lang)
+    if options[:tagger_model]
+      ::StanfordCoreNLP.set_model(
+      'pos.model', options[:tagger_model]
+      )
+    end
+    if options[:parser_model]
+      ::StanfordCoreNLP.set_model(
+      'parser.model', options[:parser_model]
+      )
+    end
+    @@parsers[lang] ||=
+    ::StanfordCoreNLP.load(
+    :tokenize, :ssplit, :pos, :lemma, :parse
+    )
+  end
+
+  # Helper method which recurses the tree supplied by
+  # the Stanford parser.
+  def self.recurse(java_node, ruby_node, additional_tags = [])
+
+    if java_node.num_children == 0
+
+      label = java_node.label
+      tag = label.get(:part_of_speech).to_s
+      tag_s, tag_opt = *tag.split('-')
+      tag_s ||= ''
+      ruby_node.value = java_node.value.to_s.strip
+      ruby_node.set :tag_set, :penn
+      ruby_node.set :tag, tag_s
+      ruby_node.set :tag_opt, tag_opt if tag_opt
+      ruby_node.set :tag_set, :penn
+      ruby_node.set :lemma, label.get(:lemma).to_s
+
+      additional_tags.each do |t|
+        lt = label.get(t)
+        ruby_node.set t, lt.to_s if lt
+      end
+
+      ruby_node
+
+    else
+
+      if java_node.num_children == 1 &&
+        java_node.children[0].num_children == 0
+        recurse(java_node.children[0],
+        ruby_node, additional_tags)
+        return
+      end
+
+      java_node.children.each do |java_child|
+        label = java_child.label
+        tag = label.get(:category).to_s
+        tag_s, tag_opt = *tag.split('-')
+        tag_s ||= ''
+
+        if Treat::Linguistics::Tags::PhraseTagToCategory[tag_s]
+          ruby_child = Treat::Entities::Phrase.new
+        else
+          l = java_child.children[0].to_s
+          v = java_child.children[0].value.to_s.strip
+          # Mhmhmhmhmhm
+          val = (l == v) ? v :  l.split(' ')[-1].gsub(')', '')
+          ruby_child = Treat::Entities::Token.from_string(val)
         end
-        
-        # Helper method which recurses the tree supplied by
-        # the Stanford parser.
-        def self.recurse(java_node, ruby_node, additional_tags = [])
-          # Leaf
-          if java_node.num_children == 0
-            label = java_node.label
-            tag = label.get(:part_of_speech).to_s
-            tag_s, tag_opt = *tag.split('-')
-            tag_s ||= ''
-            ruby_node.value = java_node.value.to_s.strip
-            ruby_node.set :tag_set, :penn
-            ruby_node.set :tag, tag_s
-            ruby_node.set :tag_opt, tag_opt if tag_opt
-            ruby_node.set :tag_set, :penn
-            ruby_node.set :lemma, label.get(:lemma).to_s
-            
-            ruby_node.set :character_offset_begin, 
-            label.get(:character_offset_begin).to_s
-            
-            ruby_node.set :character_offset_end, 
-            label.get(:character_offset_end).to_s
-            
-            ruby_node.set :begin_index, 
-            label.get(:begin_index).to_s
-            
-            ruby_node.set :end_index, 
-            label.get(:end_index).to_s
-            
-            additional_tags.each do |t|
-              lt = label.get(t)
-              ruby_node.set t, lt.to_s if lt
-            end
-            return ruby_node
-          else
-
-            if java_node.num_children == 1 &&
-              java_node.children[0].num_children == 0
-              recurse(java_node.children[0], ruby_node, additional_tags)
-              return
-            end
-
-            java_node.children.each do |java_child|
-              label = java_child.label
-              tag = label.get(:category).to_s
-              tag_s, tag_opt = *tag.split('-')
-              tag_s ||= ''
-              
-              if Treat::Languages::Tags::PhraseTagToCategory[tag_s]
-                ruby_child = Treat::Entities::Phrase.new
-              else
-                l = java_child.children[0].to_s
-                v = java_child.children[0].value.to_s.strip
-                # Mhmhmhmhmhm
-                val = (l == v) ? v :  l.split(' ')[-1].gsub(')', '')
-                ruby_child = Treat::Entities::Token.from_string(val)
-              end
-
-              ruby_child.set :tag_set, :penn
-              ruby_child.set :tag, tag_s
-              ruby_child.set :tag_opt, tag_opt if tag_opt
-              ruby_node << ruby_child
-
-              unless java_child.children.empty?
-                recurse(java_child, ruby_child, additional_tags)
-              end
-            end
-          end
+
+        ruby_child.set :tag_set, :penn
+        ruby_child.set :tag, tag_s
+        ruby_child.set :tag_opt, tag_opt if tag_opt
+        ruby_node << ruby_child
+
+        unless java_child.children.empty?
+          recurse(java_child, ruby_child, additional_tags)
         end
+
       end
+
     end
+
   end
 end

data/lib/treat/processors/segmenters/punkt.rb

@@ -1,48 +1,82 @@
-module Treat
-  module Processors
-    module Segmenters
-      # An adapter for the 'punk-segmenter' gem, which segments
-      # texts into sentences based on an unsupervised, language
-      # independent algorithm.
-      # 
-      # Original paper: Kiss, Tibor and Strunk, Jan (2006): 
-      # Unsupervised Multilingual Sentence Boundary Detection. 
-      # Computational Linguistics 32: 485-525.
-      class Punkt
-        silence_warnings { require 'punkt-segmenter' }
-        require 'psych'
-        # Hold one copy of the segmenter per language.
-        @@segmenters = {}
-        # Hold only one trainer per language.
-        @@trainers = {}
-        # Segment a text using the Punkt segmenter gem.
-        #
-        # Options: 
-        # 
-        #   :training_text => (String) Text to train the segmenter on.
-        def self.segment(entity, options = {})
-          lang = entity.language
-          if options[:model]
-            model = options[:model]
-          else
-            l = Treat::Languages.describe(lang)
-            model = "#{Treat.lib}/treat/processors/segmenters/punkt/#{l}.yaml"
-            unless File.readable?(model)
-              raise Treat::Exception,
-              "Could not get the language model for the Punkt segmenter for #{l}."
-            end
-          end
-          t = ::Psych.load(File.read(model))
-          @@segmenters[lang] ||= ::Punkt::SentenceTokenizer.new(t)
-          s = entity.to_s
-          s.gsub!(/([^\.\?!]\.|\!|\?)([^\s])/) { $1 + ' ' + $2 }
-          result = @@segmenters[lang].sentences_from_text(
-            s, :output => :sentences_text)
-          result.each do |sentence|
-            entity << Treat::Entities::Phrase.from_string(sentence)
-          end
-        end
+# An adapter for the 'punk-segmenter' gem, which segments
+# texts into sentences based on an unsupervised, language
+# independent algorithm.
+#
+# Original paper: Kiss, Tibor and Strunk, Jan (2006):
+# Unsupervised Multilingual Sentence Boundary Detection.
+# Computational Linguistics 32: 485-525.
+module Treat::Processors::Segmenters::Punkt
+
+  require 'treat/helpers/decimal_point_escaper'
+  
+  # Require silently the punkt-segmenter gem.
+  silence_warnings { require 'punkt-segmenter' }
+  
+  # Require the YAML parser.
+  silence_warnings { require 'psych' }
+  
+  # Hold one copy of the segmenter per language.
+  @@segmenters = {}
+  
+  # Hold only one trainer per language.
+  @@trainers = {}
+  
+  # Segment a text using the Punkt segmenter gem.
+  # The included models for this segmenter have 
+  # been trained on one or two lengthy books 
+  # from the corresponding language.
+  # 
+  # Options:
+  #
+  # (String) :training_text => Text to train on.
+  def self.segment(entity, options = {})
+    
+    entity.check_hasnt_children
+    
+    lang = entity.language
+    set_options(lang, options)
+    
+    s = entity.to_s
+    
+    # Replace all decimal points by ^^
+    Treat::Helpers::DecimalPointEscaper.escape!(s)
+    s.gsub!(/([^\.\?!]\.|\!|\?)([^\s])/) { $1 + ' ' + $2 }
+    
+    result = @@segmenters[lang].
+    sentences_from_text(s, 
+    :output => :sentences_text)
+    
+    result.each do |sentence|
+      Treat::Helpers::DecimalPointEscaper.
+      unescape!(sentence)
+      entity << Treat::Entities::Phrase.
+        from_string(sentence)
+    end
+    
+  end
+  
+  def self.set_options(lang, options)
+    
+    return @@segmenters[lang] if @@segmenters[lang]
+    
+    if options[:model]
+      model = options[:model]
+    else
+      l = Treat::Languages.describe(lang)
+      model = "#{Treat.models}punkt/#{l}.yaml"
+
+      unless File.readable?(model)
+        raise Treat::Exception,
+        "Could not get the language model " +
+        "for the Punkt segmenter for #{l}."
       end
     end
+    
+    t = ::Psych.load(File.read(model))
+
+    @@segmenters[lang] =
+    ::Punkt::SentenceTokenizer.new(t)
+    
   end
-end
+  
+end

data/lib/treat/processors/segmenters/stanford.rb

@@ -1,52 +1,50 @@
-module Treat
-  module Processors
-    module Segmenters
-      # A wrapper for the sentence splitter supplied by
-      # the Stanford parser.
-      class Stanford
-        require 'stanford-core-nlp'
-        DefaultOptions = { 
-          :silence => false, 
-          :log_to_file => false, 
-          :also_tokenize => false 
-        }
-        # Segment sentences using the sentence splitter supplied by
-        # the Stanford parser. By default, this segmenter also adds
-        # the tokens as children of the sentences.
-        #
-        # Options:
-        # - (Boolean) :also_tokenize - Whether to also add the tokens
-        # as children of the sentence.
-        # - (String) :log_to_file => a filename to log output to
-        # instead of displaying it.
-        # - (String) :silence => send 
-        def self.segment(entity, options = {})
-          options = DefaultOptions.merge(options)
-          options[:log_to_file] = '/dev/null' if options[:silence]
-          if options[:log_to_file]
-            ::StanfordCoreNLP.log_file = options[:log_to_file] 
-          end
-          options = DefaultOptions.merge(options)
-          pipeline =  ::StanfordCoreNLP.load(:tokenize, :ssplit)
-          text = ::StanfordCoreNLP::Text.new(entity.to_s)
-          pipeline.annotate(text)
-          text.get(:sentences).each do |sentence|
-            s = Treat::Entities::Sentence.from_string(sentence.to_s, true)
-            entity << s
-            if options[:also_tokenize]
-              sentence.get(:tokens).each do |token|
-                t = Treat::Entities::Phrase.from_string(token.value)
-                s << t
-                t.set :character_offset_begin, 
-                token.get(:character_offset_begin)
-                
-                t.set :character_offset_end, 
-                token.get(:character_offset_end)
-              end
-            end
-          end
-        end
+# A wrapper for the sentence splitter supplied by
+# the Stanford parser.
+class Treat::Processors::Segmenters::Stanford
+
+  require 'treat/loaders/stanford'
+
+  DefaultOptions = {
+    :also_tokenize => false
+  }
+
+  # Keep one copy of the Stanford Core NLP pipeline.
+  @@segmenter = nil
+
+  # Segment sentences using the sentence splitter
+  # supplied by the Stanford parser. For better
+  # performance, set the option :also_tokenize
+  # to true, and this segmenter will also add
+  # the tokens as children of the sentences.
+  #
+  # Options:
+  #
+  # - (Boolean) :also_tokenize - Whether to also
+  # add the tokens as children of the sentence.
+  def self.segment(entity, options = {})
+
+    options = DefaultOptions.merge(options)
+    entity.check_hasnt_children
+
+    @@segmenter ||=
+    ::StanfordCoreNLP.load(:tokenize, :ssplit)
+    
+    s = entity.to_s
+    
+    text = ::StanfordCoreNLP::Text.new(entity.to_s)
+
+    @@segmenter.annotate(text)
+    text.get(:sentences).each do |sentence|
+      sentence = sentence.to_s
+      s = Treat::Entities::Sentence.
+      from_string(sentence, true)
+      entity << s
+      if options[:also_tokenize]
+        Treat::Processors::Tokenizers::Stanford.
+        add_tokens(s, sentence.get(:tokens))
       end
     end
+
   end
+
 end