treat 1.2.0 → 2.0.0rc1

data/lib/treat/workers/lexicalizers/taggers/brill.rb

@@ -1,23 +1,13 @@
-# Adapter class for the 'rbtagger' gem, a port
-# of the Perl Lingua::BrillTagger class, based
-# on the rule-based tagger developped by Eric Brill.
+# POS tagging using a set of rules developped by Eric Brill.
 #
-# Original paper:
-#
-# Eric Brill. 1992. A simple rule-based part of speech tagger.
-# In Proceedings of the third conference on Applied natural
-# language processing (ANLC '92). Association for Computational
-# Linguistics, Stroudsburg, PA, USA, 152-155.
-# DOI=10.3115/974499.974526 http://dx.doi.org/10.3115/974499.974526
-#
-# Project website:
-#
-# http://rbtagger.rubyforge.org/
-module Treat::Workers::Lexicalizers::Taggers::Brill
+# Original paper: Eric Brill. 1992. A simple rule-based 
+# part of speech tagger. In Proceedings of the third
+# conference on Applied natural language processing.
+class Treat::Workers::Lexicalizers::Taggers::Brill
 
   require 'rbtagger'
 
-  require 'treat/workers/lexicalizers/taggers/brill/patch'
+  require_relative 'brill/patch'
 
   # Hold one instance of the tagger.
   @@tagger = nil
@@ -50,9 +40,8 @@ module Treat::Workers::Lexicalizers::Taggers::Brill
       return pair[1] if isolated_token
     end
     
-    if entity.is_a?(Treat::Entities::Sentence) ||
-      (entity.is_a?(Treat::Entities::Phrase) && 
-      !entity.parent_sentence)
+    if entity.is_a?(Treat::Entities::Group) && 
+      !entity.parent_sentence
         entity.set :tag_set, :penn
     end
     

data/lib/treat/workers/lexicalizers/taggers/lingua.rb

@@ -1,17 +1,7 @@
-# An adapter for the 'engtagger' gem, which
-# is a port of the Perl Lingua::EN::Tagger module.
-#
-# "This module uses part-of-speech statistics from
+# POS tagging using part-of-speech statistics from
 # the Penn Treebank to assign POS tags to English text.
 # The tagger applies a bigram (two-word) Hidden Markov
 # Model to guess the appropriate POS tag for a word.
-# That means that the tagger will try to assign a POS
-# tag based on the known POS tags for a given word and
-# the POS tag assigned to its predecessor.
-#
-# Project website: http://engtagger.rubyforge.org/
-# Original Perl module site:
-# http://cpansearch.perl.org/src/ACOBURN/Lingua-EN-Tagger-0.15/
 class Treat::Workers::Lexicalizers::Taggers::Lingua
   
   # Require the 'engtagger' gem.
@@ -71,15 +61,14 @@ class Treat::Workers::Lexicalizers::Taggers::Lingua
     end
 
     
-    if entity.is_a?(Treat::Entities::Sentence) ||
-      (entity.is_a?(Treat::Entities::Phrase) && 
-      !entity.parent_sentence)
+    if entity.is_a?(Treat::Entities::Group) && 
+      !entity.parent_sentence
         entity.set :tag_set, :penn
     end
 
     return 'S' if entity.is_a?(Treat::Entities::Sentence)
     return 'P' if entity.is_a?(Treat::Entities::Phrase)
-    
+
   end
   
 end

data/lib/treat/workers/lexicalizers/taggers/stanford.rb

@@ -1,8 +1,18 @@
-# Wrapper for the Stanford POS tagger.
+# POS tagging using (i) explicit use of both preceding 
+# and following tag contexts via a dependency network 
+# representation, (ii) broad use of lexical features, 
+# including jointly conditioning on multiple consecutive 
+# words, (iii) effective use of priors in conditional 
+# loglinear models, and (iv) fine-grained modeling of 
+# unknown word features.
+#
+# Original paper: Toutanova, Manning, Klein and Singer.
+# 2003. Feature-Rich Part-of-Speech Tagging with a 
+# Cyclic Dependency Network. In Proceedings of the 
+# Conference of the North American Chapter of the 
+# Association for Computational Linguistics.
 class Treat::Workers::Lexicalizers::Taggers::Stanford
 
-  require 'treat/loaders/stanford'
-
   # Hold one tagger per language.
   @@taggers = {}
 
@@ -15,9 +25,8 @@ class Treat::Workers::Lexicalizers::Taggers::Stanford
   def self.tag(entity, options = {})
 
     # Handle tags for sentences and phrases.
-    if entity.is_a?(Treat::Entities::Sentence) ||
-      (entity.is_a?(Treat::Entities::Phrase) &&
-      !entity.parent_sentence)
+    if entity.is_a?(Treat::Entities::Group) &&
+      !entity.parent_sentence
 
       tag_set = options[:tag_set]
       entity.set :tag_set, tag_set
@@ -31,18 +40,18 @@ class Treat::Workers::Lexicalizers::Taggers::Stanford
     
     # Handle options and initialize the tagger.
     lang = entity.language
-    options = get_options(options, lang)
     init_tagger(lang) unless @@taggers[lang]
-    tokens, list = get_token_list(entity)
+    options = get_options(options, lang)
+    tokens, t_list = get_token_list(entity)
 
     # Do the tagging.
     i = 0
     isolated_token = entity.is_a?(Treat::Entities::Token)
-
-    @@taggers[lang].apply(list).each do |tok|
-      tokens[i].set :tag, tok.tag
-      tokens[i].set :tag_set,
-      options[:tag_set] if isolated_token
+    
+    @@taggers[lang].apply(t_list).each do |tok|
+      tokens[i].set(:tag, tok.tag)
+      tokens[i].set(:tag_set,
+      options[:tag_set]) if isolated_token
       return tok.tag if isolated_token
       i += 1
     end

data/lib/treat/workers/processors/chunkers/autoselect.rb

@@ -2,12 +2,11 @@ class Treat::Workers::Processors::Chunkers::Autoselect
 
   def self.chunk(entity, options = {})
     unless entity.has?(:format)
-      raise Treat::Exception,
-      "Must have a format to autoselect chunker."
+      entity.set :format, 'txt'
     end
     begin
       k = Treat::Workers::Processors::
-      Chunkers.const_get(cc(entity.format))
+      Chunkers.const_get(entity.format.cc)
       k.chunk(entity, options)
     rescue Treat::Exception
       Treat::Workers::Processors::

data/lib/treat/workers/processors/chunkers/html.rb

@@ -3,12 +3,9 @@ class Treat::Workers::Processors::Chunkers::HTML
   require 'nokogiri'
 
   def self.chunk(entity, options = {})
-    
     entity.check_hasnt_children
-    
     doc = Nokogiri::HTML(entity.value)
-    recurse(entity, doc)
-  
+    self.recurse(entity, doc)
   end
 
   def self.recurse(node, html_node, level = 1)
@@ -16,7 +13,6 @@ class Treat::Workers::Processors::Chunkers::HTML
     html_node.children.each do |child|
       
       next if child.name == 'text'
-
       txt = child.inner_text
       
       if child.name =~ /^h([0-9]{1})$/ ||
@@ -45,7 +41,6 @@ class Treat::Workers::Processors::Chunkers::HTML
         
         t = node <<
         Treat::Entities::Title.new(txt)
-        t.set :level, level
 
       elsif child.name == 'p'
 

data/lib/treat/workers/processors/parsers/enju.rb

@@ -5,12 +5,10 @@
 # XML reader. It creates wrappers for the sentences,
 # syntactical phrases and tokens that Enju identified.
 #
-# Original paper:
-#
-# Takuya Matsuzaki, Yusuke Miyao, and Jun'ichi Tsujii.
+# Original paper: Takuya M., Yusuke M., and Jun'ichi T.
 # 2007. Efficient HPSG Parsing with Supertagging and
 # CFG-filtering. In Proceedings of IJCAI 2007.
-module Treat::Workers::Processors::Parsers::Enju
+class Treat::Workers::Processors::Parsers::Enju
 
   # Require the 'open3' library to connect
   # with the background Enju process.

data/lib/treat/workers/processors/parsers/stanford.rb

@@ -1,8 +1,14 @@
-# A wrapper class for the Stanford parser.
+# Parsing using an interface to a Java implementation 
+# of probabilistic natural language parsers, both 
+# optimized PCFG and lexicalized dependency parsers, 
+# and a lexicalized PCFG parser. 
+# 
+# Original paper: Dan Klein and Christopher D. 
+# Manning. 2003. Accurate Unlexicalized Parsing. 
+# Proceedings of the 41st Meeting of the Association 
+# for Computational Linguistics, pp. 423-430.
 class Treat::Workers::Processors::Parsers::Stanford
-
-  require 'treat/loaders/stanford'
-
+  
   Pttc = Treat.tags.aligned.phrase_tags_to_category
   
   # Hold one instance of the pipeline per language.
@@ -23,12 +29,12 @@ class Treat::Workers::Processors::Parsers::Stanford
   # instead of displaying it.
   def self.parse(entity, options = {})
 
-    entity.check_hasnt_children
-
     val = entity.to_s
     lang = entity.language
     init(lang, options)
     
+    entity.check_hasnt_children
+    
     tag_set = StanfordCoreNLP::Config::TagSets[lang]
     
     text = ::StanfordCoreNLP::Text.new(val)
@@ -46,7 +52,7 @@ class Treat::Workers::Processors::Parsers::Stanford
         recurse(s.get(:tree).children[0], entity, tag_set)
         break #######
       else
-        recurse(s.get(:tree), entity)
+        recurse(s.get(:tree), entity, tag_set)
       end
       
     end

data/lib/treat/workers/processors/segmenters/punkt.rb

@@ -1,17 +1,19 @@
-# An adapter for the 'punk-segmenter' gem, which segments
-# texts into sentences based on an unsupervised, language
-# independent algorithm.
+# Sentence segmentation based on a set of log-
+# likelihood-based heuristics to infer abbreviations
+# and common sentence starters from a large text
+# corpus. Easily adaptable but requires a large 
+# (unlabeled) indomain corpus for assembling statistics.
 #
-# Original paper: Kiss, Tibor and Strunk, Jan (2006):
+# Original paper: Kiss, Tibor and Strunk, Jan. 2006.
 # Unsupervised Multilingual Sentence Boundary Detection.
-# Computational Linguistics 32: 485-525.
-module Treat::Workers::Processors::Segmenters::Punkt
+# Computational Linguistics 32:485-525.
+class Treat::Workers::Processors::Segmenters::Punkt
   
   # Require silently the punkt-segmenter gem.
   silence_warnings { require 'punkt-segmenter' }
   
   # Require the YAML parser.
-  silence_warnings { require 'psych' }
+  # silence_warnings { require 'psych' }
   
   # Hold one copy of the segmenter per language.
   @@segmenters = {}
@@ -34,13 +36,21 @@ module Treat::Workers::Processors::Segmenters::Punkt
     lang = entity.language
     set_options(lang, options)
     
+   
     s = entity.to_s
     
     # Replace the point in all floating-point numbers
     # by ^^; this is a fix since Punkt trips on decimal 
     # numbers.
-
-    escape_floats!(s)
+    s.escape_floats!
+    
+    # Take out suspension points temporarily.
+    s.gsub!('...', '&;&.')
+    # Remove abbreviations.
+    s.scan(/(?:[A-Za-z]\.){2,}/).each do |abbr| 
+      s.gsub!(abbr, abbr.gsub(' ', '').gsub('.', '&-&'))
+    end
+    # Unstick sentences from each other.
     s.gsub!(/([^\.\?!]\.|\!|\?)([^\s"'])/) { $1 + ' ' + $2 }
     
     result = @@segmenters[lang].
@@ -49,7 +59,11 @@ module Treat::Workers::Processors::Segmenters::Punkt
     
     result.each do |sentence|
       # Unescape the sentence.
-      unescape_floats!(sentence)
+      sentence.unescape_floats!
+      # Repair abbreviations in sentences.
+      sentence.gsub!('&-&', '.')
+      # Repair suspension points.
+      sentence.gsub!('&;&.', '...')
       entity << Treat::Entities::Phrase.
         from_string(sentence)
     end
@@ -73,7 +87,7 @@ module Treat::Workers::Processors::Segmenters::Punkt
       end
     end
     
-    t = ::Psych.load(File.read(model))
+    t = ::YAML.load(File.read(model))
 
     @@segmenters[lang] =
     ::Punkt::SentenceTokenizer.new(t)

data/lib/treat/workers/processors/segmenters/scalpel.rb

@@ -0,0 +1,20 @@
+# Sentence segmentation based on a set of predefined
+# rules that handle a large number of usage cases of
+# sentence enders. The idea is to remove all cases of
+# .!? being used for other purposes than marking a
+# full stop before naively segmenting the text.
+class Treat::Workers::Processors::Segmenters::Scalpel
+
+  require 'scalpel'
+  
+  # Segment a text using the Scalpel algorithm.
+  def self.segment(entity, options = {})
+    sentences = Scalpel.cut(entity.to_s)
+    sentences.each do |sentence|
+      entity << Treat::Entities::Phrase.
+      from_string(sentence.strip)
+    end
+    entity
+  end
+
+end

data/lib/treat/workers/processors/segmenters/srx.rb

@@ -0,0 +1,42 @@
+# Sentence segmentation based on a set of predefined
+# rules defined in SRX (Segmentation Rules eXchange)
+# format and developped by Marcin Milkowski.
+#
+# Original paper: Marcin Miłkowski, Jarosław Lipski,
+# 2009. Using SRX standard for sentence segmentation
+# in LanguageTool, in: Human Language Technologies
+# as a Challenge for Computer Science and Linguistics.
+class Treat::Workers::Processors::Segmenters::SRX
+
+  @@segmenters = {}
+
+  # Require the srx-english library.
+  # Segment a text using the SRX algorithm
+  def self.segment(entity, options = {})
+
+    lang = entity.language
+    entity.check_hasnt_children
+    text = entity.to_s
+    text.escape_floats!
+
+    unless @@segmenters[lang]
+      # Require the appropriate gem.
+      require "srx/#{lang}/sentence_splitter"
+      @@segmenters[lang] = SRX.const_get(
+      lang.capitalize).const_get(
+      'SentenceSplitter')
+    end
+
+    sentences = @@segmenters[lang].new(text)
+
+    sentences.each do |sentence|
+      sentence.unescape_floats!
+      entity << Treat::Entities::Phrase.
+      from_string(sentence.strip)
+    end
+
+    entity
+
+  end
+
+end

data/lib/treat/workers/processors/segmenters/stanford.rb

@@ -1,8 +1,9 @@
-# A wrapper for the sentence splitter supplied by
-# the Stanford parser.
+# Detects sentence boundaries by first tokenizing the
+# text and deciding whether periods are sentence ending 
+# or used for other purposes (abreviations, etc.).  The  
+# obtained tokens are then grouped into sentences.
 class Treat::Workers::Processors::Segmenters::Stanford
 
-  require 'treat/loaders/stanford'
   Treat::Loaders::Stanford.load
   
   DefaultOptions = {
@@ -31,8 +32,7 @@ class Treat::Workers::Processors::Segmenters::Stanford
     ::StanfordCoreNLP.load(:tokenize, :ssplit)
     
     s = entity.to_s
-    
-    text = ::StanfordCoreNLP::Text.new(entity.to_s)
+    text = ::StanfordCoreNLP::Text.new(s)
 
     @@segmenter.annotate(text)
     text.get(:sentences).each do |sentence|

data/lib/treat/workers/processors/segmenters/tactful.rb

@@ -1,13 +1,12 @@
-# An adapter for the 'tactful_tokenizer' gem, which
-# detects sentence boundaries based on a Naive Bayesian
-# statistical model.
+# Sentence segmentation based on a Naive Bayesian
+# statistical model. Trained on Wall Street Journal 
+# news combined with the Brown Corpus, which is 
+# intended to be widely representative of written English.
 #
-# Project website: https://github.com/SlyShy/Tackful-Tokenizer
-#
-# Original paper: Dan Gillick. 2009. Sentence Boundary Detection
-# and the Problem with the U.S. University of California, Berkeley.
-# http://dgillick.com/resource/sbd_naacl_2009.pdf
-module Treat::Workers::Processors::Segmenters::Tactful
+# Original paper: Dan Gillick. 2009. Sentence Boundary 
+# Detection and the Problem with the U.S. University 
+# of California, Berkeley.
+class Treat::Workers::Processors::Segmenters::Tactful
   
   # Require the 'tactful_tokenizer' gem.
   silence_warnings { require 'tactful_tokenizer' }
@@ -27,9 +26,16 @@ module Treat::Workers::Processors::Segmenters::Tactful
     entity.check_hasnt_children
     
     s = entity.to_s
+    s.escape_floats!
     
-    escape_floats!(s)
+    # Remove abbreviations.
+    s.scan(/(?:[A-Za-z]\.){2,}/).each do |abbr| 
+      s.gsub!(abbr, abbr.gsub(' ', '').gsub('.', '&-&'))
+    end
     
+    # Take out suspension points temporarily.
+    s.gsub!('...', '&;&.')
+    # Unstick sentences from each other.
     s.gsub!(/([^\.\?!]\.|\!|\?)([^\s"'])/) { $1 + ' ' + $2 }
     
     @@segmenter ||= TactfulTokenizer::Model.new
@@ -37,7 +43,11 @@ module Treat::Workers::Processors::Segmenters::Tactful
     sentences = @@segmenter.tokenize_text(s)
     
     sentences.each do |sentence|
-      unescape_floats!(sentence)
+      sentence.unescape_floats!
+      # Repair abbreviations.
+      sentence.gsub!('&-&', '.')
+      # Repair suspension points.
+      sentence.gsub!('&;&.', '...')
       entity << Treat::Entities::Phrase.from_string(sentence)
     end
     

data/lib/treat/workers/processors/tokenizers/ptb.rb

@@ -1,45 +1,52 @@
 # encoding: utf-8
-# A native rule-basd tokenizer based on the one
-# developped by Robert Macyntyre in 1995 for the Penn
-# Treebank project. This tokenizer follows the
-# conventions used by the Penn Treebank.
+# Tokenization based on the tokenizer developped by
+# Robert Macyntyre in 1995 for the Penn Treebank
+# project. This tokenizer mostly follows the conventions
+# used by the Penn Treebank. N.B. Contrary to the 
+# standard PTB tokenization, double quotes (") are 
+# NOT changed to doubled single forward- and 
+# backward- quotes (`` and '') by default.
 #
-# Original script:
-# http://www.cis.upenn.edu/~treebank/tokenizer.sed
-#
-# Copyright (c) 2004 UTIYAMA Masao <mutiyama@nict.go.jp>
-# All rights reserved. This program is free software;
-# you can redistribute it and/or modify it under the
-# same terms as Ruby itself.
-module Treat::Workers::Processors::Tokenizers::PTB
-  
-  # Tokenize the entity using a native rule-based algorithm.
+# Authors: Utiyama Masao (mutiyama@nict.go.jp).
+# License: Ruby License.
+class Treat::Workers::Processors::Tokenizers::PTB
+
+  # Default options for the tokenizer.
+  DefaultOptions = {
+    directional_quotes: false
+  }
+
+  # Perform tokenization of the entity and add
+  # the resulting tokens as its children.
+  #
+  # Options:
+  # - (Boolean) => :directional_quotes whether to
+  # replace double quotes by `` and '' or not.
   def self.tokenize(entity, options = {})
-    
+    options = DefaultOptions.merge(options)
     entity.check_hasnt_children
-    
     if entity.has_children?
       raise Treat::Exception,
       "Cannot tokenize an #{entity.class} " +
       "that already has children."
     end
-    chunks = split(entity.to_s)
+    chunks = split(entity.to_s, options)
     chunks.each do |chunk|
       next if chunk =~ /([[:space:]]+)/
-      entity << Treat::Entities::Token.from_string(chunk)
+      entity << Treat::Entities::Token.
+      from_string(chunk)
     end
   end
-  
-  # Helper method to split the string into tokens.
-  def self.split(string)
-    
+
+  def self.split(string, options)
+
     s = " " + string + " "
-    
-    s.gsub!(/‘/,'`')
+
+    s.gsub!(/‘/,"'")
     s.gsub!(/’/,"'")
     s.gsub!(/“/,"``")
     s.gsub!(/”/,"''")
-  
+
     s.gsub!(/\s+/," ")
     s.gsub!(/(\s+)''/,'\1"')
     s.gsub!(/(\s+)``/,'\1"')
@@ -80,11 +87,14 @@ module Treat::Workers::Processors::Tokenizers::PTB
     s.gsub!(/\//, ' / ')
     s.gsub!(/\s+/,' ')
     s.strip!
-  
-    s.gsub!(/``/,'"')
-    s.gsub!(/''/,'"')
     
+    # Remove directional quotes.
+    unless options[:directional_quotes]
+      s.gsub!(/``/,'"')
+      s.gsub!(/''/,'"')
+    end
+
     s.split(/\s+/)
   end
-  
-end
+
+end

data/lib/treat/workers/processors/tokenizers/punkt.rb

@@ -1,19 +1,11 @@
-# A tokenizer that was lifted from the 'punkt-segmenter'
-# Ruby gem.
+# Tokenization script from the 'punkt-segmenter' Ruby gem.
 #
-# This code follows the terms and conditions of Apache
-# License v2 (http://www.apache.org/licenses/LICENSE-2.0)
-#
-# Authors: Willy <willy@csse.unimelb.edu.au>
-# (original Python port), Steven Bird
-# <sb@csse.unimelb.edu.au> (additions),
-# Edward Loper <edloper@gradient.cis.upenn.edu>
-# (rewrite), Joel Nothman <jnothman@student.usyd.edu.au>
-# (almost rewrite).
-#
-# Project website: https://github.com/lfcipriani/punkt-segmenter
+# Authors: Willy (willy@csse.unimelb.edu.au>), 
+# Steven Bird (sb@csse.unimelb.edu.au), Edward Loper 
+# (edloper@gradient.cis.upenn.edu), Joel Nothman 
+# (jnothman@student.usyd.edu.au).
+# License: Apache License v2.
 class Treat::Workers::Processors::Tokenizers::Punkt
-  
   SentEndChars = ['.', '?', '!']
   ReSentEndChars = /[.?!]/
   InternalPunctuation = [',', ':', ';']
@@ -24,8 +16,8 @@ class Treat::Workers::Processors::Tokenizers::Punkt
   ReWordTokenizer = /#{ReMultiCharPunct}|(?=#{ReWordStart})\S+?(?=\s|$|#{ReNonWordChars}|#{ReMultiCharPunct}|,(?=$|\s|#{ReNonWordChars}|#{ReMultiCharPunct}))|\S/
   RePeriodContext = /\S*#{ReSentEndChars}(?=(?<after_tok>#{ReNonWordChars}|\s+(?<next_tok>\S+)))/
   
-  # Tokenize the text using the algorithm lifted from
-  # the Punkt tokenizer gem.
+  # Perform tokenization of the entity and add
+  # the resulting tokens as its children.
   #
   # Options: none.
   def self.tokenize(entity, options = {})
@@ -36,10 +28,13 @@ class Treat::Workers::Processors::Tokenizers::Punkt
     
     s.scan(ReWordTokenizer).each do |token|
       if SentEndChars.include?(token[-1])
-        entity << Treat::Entities::Token.from_string(token[0..-2])
-        entity << Treat::Entities::Token.from_string(token[-1..-1])
+        entity << Treat::Entities::
+        Token.from_string(token[0..-2])
+        entity << Treat::Entities::
+        Token.from_string(token[-1..-1])
       else
-        entity << Treat::Entities::Token.from_string(token)
+        entity << Treat::Entities::
+        Token.from_string(token)
       end
     end