treat 0.2.5 → 1.0.0

data/lib/treat/processors/segmenters/tactful.rb

@@ -1,38 +1,45 @@
-module Treat
-  module Processors
-    module Segmenters
-      # An adapter for the 'tactful_tokenizer' gem, which
-      # detects sentence boundaries (the name is a misnomer;
-      # it isn't a tokenizer, but a sentence boundary detector).
-      # It uses a Naive Bayesian statistical model, and is
-      # based on Splitta, but has support for ‘?’ and ‘!’
-      # as well as primitive handling of XHTML markup.
-      #
-      # Project website: https://github.com/SlyShy/Tackful-Tokenizer
-      # Original paper: Dan Gillick. 2009. Sentence Boundary Detection 
-      # and the Problem with the U.S. University of California, Berkeley.
-      # http://dgillick.com/resource/sbd_naacl_2009.pdf
-      class Tactful
-        # Require the 'tactful_tokenizer' gem.
-        silence_warnings { require 'tactful_tokenizer' }
-        # Remove function definition 'tactful_tokenizer' by gem.
-        String.class_eval { undef :tokenize }
-        # Keep only one copy of the segmenter.
-        @@segmenter = nil
-        # Segment a text or zone into sentences
-        # using the 'tactful_tokenizer' gem.
-        #
-        # Options: none.
-        def self.segment(entity, options = {})
-          @@segmenter ||= TactfulTokenizer::Model.new
-          s = entity.to_s
-          s.gsub!(/([^\.\?!]\.|\!|\?)([^\s])/) { $1 + ' ' + $2 }
-          sentences = @@segmenter.tokenize_text(s)
-          sentences.each do |sentence|
-            entity << Entities::Phrase.from_string(sentence)
-          end
-        end
-      end
+# An adapter for the 'tactful_tokenizer' gem, which
+# detects sentence boundaries based on a Naive Bayesian
+# statistical model.
+#
+# Project website: https://github.com/SlyShy/Tackful-Tokenizer
+#
+# Original paper: Dan Gillick. 2009. Sentence Boundary Detection
+# and the Problem with the U.S. University of California, Berkeley.
+# http://dgillick.com/resource/sbd_naacl_2009.pdf
+module Treat::Processors::Segmenters::Tactful
+  
+  # Require the 'tactful_tokenizer' gem.
+  silence_warnings { require 'tactful_tokenizer' }
+  
+  # Remove function definition 'tactful_tokenizer' by gem.
+  String.class_eval { undef :tokenize }
+  
+  require 'treat/helpers/decimal_point_escaper'
+  
+  # Keep only one copy of the segmenter.
+  @@segmenter = nil
+  
+  # Segment a text or zone into sentences
+  # using the 'tactful_tokenizer' gem.
+  #
+  # Options: none.
+  def self.segment(entity, options = {})
+
+    entity.check_hasnt_children
+    
+    s = entity.to_s
+    Treat::Helpers::DecimalPointEscaper.escape!(s)
+    
+    s.gsub!(/([^\.\?!]\.|\!|\?)([^\s])/) { $1 + ' ' + $2 }
+    
+    @@segmenter ||= TactfulTokenizer::Model.new
+   
+    sentences = @@segmenter.tokenize_text(s)
+    sentences.each do |sentence|
+      Treat::Helpers::DecimalPointEscaper.unescape!(sentence)
+      entity << Treat::Entities::Phrase.from_string(sentence)
     end
   end
-end
+  
+end

data/lib/treat/processors/tokenizers/perl.rb

@@ -1,96 +1,128 @@
 # encoding: utf-8
-module Treat
-  module Processors
-    module Tokenizers
-      # Tokenize the entity using a native rule-based algorithm.
-      # This tokenizer is a port from an unknown Perl module,
-      # which I have lifted from the 'rbtagger' gem.
-      #
-      # Author: Todd A. Fisher
-      # This code is free to use under the terms of the MIT license.
-      #
-      # Original project website:
-      # https://github.com/taf2/rb-brill-tagger
-      class Perl
-        # Tokenize the entity using a native rule-based algorithm.
-        # Options: none.
-        def self.tokenize(entity, options = {})
-          # Normalize all whitespace
-          text = entity.to_s.gsub(/\s+/,' ')
-          # Translate some common extended ascii characters to quotes
-          text.gsub!(/‘/,'`')
-          text.gsub!(/’/,"'")
-          text.gsub!(/“/,"``")
-          text.gsub!(/”/,"''")
-          # Attempt to get correct directional quotes
-          # s{\"\b} { `` }g;
-          text.gsub!(/\"\b/,' `` ')
-          # s{\b\"} { '' }g;
-          text.gsub!(/\b\"/," '' ")
-          #s{\"(?=\s)} { '' }g;
-          text.gsub!(/\"(?=\s)/," '' ")
-          #s{\"} { `` }g;
-          text.gsub!(/\"(?=\s)/," `` ")
-          # Isolate ellipses
-          # s{\.\.\.}   { ... }g;
-          text.gsub!(/\.\.\./,' ... ')
-          # Isolate any embedded punctuation chars
-          #   s{([,;:\@\#\$\%&])} { $1 }g;
-          text.gsub!(/([,;:\@\#\$\%&])/, ' \1 ')
-          # Assume sentence tokenization has been done first, so split FINAL
-          # periods only.
-          # s/ ([^.]) \.  ([\]\)\}\>\"\']*) [ \t]* $ /$1 .$2 /gx;
-          text.gsub!(/ ([^.]) \.  ([\]\)\}\>\"\']*) [ \t]* $ /x, '\1 .\2 ')
-          # however, we may as well split ALL question marks and exclamation points,
-          # since they shouldn't have the abbrev.-marker ambiguity problem
-          #s{([?!])} { $1 }g;
-          text.gsub!(/([?!])/, ' \1 ')
-          # parentheses, brackets, etc.
-          #s{([\]\[\(\)\{\}\<\>])} { $1 }g;
-          text.gsub!(/([\]\[\(\)\{\}\<\>])/,' \1 ')
-          #s/(-{2,})/ $1 /g;
-          text.gsub!(/(-{2,})/,' \1 ')
-          # Add a space to the beginning and end of each line, to reduce
-          # necessary number of regexps below.
-          #s/$/ /;
-          text.gsub!(/$/," ")
-          #s/^/ /;
-          text.gsub!(/^/," ")
-          # possessive or close-single-quote
-          #s/\([^\']\)\' /$1 \' /g;
-          text.gsub!(/\([^\']\)\' /,%q(\1 ' ))
-          # as in it's, I'm, we'd
-          #s/\'([smd]) / \'$1 /ig;
-          text.gsub!(/\'([smd]) /i,%q( '\1 ))
-          #s/\'(ll|re|ve) / \'$1 /ig;
-          text.gsub!(/\'(ll|re|ve) /i,%q( '\1 ))
-          #s/n\'t / n\'t /ig;
-          text.gsub!(/n\'t /i,"  n't ")
-          #s/ (can)(not) / $1 $2 /ig;
-          text.gsub!(/ (can)(not) /i,' \1 \2 ')
-          #s/ (d\')(ye) / $1 $2 /ig;
-          text.gsub!(/ (d\')(ye) /i,' \1 \2 ')
-          #s/ (gim)(me) / $1 $2 /ig;
-          text.gsub!(/ (gim)(me) /i,' \1 \2 ')
-          #s/ (gon)(na) / $1 $2 /ig;
-          text.gsub!(/ (gon)(na) /i,' \1 \2 ')
-          #s/ (got)(ta) / $1 $2 /ig;
-          text.gsub!(/ (got)(ta) /i,' \1 \2 ')
-          #s/ (lem)(me) / $1 $2 /ig;
-          text.gsub!(/ (lem)(me) /i,' \1 \2 ')
-          #s/ (more)(\'n) / $1 $2 /ig;
-          text.gsub!(/ (more)(\'n) /i,' \1 \2 ')
-          #s/ (\'t)(is|was) / $1 $2 /ig;
-          text.gsub!(/ (\'t)(is|was) /i,' \1 \2 ')
-          #s/ (wan)(na) / $1 $2 /ig;
-          text.gsub!(/ (wan)(na) /i,' \1 \2 ')
-          tokens = text.split(/\s/)
-          tokens[1..-1].each do |token|
-            next if token =~ /([[:space:]]+)/
-            entity << Treat::Entities::Token.from_string(token)
-          end
-        end
-      end
+#
+# Tokenize the entity using a native rule-based 
+# algorithm. This tokenizer is a port from an 
+# unknown Perl module, which I have lifted from 
+# the 'rbtagger' gem.
+#
+# Author: Todd A. Fisher
+# 
+# This code is free to use under the terms of 
+# the MIT license.
+#
+# Original project website:
+# 
+# https://github.com/taf2/rb-brill-tagger
+module Treat::Processors::Tokenizers::Perl
+
+  require 'treat/helpers/decimal_point_escaper'
+  
+  # Tokenize the entity using a rule-based algorithm
+  # ported from Perl by Todd A. Fisher.
+  #
+  # Options: none.
+  def self.tokenize(entity, options = {})
+    
+    entity.check_hasnt_children
+    s = entity.to_s
+    
+    tokens = get_tokens(entity.to_s)
+    tokens[1..-1].each do |token|
+      next if token =~ /^\s*$/
+      entity << Treat::Entities::Token.
+      from_string(token)
     end
+
   end
+
+  # Helper method to perform the tokenization.
+  def self.get_tokens(string)
+
+    # Normalize all whitespace
+    text = string.gsub(/\s+/,' ')
+    
+    # Replace all decimal points by ^^
+    Treat::Helpers::DecimalPointEscaper.escape!(text)
+
+    # Translate some common extended ascii 
+    # characters to quotes
+    text.gsub!(/‘/,'`')
+    text.gsub!(/’/,"'")
+    text.gsub!(/“/,"``")
+    text.gsub!(/”/,"''")
+
+    # Attempt to get correct directional quotes
+    # s{\"\b} { `` }g;
+    text.gsub!(/\"\b/,' `` ')
+    # s{\b\"} { '' }g;
+    text.gsub!(/\b\"/," '' ")
+    #s{\"(?=\s)} { '' }g;
+    text.gsub!(/\"(?=\s)/," '' ")
+    #s{\"} { `` }g;
+    text.gsub!(/\"(?=\s)/," `` ")
+    # Isolate ellipses
+    # s{\.\.\.}   { ... }g;
+    text.gsub!(/\.\.\./,' ... ')
+    # Isolate any embedded punctuation chars
+    #   s{([,;:\@\#\$\%&])} { $1 }g;
+    text.gsub!(/([,;:\@\#\$\%&])/, ' \1 ')
+
+    # Assume sentence tokenization has been 
+    # done first, so split FINAL
+    # periods only.
+    # s/ ([^.]) \.  ([\]\)\}\>\"\']*) 
+    # [ \t]* $ /$1 .$2 /gx;
+    text.gsub!(/ ([^.]) \.  ([\]\)\}\>\"\']*) [ \t]* $ /x, '\1 .\2 ')
+    # however, we may as well split ALL 
+    # question marks and exclamation points,
+    # since they shouldn't have the abbrev.
+    # -marker ambiguity problem
+    #s{([?!])} { $1 }g;
+    text.gsub!(/([?!])/, ' \1 ')
+    # parentheses, brackets, etc.
+    #s{([\]\[\(\)\{\}\<\>])} { $1 }g;
+    text.gsub!(/([\]\[\(\)\{\}\<\>])/,' \1 ')
+    #s/(-{2,})/ $1 /g;
+    text.gsub!(/(-{2,})/,' \1 ')
+
+    # Add a space to the beginning and end of 
+    # each line, to reduce # of regexps below.
+    #s/$/ /;
+    text.gsub!(/$/," ")
+    #s/^/ /;
+    text.gsub!(/^/," ")
+
+    # possessive or close-single-quote
+    #s/\([^\']\)\' /$1 \' /g;
+    text.gsub!(/\([^\']\)\' /,%q(\1 ' ))
+    # as in it's, I'm, we'd
+    #s/\'([smd]) / \'$1 /ig;
+    text.gsub!(/\'([smd]) /i,%q( '\1 ))
+    #s/\'(ll|re|ve) / \'$1 /ig;
+    text.gsub!(/\'(ll|re|ve) /i,%q( '\1 ))
+    #s/n\'t / n\'t /ig;
+    text.gsub!(/n\'t /i,"  n't ")
+
+    #s/ (can)(not) / $1 $2 /ig;
+    text.gsub!(/ (can)(not) /i,' \1 \2 ')
+    #s/ (d\')(ye) / $1 $2 /ig;
+    text.gsub!(/ (d\')(ye) /i,' \1 \2 ')
+    #s/ (gim)(me) / $1 $2 /ig;
+    text.gsub!(/ (gim)(me) /i,' \1 \2 ')
+    #s/ (gon)(na) / $1 $2 /ig;
+    text.gsub!(/ (gon)(na) /i,' \1 \2 ')
+    #s/ (got)(ta) / $1 $2 /ig;
+    text.gsub!(/ (got)(ta) /i,' \1 \2 ')
+    #s/ (lem)(me) / $1 $2 /ig;
+    text.gsub!(/ (lem)(me) /i,' \1 \2 ')
+    #s/ (more)(\'n) / $1 $2 /ig;
+    text.gsub!(/ (more)(\'n) /i,' \1 \2 ')
+    #s/ (\'t)(is|was) / $1 $2 /ig;
+    text.gsub!(/ (\'t)(is|was) /i,' \1 \2 ')
+    #s/ (wan)(na) / $1 $2 /ig;
+    text.gsub!(/ (wan)(na) /i,' \1 \2 ')
+    text.split(/\s/)
+
+  end
+
 end

data/lib/treat/processors/tokenizers/ptb.rb

@@ -0,0 +1,81 @@
+# A native rule-basd tokenizer based on the one
+# developped by Robert Macyntyre in 1995 for the Penn
+# Treebank project. This tokenizer follows the
+# conventions used by the Penn Treebank.
+#
+# Original script:
+# http://www.cis.upenn.edu/~treebank/tokenizer.sed
+#
+# Copyright (c) 2004 UTIYAMA Masao <mutiyama@nict.go.jp>
+# All rights reserved. This program is free software;
+# you can redistribute it and/or modify it under the
+# same terms as Ruby itself.
+module Treat::Processors::Tokenizers::PTB
+
+  require 'treat/helpers/decimal_point_escaper'
+  
+  # Tokenize the entity using a native rule-based algorithm.
+  def self.tokenize(entity, options = {})
+    
+    entity.check_hasnt_children
+    
+    if entity.has_children?
+      raise Treat::Exception,
+      "Cannot tokenize an #{entity.class} " +
+      "that already has children."
+    end
+    chunks = split(entity.to_s)
+    chunks.each do |chunk|
+      next if chunk =~ /([[:space:]]+)/
+      entity << Treat::Entities::Token.from_string(chunk)
+    end
+  end
+  
+  # Helper method to split the string into tokens.
+  def self.split(string)
+    s = " " + string + " "
+    Treat::Helpers::DecimalPointEscaper.escape!(s)
+    s.gsub!(/\s+/," ")
+    s.gsub!(/(\s+)''/,'\1"')
+    s.gsub!(/(\s+)``/,'\1"')
+    s.gsub!(/''(\s+)/,'"\1')
+    s.gsub!(/``(\s+)/,'"\1')
+    s.gsub!(/ (['`]+)([^0-9].+) /,' \1 \2 ')
+    s.gsub!(/([ (\[{<])"/,'\1 `` ')
+    s.gsub!(/\.\.\./,' ... ')
+    s.gsub!(/[,;:@\#$%&]/,' \& ')
+    s.gsub!(/([^.])([.])([\])}>"']*)[ 	]*$/,'\1 \2\3 ')
+    s.gsub!(/[?!]/,' \& ')
+    s.gsub!(/[\]\[(){}<>]/,' \& ')
+    s.gsub!(/--/,' -- ')
+    s.sub!(/$/,' ')
+    s.sub!(/^/,' ')
+    s.gsub!(/"/,' \'\' ')
+    s.gsub!(/([^'])' /,'\1 \' ')
+    s.gsub!(/'([sSmMdD]) /,' \'\1 ')
+    s.gsub!(/'ll /,' \'ll ')
+    s.gsub!(/'re /,' \'re ')
+    s.gsub!(/'ve /,' \'ve ')
+    s.gsub!(/n't /,' n\'t ')
+    s.gsub!(/'LL /,' \'LL ')
+    s.gsub!(/'RE /,' \'RE ')
+    s.gsub!(/'VE /,' \'VE ')
+    s.gsub!(/N'T /,' N\'T ')
+    s.gsub!(/ ([Cc])annot /,' \1an not ')
+    s.gsub!(/ ([Dd])'ye /,' \1\' ye ')
+    s.gsub!(/ ([Gg])imme /,' \1im me ')
+    s.gsub!(/ ([Gg])onna /,' \1on na ')
+    s.gsub!(/ ([Gg])otta /,' \1ot ta ')
+    s.gsub!(/ ([Ll])emme /,' \1em me ')
+    s.gsub!(/ ([Mm])ore'n /,' \1ore \'n ')
+    s.gsub!(/ '([Tt])is /,' \'\1 is ')
+    s.gsub!(/ '([Tt])was /,' \'\1 was ')
+    s.gsub!(/ ([Ww])anna /,' \1an na ')
+    while s.sub!(/(\s)([0-9]+) , ([0-9]+)(\s)/, '\1\2,\3\4'); end
+    s.gsub!(/\//, ' / ')
+    s.gsub!(/\s+/,' ')
+    s.strip!
+    s.split(/\s+/)
+  end
+  
+end

data/lib/treat/processors/tokenizers/punkt.rb

@@ -1,45 +1,51 @@
-module Treat
-  module Processors
-    module Tokenizers
-      # A tokenizer that was lifted from the 'punkt-segmenter'
-      # Ruby gem.
-      #
-      # This code follows the terms and conditions of Apache 
-      # License v2 (http://www.apache.org/licenses/LICENSE-2.0)
-      #
-      # Authors: Willy <willy@csse.unimelb.edu.au>
-      # (original Python port), Steven Bird
-      # <sb@csse.unimelb.edu.au> (additions),
-      # Edward Loper <edloper@gradient.cis.upenn.edu>
-      # (rewrite), Joel Nothman <jnothman@student.usyd.edu.au>
-      # (almost rewrite).
-      #
-      # Project website: https://github.com/lfcipriani/punkt-segmenter
-      class Punkt
-        SentEndChars = ['.', '?', '!']
-        ReSentEndChars = /[.?!]/
-        InternalPunctuation = [',', ':', ';']
-        ReBoundaryRealignment = /^["\')\]}]+?(?:\s+|(?=--)|$)/m
-        ReWordStart = /[^\(\"\`{\[:;&\#\*@\)}\]\-,]/
-        ReNonWordChars = /(?:[?!)\";}\]\*:@\'\({\[])/
-        ReMultiCharPunct = /(?:\-{2,}|\.{2,}|(?:\.\s){2,}\.)/
-        ReWordTokenizer = /#{ReMultiCharPunct}|(?=#{ReWordStart})\S+?(?=\s|$|#{ReNonWordChars}|#{ReMultiCharPunct}|,(?=$|\s|#{ReNonWordChars}|#{ReMultiCharPunct}))|\S/
-        RePeriodContext = /\S*#{ReSentEndChars}(?=(?<after_tok>#{ReNonWordChars}|\s+(?<next_tok>\S+)))/
-        # Tokenize the text using the algorithm lifted from
-        # the Punkt tokenizer gem.
-        #
-        # Options: none.
-        def self.tokenize(entity, options = {})
-          entity.to_s.scan(ReWordTokenizer).each do |token|
-            if SentEndChars.include?(token[-1])
-              entity << Treat::Entities::Token.from_string(token[0..-2])
-              entity << Treat::Entities::Token.from_string(token[-1..-1])
-            else
-              entity << Treat::Entities::Token.from_string(token)
-            end   
-          end
-        end
+# A tokenizer that was lifted from the 'punkt-segmenter'
+# Ruby gem.
+#
+# This code follows the terms and conditions of Apache
+# License v2 (http://www.apache.org/licenses/LICENSE-2.0)
+#
+# Authors: Willy <willy@csse.unimelb.edu.au>
+# (original Python port), Steven Bird
+# <sb@csse.unimelb.edu.au> (additions),
+# Edward Loper <edloper@gradient.cis.upenn.edu>
+# (rewrite), Joel Nothman <jnothman@student.usyd.edu.au>
+# (almost rewrite).
+#
+# Project website: https://github.com/lfcipriani/punkt-segmenter
+class Treat::Processors::Tokenizers::Punkt
+  
+  require 'treat/helpers/decimal_point_escaper'
+  
+  SentEndChars = ['.', '?', '!']
+  ReSentEndChars = /[.?!]/
+  InternalPunctuation = [',', ':', ';']
+  ReBoundaryRealignment = /^["\')\]}]+?(?:\s+|(?=--)|$)/m
+  ReWordStart = /[^\(\"\`{\[:;&\#\*@\)}\]\-,]/
+  ReNonWordChars = /(?:[?!)\";}\]\*:@\'\({\[])/
+  ReMultiCharPunct = /(?:\-{2,}|\.{2,}|(?:\.\s){2,}\.)/
+  ReWordTokenizer = /#{ReMultiCharPunct}|(?=#{ReWordStart})\S+?(?=\s|$|#{ReNonWordChars}|#{ReMultiCharPunct}|,(?=$|\s|#{ReNonWordChars}|#{ReMultiCharPunct}))|\S/
+  RePeriodContext = /\S*#{ReSentEndChars}(?=(?<after_tok>#{ReNonWordChars}|\s+(?<next_tok>\S+)))/
+  
+  # Tokenize the text using the algorithm lifted from
+  # the Punkt tokenizer gem.
+  #
+  # Options: none.
+  def self.tokenize(entity, options = {})
+    
+    entity.check_hasnt_children
+    
+    s = entity.to_s
+    Treat::Helpers::DecimalPointEscaper.escape!(s)
+    
+    s.scan(ReWordTokenizer).each do |token|
+      if SentEndChars.include?(token[-1])
+        entity << Treat::Entities::Token.from_string(token[0..-2])
+        entity << Treat::Entities::Token.from_string(token[-1..-1])
+      else
+        entity << Treat::Entities::Token.from_string(token)
       end
     end
+    
   end
-end
+  
+end