treat 0.1.1

data/lib/treat/processors/segmenters/stanford.rb

@@ -0,0 +1,45 @@
+module Treat
+  module Processors
+    module Segmenters
+      class Stanford
+        # Require the Ruby-Java bridge.
+        silently do
+          require 'rjb'
+          jar = "#{Treat.bin}/stanford_parser/stanford-parser.jar"
+          unless File.readable?(jar)
+            raise "Could not find stanford parser JAR file in #{jar}."+
+            " You may need to set Treat.bin to a custom value."
+          end
+          DocumentPreprocessor =
+          ::Rjb::import('edu.stanford.nlp.process.DocumentPreprocessor')
+          StringReader = ::Rjb::import('java.io.StringReader')
+        end
+        def self.segment(entity, options = {})
+          sr = StringReader.new(entity.to_s)
+          sit = DocumentPreprocessor.new(sr).iterator
+          while sit.has_next
+            str = sit.next.to_string
+            str.gsub!(', ', ' ')              # Fix - find better way to implode.
+            str.gsub!(' \'s', '\'s')
+            str.gsub!(' .', '.')
+            str.gsub!(' ,', ',')
+            str.gsub!(' ;', ';')
+            str.gsub!(/-[A-Z]{3}-/, '')
+            str = str[1..-2]
+            sentence = Entities::Entity.from_string(str)
+            if options[:tokenize] == true
+              tit = s.iterator
+              while tit.has_next
+                w = tit.next.word
+                next if w[0] == '-' && w[-1] == '-'
+                sentence << Entities::Entity.from_string(w)
+              end
+            end
+            entity << sentence
+          end
+          entity
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/segmenters/tactful.rb

@@ -0,0 +1,34 @@
+module Treat
+  module Processors
+    module Segmenters
+      # An adapter for the 'tactful_tokenizer' gem, which
+      # detects sentence boundaries (the name is a misnomer;
+      # it isn't a tokenizer, but a sentence boundary detector).
+      # It uses a Naive Bayesian statistical model, and is
+      # based on Splitta, but has support for ‘?’ and ‘!’
+      # as well as primitive handling of XHTML markup.
+      #
+      # Project website:
+      class Tactful
+        # Require the 'tactful_tokenizer' gem.
+        silently { require 'tactful_tokenizer' }
+        # Somewhere in the depths of the code this is defined...
+        String.class_eval { undef :tokenize }
+        # Keep only one copy of the segmenter.
+        @@segmenter = nil
+        # Segment a text or zone into sentences
+        # using the 'tactful_tokenizer' gem.
+        #
+        # Options: none.
+        def self.segment(entity, options = {})
+          @@segmenter ||= TactfulTokenizer::Model.new
+          sentences = @@segmenter.tokenize_text(entity.to_s)
+          sentences.each do |sentence|
+            entity << Entities::Entity.from_string(sentence)
+          end
+          entity
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/tokenizers/macintyre.rb

@@ -0,0 +1,76 @@
+module Treat
+  module Processors
+    module Tokenizers
+      # A native rule-basd tokenizer based on the one
+      # developped by Robert Macyntyre in 1995 for the Penn
+      # Treebank project. This tokenizer follows the 
+      # conventions used by the Penn Treebank.
+      # 
+      # Original script: 
+      # http://www.cis.upenn.edu/~treebank/tokenizer.sed
+      #
+      # Copyright (c) 2004 UTIYAMA Masao <mutiyama@nict.go.jp>
+      # All rights reserved. This program is free software; 
+      # you can redistribute it and/or modify it under the 
+      # same terms as Ruby itself.
+      class Macintyre
+        # Tokenize the entity using a native rule-based algorithm.
+        def self.tokenize(entity, options = {})
+          raise 'Error' if entity.has_children?
+          chunks = self.split(entity.to_s)
+          chunks.each do |chunk|
+            next if chunk =~ /([[:space:]]+)/
+            entity << Treat::Entities::Entity.from_string(chunk)
+          end 
+          entity
+        end
+        # Helper method to split the string into tokens.
+        def self.split(string)
+          s = " " + string + " "
+          s.gsub!(/\s+/," ")
+          s.gsub!(/(\s+)''/,'\1"')
+          s.gsub!(/(\s+)``/,'\1"')
+          s.gsub!(/''(\s+)/,'"\1')
+          s.gsub!(/``(\s+)/,'"\1')
+          s.gsub!(/ (['`]+)([^0-9].+) /,' \1 \2 ')
+          s.gsub!(/([ (\[{<])"/,'\1 `` ')
+          s.gsub!(/\.\.\./,' ... ')
+          s.gsub!(/[,;:@\#$%&]/,' \& ')
+          s.gsub!(/([^.])([.])([\])}>"']*)[ 	]*$/,'\1 \2\3 ')
+          s.gsub!(/[?!]/,' \& ')
+          s.gsub!(/[\]\[(){}<>]/,' \& ')
+          s.gsub!(/--/,' -- ')
+          s.sub!(/$/,' ')
+          s.sub!(/^/,' ')
+          s.gsub!(/"/,' \'\' ')
+          s.gsub!(/([^'])' /,'\1 \' ')
+          s.gsub!(/'([sSmMdD]) /,' \'\1 ')
+          s.gsub!(/'ll /,' \'ll ')
+          s.gsub!(/'re /,' \'re ')
+          s.gsub!(/'ve /,' \'ve ')
+          s.gsub!(/n't /,' n\'t ')
+          s.gsub!(/'LL /,' \'LL ')
+          s.gsub!(/'RE /,' \'RE ')
+          s.gsub!(/'VE /,' \'VE ')
+          s.gsub!(/N'T /,' N\'T ')
+          s.gsub!(/ ([Cc])annot /,' \1an not ')
+          s.gsub!(/ ([Dd])'ye /,' \1\' ye ')
+          s.gsub!(/ ([Gg])imme /,' \1im me ')
+          s.gsub!(/ ([Gg])onna /,' \1on na ')
+          s.gsub!(/ ([Gg])otta /,' \1ot ta ')
+          s.gsub!(/ ([Ll])emme /,' \1em me ')
+          s.gsub!(/ ([Mm])ore'n /,' \1ore \'n ')
+          s.gsub!(/ '([Tt])is /,' \'\1 is ')
+          s.gsub!(/ '([Tt])was /,' \'\1 was ')
+          s.gsub!(/ ([Ww])anna /,' \1an na ')
+          while s.sub!(/(\s)([0-9]+) , ([0-9]+)(\s)/, '\1\2,\3\4')
+          end
+          s.gsub!(/\//, ' / ')
+          s.gsub!(/\s+/,' ')
+          s.strip!
+          s.split(/\s+/)
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/tokenizers/multilingual.rb

@@ -0,0 +1,31 @@
+module Treat
+  module Processors
+    module Tokenizers
+      # An adapter for the 'tokenizer' gem, which performs
+      # rule-based tokenizing of texts in English, German
+      # or French.
+      class Multilingual
+        # Hold one tokenizer per language.
+        @@tokenizers = {}
+        # Require the 'tokenizer' gem.
+        silently { require 'tokenizer' }
+        # Perform the tokenization of English, German or French text.
+        # Options:
+        # :language => (Symbol) Force a language for the tokenizer.
+        def self.tokenize(entity, options = {})
+          lang = options[:language] ? options[:language] : entity.language
+          lang = Treat::Resources::Languages.find(lang, 1)    
+          if @@tokenizers[lang].nil?
+            @@tokenizers[lang] = ::Tokenizer::Tokenizer.new(lang)
+          end
+          tokens = @@tokenizers[lang].tokenize(entity.to_s)
+          tokens.each do |token|
+            next if token =~ /([[:space:]]+)/
+            entity << Treat::Entities::Entity.from_string(token)
+          end
+          entity
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/tokenizers/perl.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+module Treat
+  module Processors
+    module Tokenizers
+      # Tokenize the entity using a native rule-based algorithm.
+      # This tokenizer is a port from an unknown Perl module,
+      # which I have lifted from the 'rbtagger' gem.
+      #
+      # Author: Todd A. Fisher
+      # This code is free to use under the terms of the MIT license.
+      #
+      # Original project website:
+      # https://github.com/taf2/rb-brill-tagger
+      class Perl
+        # Tokenize the entity using a native rule-based algorithm.
+        # Options: none.
+        def self.tokenize(entity, options = {})
+          # Normalize all whitespace
+          text = entity.to_s.gsub(/\s+/,' ')
+          # Translate some common extended ascii characters to quotes
+          text.gsub!(/‘/,'`')
+          text.gsub!(/’/,"'")
+          text.gsub!(/“/,"``")
+          text.gsub!(/”/,"''")
+          # Attempt to get correct directional quotes
+          # s{\"\b} { `` }g;
+          text.gsub!(/\"\b/,' `` ')
+          # s{\b\"} { '' }g;
+          text.gsub!(/\b\"/," '' ")
+          #s{\"(?=\s)} { '' }g;
+          text.gsub!(/\"(?=\s)/," '' ")
+          #s{\"} { `` }g;
+          text.gsub!(/\"(?=\s)/," `` ")
+          # Isolate ellipses
+          # s{\.\.\.}   { ... }g;
+          text.gsub!(/\.\.\./,' ... ')
+          # Isolate any embedded punctuation chars
+          #   s{([,;:\@\#\$\%&])} { $1 }g;
+          text.gsub!(/([,;:\@\#\$\%&])/, ' \1 ')
+          # Assume sentence tokenization has been done first, so split FINAL
+          # periods only.
+          # s/ ([^.]) \.  ([\]\)\}\>\"\']*) [ \t]* $ /$1 .$2 /gx;
+          text.gsub!(/ ([^.]) \.  ([\]\)\}\>\"\']*) [ \t]* $ /x, '\1 .\2 ')
+          # however, we may as well split ALL question marks and exclamation points,
+          # since they shouldn't have the abbrev.-marker ambiguity problem
+          #s{([?!])} { $1 }g;
+          text.gsub!(/([?!])/, ' \1 ')
+          # parentheses, brackets, etc.
+          #s{([\]\[\(\)\{\}\<\>])} { $1 }g;
+          text.gsub!(/([\]\[\(\)\{\}\<\>])/,' \1 ')
+          #s/(-{2,})/ $1 /g;
+          text.gsub!(/(-{2,})/,' \1 ')
+          # Add a space to the beginning and end of each line, to reduce
+          # necessary number of regexps below.
+          #s/$/ /;
+          text.gsub!(/$/," ")
+          #s/^/ /;
+          text.gsub!(/^/," ")
+          # possessive or close-single-quote
+          #s/\([^\']\)\' /$1 \' /g;
+          text.gsub!(/\([^\']\)\' /,%q(\1 ' ))
+          # as in it's, I'm, we'd
+          #s/\'([smd]) / \'$1 /ig;
+          text.gsub!(/\'([smd]) /i,%q( '\1 ))
+          #s/\'(ll|re|ve) / \'$1 /ig;
+          text.gsub!(/\'(ll|re|ve) /i,%q( '\1 ))
+          #s/n\'t / n\'t /ig;
+          text.gsub!(/n\'t /i,"  n't ")
+          #s/ (can)(not) / $1 $2 /ig;
+          text.gsub!(/ (can)(not) /i,' \1 \2 ')
+          #s/ (d\')(ye) / $1 $2 /ig;
+          text.gsub!(/ (d\')(ye) /i,' \1 \2 ')
+          #s/ (gim)(me) / $1 $2 /ig;
+          text.gsub!(/ (gim)(me) /i,' \1 \2 ')
+          #s/ (gon)(na) / $1 $2 /ig;
+          text.gsub!(/ (gon)(na) /i,' \1 \2 ')
+          #s/ (got)(ta) / $1 $2 /ig;
+          text.gsub!(/ (got)(ta) /i,' \1 \2 ')
+          #s/ (lem)(me) / $1 $2 /ig;
+          text.gsub!(/ (lem)(me) /i,' \1 \2 ')
+          #s/ (more)(\'n) / $1 $2 /ig;
+          text.gsub!(/ (more)(\'n) /i,' \1 \2 ')
+          #s/ (\'t)(is|was) / $1 $2 /ig;
+          text.gsub!(/ (\'t)(is|was) /i,' \1 \2 ')
+          #s/ (wan)(na) / $1 $2 /ig;
+          text.gsub!(/ (wan)(na) /i,' \1 \2 ')
+          tokens = text.split(/\s/)
+          tokens.each do |token|
+            next if token =~ /([[:space:]]+)/
+            entity << Treat::Entities::Entity.from_string(token)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/tokenizers/punkt.rb

@@ -0,0 +1,42 @@
+module Treat
+  module Processors
+    module Tokenizers
+      # A tokenizer that was lifted from the 'punkt-segmenter'
+      # Ruby gem.
+      #
+      # This code follows the terms and conditions of Apache 
+      # License v2 (http://www.apache.org/licenses/LICENSE-2.0)
+      #
+      # Authors: Willy <willy@csse.unimelb.edu.au>
+      # (original Python port), Steven Bird
+      # <sb@csse.unimelb.edu.au> (additions),
+      # Edward Loper <edloper@gradient.cis.upenn.edu>
+      # (rewrite), Joel Nothman <jnothman@student.usyd.edu.au>
+      # (almost rewrite).
+      #
+      # Project website: https://github.com/lfcipriani/punkt-segmenter
+      class Punkt
+        SentEndChars = ['.', '?', '!']
+        ReSentEndChars = /[.?!]/
+        InternalPunctuation = [',', ':', ';']
+        ReBoundaryRealignment = /^["\')\]}]+?(?:\s+|(?=--)|$)/m
+        ReWordStart = /[^\(\"\`{\[:;&\#\*@\)}\]\-,]/
+        ReNonWordChars = /(?:[?!)\";}\]\*:@\'\({\[])/
+        ReMultiCharPunct = /(?:\-{2,}|\.{2,}|(?:\.\s){2,}\.)/
+        ReWordTokenizer = /#{ReMultiCharPunct}|(?=#{ReWordStart})\S+?(?=\s|$|#{ReNonWordChars}|#{ReMultiCharPunct}|,(?=$|\s|#{ReNonWordChars}|#{ReMultiCharPunct}))|\S/
+        RePeriodContext = /\S*#{ReSentEndChars}(?=(?<after_tok>#{ReNonWordChars}|\s+(?<next_tok>\S+)))/
+        # Tokenize the text using the algorithm lifted from
+        # the Punkt tokenizer.
+        #
+        # Options: none.
+        def self.tokenize(entity, options = {})
+          entity.to_s.scan(ReWordTokenizer).each do |token|
+            puts token
+            entity << Treat::Entities::Entity.from_string(token)
+          end
+          entity
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/tokenizers/stanford.rb

@@ -0,0 +1,33 @@
+module Treat
+  module Processors
+    module Tokenizers
+      class Stanford
+        # Require the Ruby-Java bridge.
+        silently do
+          require 'rjb'
+          # Load the Stanford Parser Java files.
+          jar = "#{Treat.bin}/stanford_parser/stanford-parser.jar"
+          unless File.readable?(jar)
+            raise "Could not find stanford parser JAR file in #{jar}."+
+            " You may need to set Treat.bin to a custom value."
+          end
+          # Load the Stanford Parser classes.
+          PTBTokenizer = ::Rjb::import('edu.stanford.nlp.process.PTBTokenizer')
+          CoreLabelTokenFactory = ::Rjb::import('edu.stanford.nlp.process.CoreLabelTokenFactory')
+          StringReader = ::Rjb::import('java.io.StringReader')
+        end
+        def self.tokenize(entity, options = {})
+          ptbt = PTBTokenizer.new(
+            StringReader.new(entity.to_s),
+            CoreLabelTokenFactory.new, '')
+          while ptbt.has_next
+            w = ptbt.next.word
+            next if w[0] == '-' && w[-1] == '-'
+            entity << Treat::Entities::Entity.from_string(w)
+          end
+          entity
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors/tokenizers/tactful.rb

@@ -0,0 +1,59 @@
+module Treat
+  module Processors
+    module Tokenizers
+      # A tokenizer class lifted from the 'tactful-tokenizer' gem.
+      #
+      # Copyright © 2010 Matthew Bunday. All rights reserved. 
+      # Released under the GNU GPL v3. Modified by Louis Mullie.
+      # 
+      # Project website: https://github.com/SlyShy/Tactful_Tokenizer
+      class Tactful
+        ReTokenize = [
+          # Uniform Quotes
+          [/''|``/, '"'],
+          # Separate punctuation from words.
+          [/(^|\s)(')/, '\1\2'],
+          [/(?=[\("`{\[:;&#*@\.])(.)/, '\1 '],
+          [/(.)(?=[?!\)";}\]*:@\.'])|(?=[\)}\]])(.)|(.)(?=[({\[])|((^|\s)-)(?=[^-])/, '\1 '],
+          # Treat double-hyphen as a single token.
+          [/([^-])(--+)([^-])/, '\1 \2 \3'],
+          [/(\s|^)(,)(?=(\S))/, '\1\2 '],
+          # Only separate a comma if a space follows.
+          [/(.)(,)(\s|$)/, '\1 \2\3'],
+          # Combine dots separated by whitespace to be a single token.
+          [/\.\s\.\s\./, '...'],
+          # Separate "No.6"
+          [/([\W]\.)(\d+)/, '\1 \2'],
+          # Separate words from ellipses
+          [/([^\.]|^)(\.{2,})(.?)/, '\1 \2 \3'],
+          [/(^|\s)(\.{2,})([^\.\s])/, '\1\2 \3'],
+          [/(^|\s)(\.{2,})([^\.\s])/, '\1 \2\3'],
+          ##### Some additional fixes.
+          # Fix %, $, &
+          [/(\d)%/, '\1 %'],
+          [/\$(\.?\d)/, '$ \1'],
+          [/(\W)& (\W)/, '\1&\2'],
+          [/(\W\W+)&(\W\W+)/, '\1 & \2'],
+          # Fix (n 't) -> ( n't)
+          [/n 't( |$)/, " n't\\1"],
+          [/N 'T( |$)/, " N'T\\1"],
+          # Treebank tokenizer special words
+          [/([Cc])annot/, '\1an not']
+        ]
+        # Tokenize the entity using a rule-based algorithm
+        # which has been lifted from the 'tactful-tokenizer'
+        # gem.
+        def self.tokenize(entity, options = {})
+          s = entity.to_s
+          ReTokenize.each do |rules| 
+            s.gsub!(rules[0], rules[1])
+          end
+          s.split(' ').each do |token|
+            entity << Entities::Entity.from_string(token)
+          end
+          entity
+        end
+      end
+    end
+  end
+end

data/lib/treat/proxies.rb

@@ -0,0 +1,66 @@
+module Treat
+  # Proxies install Treat functions on Rubycore classes.
+  module Proxies
+    # The module proxy provides functionanaty common
+    # to the different types of proxies.
+    module Proxy
+      def method_missing(sym, *args, &block)
+        if Categories.have_method?(sym)
+          to_entity.send(sym, *args)
+        else
+          super(sym, *args, &block)
+        end
+      end
+      def to_entity(builder = nil)
+        Treat::Entities::Unknown(self.to_s)
+      end
+    end
+    # Install Treat functions on String objects.
+    module StringProxy
+      include Proxy
+      # Save the string to the specified file.
+      def save(file)
+        File.open(file, 'w') { |f| f.write(self) }
+      end
+      # Return the entity corresponding to the string.
+      def to_entity
+        Treat::Entities::Entity.from_string(self.to_s)  
+      end
+    end
+    # Install Treat functions on Numeric objects.
+    module NumericProxy
+      include Proxy
+      # Return the entity corresponding to the number.
+      def to_entity(builder = nil)
+        Treat::Entities::Entity.from_numeric(self)
+      end
+    end
+    # Install Treat functions on Array objects.
+    module ArrayProxy
+      include Proxy
+      def method_missing(sym, *args, &block)
+        if Category.has_method?(sym)
+          array = []
+          each do |element|
+            if element.is_a? Treat::Entities::Entity
+              array << element.send(sym, *args)
+            else
+              unless [Numeric, String, Array].include?(element.class)
+                raise Treat::Exception "Cannot convert object with type " +
+                "#{element.class} into an entity."
+              end
+              array << element.to_entity.send(sym, *args)
+            end
+          end
+          array
+        else
+          super(sym, *args, &block)
+        end
+      end
+    end
+    # Include the proxies in the core classes.
+    String.class_eval { include StringProxy }
+    Numeric.class_eval { include NumericProxy }
+    Array.class_eval { include ArrayProxy }
+  end
+end