treat 0.1.4 → 0.2.0

data/lib/treat/processors/segmenters/stanford.rb

@@ -1,49 +1,50 @@
 module Treat
   module Processors
     module Segmenters
-      # A wrapper for the sentence splitter supplied by 
+      # A wrapper for the sentence splitter supplied by
       # the Stanford parser.
       class Stanford
-        # Require the Ruby-Java bridge.
-        silence_warnings do
-          require 'rjb'
-          jar = "#{Treat.bin}/stanford-parser*/stanford-parser*.jar"
-          jars = Dir.glob(jar)
-          if jars.empty? || !File.readable?(jars[0])
-            raise "Could not find stanford parser JAR file (lookin in #{jar})."+
-            " You may need to manually download the JAR files and/or set Treat.bin."
-          end
-          ::Rjb::load(jars[0])
-          DocumentPreprocessor =
-          ::Rjb::import('edu.stanford.nlp.process.DocumentPreprocessor')
-          StringReader = ::Rjb::import('java.io.StringReader')
-        end
-        # Segment sentences using the sentence splitter supplied by 
-        # the Stanford parser.
+        require 'stanford-core-nlp'
+        DefaultOptions = { 
+          :silence => false, 
+          :log_to_file => false, 
+          :also_tokenize => false 
+        }
+        # Segment sentences using the sentence splitter supplied by
+        # the Stanford parser. By default, this segmenter also adds
+        # the tokens as children of the sentences.
+        #
+        # Options:
+        # - (Boolean) :also_tokenize - Whether to also add the tokens
+        # as children of the sentence.
+        # - (String) :log_to_file => a filename to log output to
+        # instead of displaying it.
+        # - (String) :silence => send 
         def self.segment(entity, options = {})
-          sr = StringReader.new(entity.to_s)
-          sit = DocumentPreprocessor.new(sr).iterator
-          while sit.has_next
-            str = sit.next.to_string
-            str.gsub!(', ', ' ')              # Fix - find better way to implode.
-            str.gsub!(' \'s', '\'s')
-            str.gsub!(' .', '.')
-            str.gsub!(' ,', ',')
-            str.gsub!(' ;', ';')
-            str.gsub!(/-[A-Z]{3}-/, '')
-            str = str[1..-2]
-            sentence = Entities::Entity.from_string(str)
-            if options[:tokenize] == true
-              tit = s.iterator
-              while tit.has_next
-                w = tit.next.word
-                next if w[0] == '-' && w[-1] == '-'
-                sentence << Entities::Entity.from_string(w)
+          options = DefaultOptions.merge(options)
+          options[:log_to_file] = '/dev/null' if options[:silence]
+          if options[:log_to_file]
+            ::StanfordCoreNLP.log_file = options[:log_to_file] 
+          end
+          options = DefaultOptions.merge(options)
+          pipeline =  ::StanfordCoreNLP.load(:tokenize, :ssplit)
+          text = ::StanfordCoreNLP::Text.new(entity.to_s)
+          pipeline.annotate(text)
+          text.get(:sentences).each do |sentence|
+            s = Treat::Entities::Sentence.from_string(sentence.to_s, true)
+            entity << s
+            if options[:also_tokenize]
+              sentence.get(:tokens).each do |token|
+                t = Treat::Entities::Phrase.from_string(token.value)
+                s << t
+                t.set :character_offset_begin, 
+                token.get(:character_offset_begin)
+                
+                t.set :character_offset_end, 
+                token.get(:character_offset_end)
               end
             end
-            entity << sentence
           end
-          entity
         end
       end
     end

data/lib/treat/processors/segmenters/tactful.rb

@@ -15,7 +15,7 @@ module Treat
       class Tactful
         # Require the 'tactful_tokenizer' gem.
         silence_warnings { require 'tactful_tokenizer' }
-        # Somewhere in the depths of the code this is defined...
+        # Remove function definition 'tactful_tokenizer' by gem.
         String.class_eval { undef :tokenize }
         # Keep only one copy of the segmenter.
         @@segmenter = nil
@@ -25,11 +25,12 @@ module Treat
         # Options: none.
         def self.segment(entity, options = {})
           @@segmenter ||= TactfulTokenizer::Model.new
-          sentences = @@segmenter.tokenize_text(entity.to_s)
+          s = entity.to_s
+          s.gsub!(/([^\.\?!]\.|\!|\?)([^\s])/) { $1 + ' ' + $2 }
+          sentences = @@segmenter.tokenize_text(s)
           sentences.each do |sentence|
-            entity << Entities::Entity.from_string(sentence)
+            entity << Entities::Phrase.from_string(sentence)
           end
-          entity
         end
       end
     end

data/lib/treat/processors/tokenizers/macintyre.rb

@@ -16,13 +16,15 @@ module Treat
       class Macintyre
         # Tokenize the entity using a native rule-based algorithm.
         def self.tokenize(entity, options = {})
-          raise 'Error' if entity.has_children?
-          chunks = self.split(entity.to_s)
+          if entity.has_children?
+            raise Treat::Exception,
+              'Cannot tokenize a Phrase that already has children.'
+          end
+          chunks = split(entity.to_s)
           chunks.each do |chunk|
             next if chunk =~ /([[:space:]]+)/
-            entity << Treat::Entities::Entity.from_string(chunk)
+            entity << Treat::Entities::Token.from_string(chunk)
           end 
-          entity
         end
         # Helper method to split the string into tokens.
         def self.split(string)
@@ -63,8 +65,7 @@ module Treat
           s.gsub!(/ '([Tt])is /,' \'\1 is ')
           s.gsub!(/ '([Tt])was /,' \'\1 was ')
           s.gsub!(/ ([Ww])anna /,' \1an na ')
-          while s.sub!(/(\s)([0-9]+) , ([0-9]+)(\s)/, '\1\2,\3\4')
-          end
+          while s.sub!(/(\s)([0-9]+) , ([0-9]+)(\s)/, '\1\2,\3\4'); end
           s.gsub!(/\//, ' / ')
           s.gsub!(/\s+/,' ')
           s.strip!

data/lib/treat/processors/tokenizers/multilingual.rb

@@ -14,16 +14,15 @@ module Treat
         # :language => (Symbol) Force a language for the tokenizer.
         def self.tokenize(entity, options = {})
           lang = options[:language] ? options[:language] : entity.language
-          lang = Treat::Languages.find(lang, 1)    
+          lang = Treat::Languages.code(lang, 1)    
           if @@tokenizers[lang].nil?
             @@tokenizers[lang] = ::Tokenizer::Tokenizer.new(lang)
           end
           tokens = @@tokenizers[lang].tokenize(entity.to_s)
           tokens.each do |token|
             next if token =~ /([[:space:]]+)/
-            entity << Treat::Entities::Entity.from_string(token)
+            entity << Treat::Entities::Token.from_string(token)
           end
-          entity
         end
       end
     end

data/lib/treat/processors/tokenizers/perl.rb

@@ -85,9 +85,9 @@ module Treat
           #s/ (wan)(na) / $1 $2 /ig;
           text.gsub!(/ (wan)(na) /i,' \1 \2 ')
           tokens = text.split(/\s/)
-          tokens.each do |token|
+          tokens[1..-1].each do |token|
             next if token =~ /([[:space:]]+)/
-            entity << Treat::Entities::Entity.from_string(token)
+            entity << Treat::Entities::Token.from_string(token)
           end
         end
       end

data/lib/treat/processors/tokenizers/punkt.rb

@@ -31,9 +31,13 @@ module Treat
         # Options: none.
         def self.tokenize(entity, options = {})
           entity.to_s.scan(ReWordTokenizer).each do |token|
-            entity << Treat::Entities::Entity.from_string(token)
+            if SentEndChars.include?(token[-1])
+              entity << Treat::Entities::Token.from_string(token[0..-2])
+              entity << Treat::Entities::Token.from_string(token[-1..-1])
+            else
+              entity << Treat::Entities::Token.from_string(token)
+            end   
           end
-          entity
         end
       end
     end

data/lib/treat/processors/tokenizers/stanford.rb

@@ -4,34 +4,35 @@ module Treat
       # A wrapper for the Stanford parser's Penn-Treebank
       # style tokenizer.
       class Stanford
-        # Require the Ruby-Java bridge.
-        silence_warnings do
-          require 'rjb'
-          # Load the Stanford Parser Java files.
-          jar = "#{Treat.bin}/stanford-parser/stanford-parser.jar"
-          jars = Dir.glob(jar)
-          if jars.empty? || !File.readable?(jars[0])
-            raise "Could not find stanford parser JAR file (looking in #{jar})."+
-            " You may need to manually download the JAR files and/or set Treat.bin."
-          end
-          ::Rjb::load(jars[0])
-          # Load the Stanford Parser classes.
-          PTBTokenizer = ::Rjb::import('edu.stanford.nlp.process.PTBTokenizer')
-          CoreLabelTokenFactory = ::Rjb::import('edu.stanford.nlp.process.CoreLabelTokenFactory')
-          StringReader = ::Rjb::import('java.io.StringReader')
-        end
+        require 'stanford-core-nlp'
+        DefaultOptions = {
+          :silence => false, 
+          :log_to_file => nil
+        }
+        @@tokenizer = nil
         # Tokenize the entity using a Penn-Treebank style tokenizer
         # included with the Stanford Parser.
+        #
+        # Options:
+        # - (String) :log_to_file => a filename to log output to
+        # instead of displaying it.
         def self.tokenize(entity, options = {})
-          ptbt = PTBTokenizer.new(
-            StringReader.new(entity.to_s),
-            CoreLabelTokenFactory.new, '')
-          while ptbt.has_next
-            w = ptbt.next.word
-            next if w[0] == '-' && w[-1] == '-'
-            entity << Treat::Entities::Entity.from_string(w)
+          options = DefaultOptions.merge(options)
+          options[:log_to_file] = '/dev/null' if options[:silence]
+          if options[:log_to_file]
+            ::StanfordCoreNLP.log_file = options[:log_to_file] 
+          end
+          @@tokenizer ||= ::StanfordCoreNLP.load(:tokenize)
+          text = ::StanfordCoreNLP::Text.new(entity.to_s)
+          @@tokenizer.annotate(text)
+          text.get(:tokens).each do |token|
+            t = Treat::Entities::Token.from_string(token.value)
+            entity << t
+            t.set :character_offset_begin, 
+            token.get(:character_offset_begin)
+            t.set :character_offset_end, 
+            token.get(:character_offset_end)
           end
-          entity
         end
       end
     end

data/lib/treat/processors/tokenizers/tactful.rb

@@ -49,9 +49,8 @@ module Treat
             s.gsub!(rules[0], rules[1])
           end
           s.split(' ').each do |token|
-            entity << Entities::Entity.from_string(token)
+            entity << Entities::Token.from_string(token)
           end
-          entity
         end
       end
     end

data/lib/treat/proxies.rb

@@ -7,7 +7,7 @@ module Treat
       # Build the entity corresponding to the proxied
       # object and send the method call to the entity.
       def method_missing(sym, *args, &block)
-        if Treat::Categories.have_method?(sym)
+        if sym == :do || Treat::Categories.lookup(sym)
           to_entity.send(sym, *args)
         else
           super(sym, *args, &block)
@@ -20,10 +20,6 @@ module Treat
     # Install Treat functions on String objects.
     module String
       include Treat::Proxies::Proxy
-      # Save the string to the specified file.
-      def save(file)
-        File.open(file, 'w') { |f| f.write(self) }
-      end
       # Return the entity corresponding to the string.
       def to_entity
         Treat::Entities::Entity.from_string(self.to_s)  
@@ -34,40 +30,11 @@ module Treat
       include Treat::Proxies::Proxy
       # Return the entity corresponding to the number.
       def to_entity(builder = nil)
-        Treat::Entities::Entity.from_numeric(self)
-      end
-    end
-    # Install Treat functions on Array objects.
-    module Array
-      include Treat::Proxies::Proxy
-      # The behaviour of this proxy is special:
-      # if a Treat function is called on an array,
-      # the function will be called on each element
-      # of the array and a new array with the 
-      # results will be returned.
-      def method_missing(sym, *args, &block)
-        if Category.has_method?(sym)
-          array = []
-          each do |element|
-            if element.is_a? Treat::Entities::Entity
-              array << element.send(sym, *args)
-            else
-              unless [Numeric, String, Array].include?(element.class)
-                raise Treat::Exception "Cannot convert object with type " +
-                "#{element.class} into an entity."
-              end
-              array << element.to_entity.send(sym, *args)
-            end
-          end
-          array
-        else
-          super(sym, *args, &block)
-        end
+        Treat::Entities::Number.from_numeric(self)
       end
     end
     # Include the proxies in the core classes.
     ::String.class_eval { include Treat::Proxies::String }
     ::Numeric.class_eval { include Treat::Proxies::Numeric }
-    ::Array.class_eval { include Treat::Proxies::Array }
   end
 end

data/lib/treat/registrable.rb

@@ -1,32 +1,27 @@
 module Treat
   module Registrable
-    # Registers a token in the @token_registry
-    # hash in the root node.
+    # Registers a token in the @token_registry hash.
     def register_token(token)
-      if is_root? || type == :document
-        @token_registry ||= {value: {}, id: {}}
-        @token_registry[:id][token.id] = token
-        @token_registry[:value][token.to_s] ||= []
-        @token_registry[:value][token.to_s] << token
-        if has_parent? && type == :document
-          @parent.register_token(token)
-        end
-      else
-        @parent.register_token(token)
-      end
+      @token_registry ||= {:value => {}, :id => {}}
+      @token_registry[:id][token.id] = token
+      v = token.to_s.downcase
+      @token_registry[:value][v] ||= []
+      @token_registry[:value][v] << token
+      @parent.register_token(token) if has_parent?
     end
-    # Find the token registry, which is 
-    # always in the root node.
+    # Find the token registry, by default the one
+    # in the root node.
     def token_registry(type = nil)
-      if self.type == type
-        @token_registry ||= {value: {}, id: {}}
+      if (type == nil && is_root?) || type == self.type
+        @token_registry ||= {:value => {}, :id => {}}
         return @token_registry
-      end
-      if has_parent?
-        @parent.token_registry(type)
       else
-        @token_registry ||= {value: {}, id: {}}
-        @token_registry
+        if has_parent?
+          @parent.token_registry(type)
+        else
+          @token_registry ||= {:value => {}, :id => {}}
+          @token_registry
+        end
       end
     end
   end

data/lib/treat/sugar.rb

@@ -5,9 +5,9 @@ module Treat
   # Treat::Entities::Word can now be referred to as simply 'Word'.
   module Sugar
     # Installs syntactic sugar.
-    def edulcorate
-      return if @@edulcorated
-      @@edulcorated = true
+    def sweeten!
+      return if @@sweetened
+      @@sweetened = true
       each_entity_class do |type, klass|
         unless type == :Symbol
           Object.class_eval do
@@ -18,11 +18,11 @@ module Treat
         end
       end
     end
-    alias :sweeten :edulcorate
+
     # Uninstalls syntactic sugar.
-    def unedulcorate
-      return unless @@edulcorated
-      @@edulcorated = false
+    def unsweeten!
+      return unless @@sweetened
+      @@sweetened = false
       each_entity_class do |type, klass| 
         unless type == :Symbol
           Object.class_eval do
@@ -31,17 +31,17 @@ module Treat
         end
       end
     end
-    alias :unsweeten :unedulcorate
+
     # Boolean - whether syntactic sugar is
     # enabled or not.
-    def edulcorated?; @@edulcorated; end
+    def sweetened?; @@sweetened; end
     # Syntactic sugar is disabled by default.
-    @@edulcorated = false
+    @@sweetened = false
     private
     # Helper method, yields each entity type and class.
     def each_entity_class
       Treat::Entities.list.each do |entity_type|
-        type = :"#{cc(entity_type)}"
+        type = cc(entity_type).intern
         klass = Treat::Entities.const_get(type, klass)
         yield type, klass
       end

data/lib/treat/tree.rb

@@ -1,10 +1,10 @@
 module Treat
   # This module provides an abstract tree structure with
-  # nodes having an id, a value, children, features and edges.
+  # nodes having an id, a value, children, features and dependencies.
   module Tree
     # This class models the nodes for an N-ary tree data structue
     # with unique identifiers, text value, children, features
-    # (annotations) and edges. 
+    # (annotations) and dependencies. 
     #
     # This class was tightly based on the 'rubytree' gem.
     # RubyTree is licensed under the BSD license and can
@@ -24,28 +24,30 @@ module Treat
       attr_reader :children
       # A hash containing the features of this node.
       attr_accessor :features
-      # A hash containing the edges that link this 
+      # An array containing the dependencies that link this 
       # node to other nodes.
-      attr_accessor :edges
+      attr_accessor :dependencies
+      # A struct for dependencies.
+      Struct.new('Dependency', :target, :type, :directed, :direction)
       # The parent of the node.
       attr_accessor :parent
       # Initialize the node with its value and id.
       # Setup containers for the children, features
-      # and edges of this node.
+      # and dependencies of this node.
       def initialize(value, id = nil)
         @parent = nil
         @value, @id = value, id
         @children = []
         @children_hash = {}
         @features = {}
-        @edges = {}
+        @dependencies = []
       end
-      # Boolean - does the node have edges?
-      def has_edges?; !@edges.empty?; end
+      # Boolean - does the node have dependencies?
+      def has_dependencies?; !(@dependencies.size == 0); end
       # Boolean - does the node have children?
-      def has_children?; !@children.empty?; end
+      def has_children?; !(@children.size == 0); end
       # Boolean - does the node have features?
-      def has_features?; !@features.empty?; end
+      def has_features?; !(@features.size == 0); end
       # Boolean - does the node have a parent?
       def has_parent?; !@parent.nil?; end
       # Boolean - does the node not have a parent?
@@ -132,6 +134,10 @@ module Treat
         @features ||= {}
         @features[feature] = value
       end
+      # Unset a feature.
+      def unset(feature)
+        @features.delete(feature)
+      end
       # Return the depth of this node in the tree.
       def depth
         return 0 if is_root?
@@ -139,23 +145,26 @@ module Treat
       end
       # Does the entity have a feature ?
       def has_feature?(feature)
-        @features.has_key?(feature) ||
-        [:id, :value, :children, :edges].include?(feature)
+        (@features.has_key?(feature) && 
+        !@features[feature].nil?) ||
+        [:id, :value, :children, :dependencies].include?(feature)
       end
       alias :has? :has_feature?
       # Link this node to the target node with
-      # the supplied edge type.
-      def associate(id_or_node, edge_type = nil)
-        if id_or_node.is_a? Treat::Tree::Node
+      # the supplied dependency type.
+      def link(id_or_node, type = nil, directed = true, direction = 1)
+        if id_or_node.is_a?(Treat::Tree::Node)
           id = root.find(id_or_node).id
         else
           id = id_or_node
         end
-        @edges[id] = edge_type if id
+        @dependencies.each { |d| return if d.target == id }
+        @dependencies << 
+          Struct::Dependency.new(id, type, directed, direction)
       end
       # Find the node in the tree with the given id.
       def find(id_or_node)
-        if id_or_node.is_a? self.class
+        if id_or_node.is_a?(Treat::Tree::Node)
           id = id_or_node.id
         else
           id = id_or_node
@@ -165,6 +174,7 @@ module Treat
           r = child.find(id)
           return r if r.is_a? Tree::Node
         end
+        nil
       end
       # Find the root of the tree within which
       # this node is contained.