treat 0.1.4 → 0.2.0

data/lib/treat/lexicalizers/tag/tagger.rb

@@ -0,0 +1,29 @@
+module Treat
+  module Lexicalizers
+    module Tag
+      class Tagger
+        def self.tag(entity, options = {})
+          if (entity.is_a?(Treat::Entities::Sentence) ||
+             entity.is_a?(Treat::Entities::Phrase)) && 
+             !entity.has_children?
+              raise Treat::Exception,
+              "Annotator 'tag' requires processor 'tokenize'."
+          elsif entity.is_a?(Treat::Entities::Word)
+            if entity.has_parent?
+              ps = entity.parent_sentence
+              pp = entity.parent_phrase
+              if ps
+                self.tag(ps, options)
+              elsif pp
+                self.tag(pp, options)
+              end
+              return entity.features[:tag]
+            else
+              return :isolated_word
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/processors.rb

@@ -9,15 +9,15 @@ module Treat
   #   - Chunkers : split a text into zone objects.
   #   - Segmenters : split a text or zone into sentence objects.
   #   - Tokenizers : split a sentence into Token objects.
-  #   - Parsers: split a sentence into a tree of constituents
-  #     containing other constituents and Token objects, representing
+  #   - Parsers: split a sentence into a tree of phrases
+  #     containing other phrases and Token objects, representing
   #     the syntactic structure.
   module Processors
     # Chunkers split a text into zones.
     module Chunkers
       extend Group
       self.type = :transformer
-      self.targets = [:document, :zone]
+      self.targets = [:document, :section]
     end
     # Segmenters split a text or zone into sentences.
     module Segmenters
@@ -29,17 +29,17 @@ module Treat
     module Tokenizers
       extend Group
       self.type = :transformer
-      self.targets = [:document, :zone, :sentence, :constituent]
+      self.targets = [:document, :zone, :phrase]
     end
-    # Parsers split a sentence into constituent objects
+    # Parsers split a sentence into phrase objects
     # representing its syntactic structure, with the
-    # Token objects as children of the constituents.
+    # Token objects as children of the phrases.
     module Parsers
       extend Group
       self.type = :transformer
-      self.targets = [:document, :zone, :sentence, :constituent]
+      self.targets = [:document, :zone, :phrase]
     end
-    # Makes all the groups autoloadable and creates the delegators.
+    # Makes all the groups autoloadable and creates the workers.
     extend Treat::Category
   end
 end

data/lib/treat/processors/chunkers/txt.rb

@@ -2,14 +2,15 @@ module Treat
   module Processors
     module Chunkers
       # This class separates a plain text file into 
-      # zones based on a very naive analysis of the 
-      # file.
+      # zones based on an extremely naive analysis of the 
+      # file. Suprisingly, this works pretty well.
       class Txt
         # Split a document into Zone objects.
         def self.chunk(text, options = {})
           zones = text.to_s.split("\n")
           zones.each do |zone|
-            next if zone.strip == ''
+            zone.strip!
+            next if zone == ''
             if false # fix
               text << Treat::Entities::List.new(zone)
             end
@@ -19,7 +20,6 @@ module Treat
               text << Treat::Entities::Paragraph.new(zone)
             end
           end
-          text
         end
       end
     end

data/lib/treat/processors/parsers/enju.rb

@@ -6,7 +6,7 @@ module Treat
       # the parser formats it runs it through Enju, and
       # parses the XML output by Enju using the Nokogiri
       # XML reader. It creates wrappers for the sentences,
-      # syntactical constituents and  tokens that Enju identified.
+      # syntactical phrases and  tokens that Enju identified.
       #
       # Original paper:
       # Takuya Matsuzaki, Yusuke Miyao, and Jun'ichi Tsujii.
@@ -29,20 +29,15 @@ module Treat
           @@i = 0 if @@i == @@parsers.size
           @@parsers[@@i-1]
         end
-        # Parse the entity into its syntactical constituents
-        # using Enju
+        # Parse the entity into its syntactical phrases using Enju.
+        # Calls #build to initiate XML parsing.
         def self.parse(entity, options = {})
           options[:processes] ||= 1
           @@options = options
+          @@id_table = {}
+          @@dependencies_table = {}
           stdin, stdout = proc
-          if entity.to_s.count('.') == 0
-            remove_last = true
-            text = entity.to_s + '.'
-          else
-            remove_last = false
-            text = entity.to_s.gsub('.', '')
-            text += '.' unless ['!', '?'].include?(text[-1])
-          end
+          text, remove_last = valid_text(entity)
           stdin.puts(text + "\n")
           parsed = build(stdout.gets, remove_last)
           if not parsed.nil?
@@ -50,10 +45,16 @@ module Treat
             parsed.children.each do |child|
               entity << child
             end
+            # Remove the period we added at the end.
+            if remove_last
+              last = entity.punctuations[-1]
+              entity.remove!(last)
+            end
           else
             warn "Couldn't parse the text '#{entity.to_s}'."
           end
-          entity
+          link_heads(entity)
+          add_dependencies(entity)
         end
         # Parses an Enju XML output file using the Nogoriki
         # XML reader and converts that structure into a tree
@@ -63,8 +64,6 @@ module Treat
           xml_reader = Nokogiri::XML::Reader.from_memory(xml)
           current_element = nil
           previous_depth = 0
-          id_table = {}
-          edges_table = {}
           # Read the XML file entity by entity.
           while xml_reader.read
             # The depth in the XML tree.
@@ -81,119 +80,135 @@ module Treat
               previous_depth = current_depth
               next
             end
+            # Get and format attributes and dependencies.
             attributes = xml_reader.attributes
-            prefix = ['schema', 'lexentry', 'type']
-            # If the entity has entributes, add them.
-            unless attributes.empty?
-              new_attributes = {}
-              edges = {}
-              id = attributes.delete('id')
-              pred = attributes.delete('pred')
-              attributes.each_pair do |attribute, value|
-                if ['arg1', 'arg2'].include?(attribute)
-                  edges[value] = pred
-                else
-                  if attribute == 'cat'
-                    if xml_reader.name == 'tok'
-                      if value.length > 1 && ['P', 'X'].include?(value[-1]) &&
-                        value != 'PN'
-                        new_attributes[:saturated] = (value[-1] == 'P')
-                        value = value[0..-2]
-                      end
-                      cat = Treat::Languages::English::EnjuCatToCategory[value]
-                      new_attributes[:cat] = cat
-                    else
-                      new_attributes[:enju_cat] = value
-                      xcat = attributes['xcat'].split(' ')[0]
-                      xcat ||= ''
-                      tags = Treat::Languages::English::EnjuCatXcatToPTB.select do |m|
-                        m[0] == value && m[1] == xcat
-                      end
-                      if tags.empty?
-                        tag = 'UK'
-                      else
-                        tag = tags[0][2]
-                      end
-                      new_attributes[:enju_xcat] = xcat
-                      attributes.delete('xcat')
-                      new_attributes[:tag] = tag
-                    end
-                  else
-                    pre = prefix.include?(attribute) ? 'enju_' : ''
-                    new_attributes[:"#{pre+attribute}"] = value
-                  end
-                end
-              end
-              attributes.delete('arg1')
-              attributes.delete('arg2')
-            end
-            # Handle naming conventions.
-            if attributes.has_key?('pos')
-              new_attributes[:tag] = new_attributes[:pos]
-              new_attributes[:tag_set] = :penn
-              new_attributes.delete :pos
+            id = attributes.delete('id')
+            new_attributes = {}; dependencies = {}
+            unless attributes.size == 0
+              new_attributes, dependencies =
+              cleanup_attributes(xml_reader.name, attributes)
             end
             # Create the appropriate entity for the
             # element.
             current_value = ''
-            attributes = new_attributes
             case xml_reader.name
             when 'sentence'
               current_element = Treat::Entities::Sentence.new('')
-              id_table[id] = current_element.id
-              edges_table[current_element.id] = edges
-              current_element.features = attributes
+              @@id_table[id] = current_element.id
+              @@dependencies_table[current_element.id] = dependencies
+              current_element.features = new_attributes
             when 'cons'
               current_element = current_element <<
               Treat::Entities::Phrase.new('')
-              id_table[id] = current_element.id
-              edges_table[current_element.id] = edges
-              current_element.features = attributes
+              @@id_table[id] = current_element.id
+              @@dependencies_table[current_element.id] = dependencies
+              current_element.features = new_attributes
             when 'tok'
-              tmp_attributes = attributes
-              tmp_edges = edges
+              tmp_attributes = new_attributes
+              tmp_dependencies = dependencies
             else
               current_value = xml_reader.value.gsub(/\s+/, "")
-              if !current_value.empty?
+              unless current_value.size == 0
                 current_element = current_element <<
-                Treat::Entities::Entity.from_string(current_value)
+                Treat::Entities::Token.from_string(current_value)
                 if current_element.is_a?(Treat::Entities::Word)
                   current_element.features = tmp_attributes
-                  id_table[id] = current_element.id
-                  edges_table[current_element.id] = tmp_edges                  
+                  @@id_table[id] = current_element.id
+                  @@dependencies_table[current_element.id] = tmp_dependencies
                 end
               end
             end
             previous_depth = current_depth
           end
-          # Add the edges to the entity.
-          unless current_element.nil?
-            root = current_element.root
-            edges_table.each_pair do |id2, edges2|
-              # Next if there are no edges.
-              next if edges2.nil?
-              entity = root.find(id2)
-              edges2.each_pair do |argument, type|
-                # Skip this argument if we don't know
-                # the target node.
+          current_element
+        end
+        # Validate a text - Enju wants period to parse a sentence.
+        def self.valid_text(entity)
+          if entity.to_s.count('.') == 0
+            remove_last = true
+            text = entity.to_s + '.'
+          else
+            remove_last = false
+            text = entity.to_s.gsub('.', '')
+            text += '.' unless ['!', '?'].include?(text[-1])
+          end
+          return text, remove_last
+        end
+        # Link the head and sem_head to their entities.
+        def self.link_heads(entity)
+          entity.each_phrase do |phrase|
+            if phrase.has?(:head)
+              phrase.link(@@id_table[phrase.head], 'head', true, -1)
+              phrase.unset(:head)
+            end
+            if phrase.has?(:sem_head)
+              phrase.link(@@id_table[phrase.sem_head], 'sem_head', true, -1)
+              phrase.unset(:sem_head)
+            end
+          end
+        end
+        # Add dependencies a posterior to a parsed entity.
+        def self.add_dependencies(entity2)
+          entity2.each_entity(:word, :phrase) do |entity|
+            @@dependencies_table.each_pair do |id2, dependencies2|
+              # Next if there are no dependencies.
+              next if dependencies2.nil?
+              entity = entity2.root.find(id2)
+              next if entity.nil?
+              dependencies2.each_pair do |argument, type|
+                # Skip this argument if we don't know the target node.
                 next if argument == 'unk'
-                entity.associate(id_table[argument], type)
+                entity.link(@@id_table[argument], type.intern)
               end
             end
-            # Link the head and sem_head to their entities.
-            root.each_constituent do |constituent|
-              constituent.set :head,
-              root.find(id_table[constituent.head])
-              constituent.set :sem_head,
-              root.find(id_table[constituent.sem_head])
+          end
+        end
+        # Helper function to convert Enju attributes to Treat attributes.
+        def self.cleanup_attributes(name, attributes)
+          new_attributes = {}
+          dependencies = {}
+          pred = attributes.delete('pred')
+          attributes.each_pair do |attribute2, value|
+            attribute = attribute2.strip
+            if attribute == 'arg1' || attribute == 'arg2'
+              dependencies[value] = pred
+              next
+            end
+            if attribute == 'cat'
+              new_attributes[:cat] = value
+              if name == 'tok'
+                if value.length > 1 && ['P', 'X'].include?(value[-1]) &&
+                  value != 'PN'
+                  new_attributes[:saturated] = (value[-1] == 'P')
+                  value = value[0..-2]
+                end
+                new_attributes[:category] =
+                Treat::Languages::Tags::EnjuCatToCategory[value]
+              else
+                tags = Treat::Languages::Tags::EnjuCatXcatToPTB.select do |m|
+                  m[0] == value && m[1] == attributes['xcat']
+                end
+                tag = (tags.size == 0) ? 'FW' : tags[0][2]
+                new_attributes[:tag] = tag
+              end
+            else
+              new_attributes[:"#{attribute}"] = value
             end
           end
-          # Remove the period we added at the end.
-          if remove_last
-            last = current_element.punctuations[-1]
-            current_element.remove!(last)
+          # Delete after iteration.
+          attributes.delete('arg1')
+          attributes.delete('arg2')
+          # Handle naming conventions.
+          if attributes.has_key?('pos')
+            new_attributes[:tag] = new_attributes[:pos]
+            new_attributes[:tag_set] = :penn
+            new_attributes.delete :pos
           end
-          current_element
+          if attributes.has_key?('base')
+            new_attributes[:lemma] = new_attributes[:base]
+            new_attributes.delete :base
+          end
+          return new_attributes, dependencies
         end
       end
     end

data/lib/treat/processors/parsers/stanford.rb

@@ -3,60 +3,128 @@ module Treat
     module Parsers
       # A wrapper class for the Stanford parser.
       class Stanford
-        # Require the Ruby-Java bridge.
-        silence_warnings { require 'rjb' }
-        jar = "#{Treat.bin}/stanford-parser*/stanford-parser*.jar"
-        jars = Dir.glob(jar)
-        if jars.empty? || !File.readable?(jars[0])
-          raise "Could not find stanford parser JAR file (looking in #{jar})"+
-          " You may need to manually download the JAR files and/or set Treat.bin."
-        end
-        Rjb::load(jars[0], ['-Xms256M', '-Xmx512M'])
-        LexicalizedParser = ::Rjb::import('edu.stanford.nlp.parser.lexparser.LexicalizedParser')
-        @@parsers = {}
+        require 'stanford-core-nlp'
+        @@parser = {}
+        DefaultOptions = {
+          :silence => false, 
+          :log_to_file => nil, 
+          :parser_model => nil,
+          :tagger_model => nil
+        }
         # Parse the entity using the Stanford parser.
+        #
+        # Options:
+        # - (String) :log_to_file => a filename to log output to
+        # instead of displaying it.
         def self.parse(entity, options = {})
-          lang = Treat::Languages.describe(entity.language).to_s.upcase
-          pcfg = "#{Treat.bin}/stanford-parser*/grammar/#{lang.upcase}PCFG.ser.gz"
-          pcfgs = Dir.glob(pcfg)
-          if pcfgs.empty? || !File.readable?(pcfgs[0])
-            raise "Could not find a language model for #{lang.downcase} (looking in #{pcfg})."
+          options = DefaultOptions.merge(options)
+          lang = entity.language
+          StanfordCoreNLP.use(lang)
+          if options[:tagger_model]
+            ::StanfordCoreNLP.set_model(
+              'pos.model', options[:tagger_model]
+            )
+          end
+          if options[:parser_model]
+            ::StanfordCoreNLP.set_model(
+              'parser.model', options[:parser_model]
+            )
+          end
+          if options[:silence]
+            options[:log_to_file] = '/dev/null'
+          end
+          if options[:log_to_file]
+            ::StanfordCoreNLP.log_file = 
+              options[:log_to_file] 
+          end
+          @@parser[lang] ||= 
+            ::StanfordCoreNLP.load(
+              :tokenize, :ssplit, :pos, :lemma, :parse
+            )
+          text = ::StanfordCoreNLP::Text.new(entity.to_s)
+          @@parser[lang].annotate(text)
+          
+          text.get(:sentences).each do |s|
+            if entity.is_a?(Treat::Entities::Sentence) ||
+              entity.is_a?(Treat::Entities::Phrase)
+              tag = s.get(:category).to_s
+              tag_s, tag_opt = *tag.split('-')
+              tag_s ||= 'S'
+              entity.set :tag_set, :penn
+              entity.set :tag, tag_s
+              entity.set :tag_opt, tag_opt if tag_opt
+              recurse(s.get(:tree), entity)
+              break
+            else
+              recurse(s.get(:tree), entity)
+            end
           end
-          @@parsers[lang] ||= LexicalizedParser.new(pcfgs[0])
-          parse = @@parsers[lang].apply(entity.to_s)
-          entity.remove_all!          
-          recurse(parse, entity)
-          entity
         end
-        # Helper method which recurses the tree supplied by 
+        
+        # Helper method which recurses the tree supplied by
         # the Stanford parser.
-        def self.recurse(java_node, ruby_node)
+        def self.recurse(java_node, ruby_node, additional_tags = [])
           # Leaf
           if java_node.num_children == 0
-            ruby_child = Treat::Entities::Entity.from_string(java_node.value)
-            labels = java_node.labels.iterator
-            while labels.has_next
-              label = labels.next
-              ruby_child.set :begin_char, label.begin_position
-              ruby_child.set :end_char, label.end_position
-              ruby_child.set :tag, ruby_node.tag
+            label = java_node.label
+            tag = label.get(:part_of_speech).to_s
+            tag_s, tag_opt = *tag.split('-')
+            tag_s ||= ''
+            ruby_node.value = java_node.value.to_s.strip
+            ruby_node.set :tag_set, :penn
+            ruby_node.set :tag, tag_s
+            ruby_node.set :tag_opt, tag_opt if tag_opt
+            ruby_node.set :tag_set, :penn
+            ruby_node.set :lemma, label.get(:lemma).to_s
+            
+            ruby_node.set :character_offset_begin, 
+            label.get(:character_offset_begin).to_s
+            
+            ruby_node.set :character_offset_end, 
+            label.get(:character_offset_end).to_s
+            
+            ruby_node.set :begin_index, 
+            label.get(:begin_index).to_s
+            
+            ruby_node.set :end_index, 
+            label.get(:end_index).to_s
+            
+            additional_tags.each do |t|
+              lt = label.get(t)
+              ruby_node.set t, lt.to_s if lt
             end
-            ruby_node << ruby_child
+            return ruby_node
           else
-            if java_node.num_children == 1
-              return recurse(java_node.children[0], ruby_node)
+
+            if java_node.num_children == 1 &&
+              java_node.children[0].num_children == 0
+              recurse(java_node.children[0], ruby_node, additional_tags)
+              return
             end
+
             java_node.children.each do |java_child|
-              # dependencies = java_child.dependencies.iterator
-              # while dependencies.has_next
-                #dependency = dependencies.next
-              # end
-              ruby_child = Treat::Entities::Phrase.new
-              ruby_child.set :tag, java_child.value 
+              label = java_child.label
+              tag = label.get(:category).to_s
+              tag_s, tag_opt = *tag.split('-')
+              tag_s ||= ''
+              
+              if Treat::Languages::Tags::PhraseTagToCategory[tag_s]
+                ruby_child = Treat::Entities::Phrase.new
+              else
+                l = java_child.children[0].to_s
+                v = java_child.children[0].value.to_s.strip
+                # Mhmhmhmhmhm
+                val = (l == v) ? v :  l.split(' ')[-1].gsub(')', '')
+                ruby_child = Treat::Entities::Token.from_string(val)
+              end
+
               ruby_child.set :tag_set, :penn
+              ruby_child.set :tag, tag_s
+              ruby_child.set :tag_opt, tag_opt if tag_opt
               ruby_node << ruby_child
+
               unless java_child.children.empty?
-                recurse(java_child, ruby_child)
+                recurse(java_child, ruby_child, additional_tags)
               end
             end
           end
@@ -64,4 +132,4 @@ module Treat
       end
     end
   end
-end
+end