treat 0.2.5 → 1.0.0

data/lib/treat/loaders/linguistics.rb

@@ -0,0 +1,30 @@
+class Treat::Loaders
+
+  # A helper class to load a language class
+  # registered with the Linguistics gem.
+  class Linguistics
+
+    silence_warnings { require 'linguistics' }
+    @@languages = {}
+
+    def self.load(language)
+      if @@languages[language]
+        return @@languages[language]
+      end
+      begin
+        l = language.to_s.upcase
+        silence_warnings do
+          @@languages[language] =
+          ::Linguistics.const_get(l)
+        end
+      rescue RuntimeError
+        raise "Ruby Linguistics does " +
+        "not have a module installed " +
+        "for the #{language} language."
+      end
+
+    end
+
+  end
+
+end

data/lib/treat/loaders/stanford.rb

@@ -0,0 +1,27 @@
+class Treat::Loaders
+
+  # A helper class to load a language class
+  # registered with the Linguistics gem.
+  class Stanford
+
+    require 'stanford-core-nlp'
+    
+    StanfordCoreNLP.jar_path = 
+    Treat.bin + 'stanford/'
+    
+    StanfordCoreNLP.model_path = 
+    Treat.models + 'stanford/'
+    
+    StanfordCoreNLP.use(
+    Treat::Languages.describe(
+    Treat.default_language))
+
+    StanfordCoreNLP.log_file = 
+    NULL_DEVICE if Treat.silence
+    
+    StanfordCoreNLP.init
+    @@loaded = true
+    
+  end
+
+end

data/lib/treat/object.rb

@@ -1,5 +1,6 @@
 # Make undefining constants publicly available on any object.
 Object.module_eval do
+  # Unset a constant without private access.
   def self.const_unset(const)
     Object.instance_eval { remove_const(const) }
   end

data/lib/treat/processors.rb

@@ -1,45 +1,38 @@
-module Treat
-  # Category for processor groups.
-  #
-  # A processor group is a group of algorithms for the building
-  # of trees representing textual entities.
-  #
-  # The processor groups include:
-  #
-  #   - Chunkers : split a text into zone objects.
-  #   - Segmenters : split a text or zone into sentence objects.
-  #   - Tokenizers : split a sentence into Token objects.
-  #   - Parsers: split a sentence into a tree of phrases
-  #     containing other phrases and Token objects, representing
-  #     the syntactic structure.
-  module Processors
-    # Chunkers split a text into zones.
-    module Chunkers
-      extend Group
-      self.type = :transformer
-      self.targets = [:document, :section]
-    end
-    # Segmenters split a text or zone into sentences.
-    module Segmenters
-      extend Group
-      self.type = :transformer
-      self.targets = [:document, :zone]
-    end
-    # Tokenizers splits a sentence into Token objects.
-    module Tokenizers
-      extend Group
-      self.type = :transformer
-      self.targets = [:document, :zone, :phrase]
-    end
-    # Parsers split a sentence into phrase objects
-    # representing its syntactic structure, with the
-    # Token objects as children of the phrases.
-    module Parsers
-      extend Group
-      self.type = :transformer
-      self.targets = [:document, :zone, :phrase]
-    end
-    # Makes all the groups autoloadable and creates the workers.
-    extend Treat::Category
+# Processors build trees representing textual entities.
+module Treat::Processors
+  
+  # Chunkers split a document into sections and zones.
+  module Chunkers
+    extend Treat::Groupable
+    self.type = :transformer
+    self.targets = [:document]
+    self.default = :autoselect
   end
-end
+  
+  # Segmenters split a document or zone into sentences.
+  module Segmenters
+    extend Treat::Groupable
+    self.type = :transformer
+    self.targets = [:zone]
+  end
+  
+  # Tokenizers splits a sentence into Token objects.
+  module Tokenizers
+    extend Treat::Groupable
+    self.type = :transformer
+    self.targets = [:phrase]
+  end
+  
+  # Parsers split a sentence into phrase objects
+  # representing its syntactic structure, with the
+  # Token objects as children of the phrases.
+  module Parsers
+    extend Treat::Groupable
+    self.type = :transformer
+    self.targets = [:phrase]
+  end
+  
+  # Make Processors categorizable.
+  extend Treat::Categorizable
+  
+end

data/lib/treat/processors/chunkers/autoselect.rb

@@ -0,0 +1,16 @@
+class Treat::Processors::Chunkers::Autoselect
+
+  def self.chunk(entity, options = {})
+    entity.check_has(:format)
+    begin
+      k = Treat::Processors::
+      Chunkers.const_get(cc(entity.format))
+      k.chunk(entity, options)
+    rescue Treat::Exception
+      Treat::Processors::
+      Chunkers::TXT.chunk(entity, options)
+    end
+    
+  end
+
+end

data/lib/treat/processors/chunkers/html.rb

@@ -0,0 +1,71 @@
+class Treat::Processors::Chunkers::HTML
+
+  require 'nokogiri'
+
+  def self.chunk(entity, options = {})
+    
+    entity.check_hasnt_children
+    
+    doc = Nokogiri::HTML(entity.value)
+    recurse(entity, doc)
+  
+  end
+
+  def self.recurse(node, html_node, level = 1)
+
+    html_node.children.each do |child|
+      
+      next if child.name == 'text'
+
+      txt = child.inner_text
+      
+      if child.name =~ /^h([0-9]{1})$/ ||
+        (child.name == 'p' && txt.length < 45 && 
+        node.parent && node.parent.type == :section)
+        
+        if $1
+          lvl = $1.to_i
+          if lvl <= level
+            node.ancestors_with_type(:section).
+              each do |s|
+                l = s.has?(:level) ? s.level : 1
+                node = s if l == lvl - 1
+            end
+            node = node <<
+            Treat::Entities::Section.new
+          elsif lvl > level
+            node = node <<
+            Treat::Entities::Section.new
+          end
+          level = lvl
+          node.set :level, level
+
+        end
+
+        t = node <<
+        Treat::Entities::Title.new(txt)
+        t.set :level, level
+
+      elsif child.name == 'p'
+        
+        node << Treat::Entities::Zone.
+        from_string(txt)
+
+      elsif ['ul', 'ol'].include?(child.name)
+        node = node <<
+        Treat::Entities::List.new
+      elsif ['li'].include?(child.name)
+        n = Treat::Entities::Entity.
+        zone_from_string(txt)
+        node << n
+      end
+
+      if child.children.size > 0
+        recurse(node, child, level)
+      end
+
+    end
+
+  end
+
+end

data/lib/treat/processors/chunkers/txt.rb

@@ -1,27 +1,21 @@
-module Treat
-  module Processors
-    module Chunkers
-      # This class separates a plain text file into 
-      # zones based on an extremely naive analysis of the 
-      # file. Suprisingly, this works pretty well.
-      class Txt
-        # Split a document into Zone objects.
-        def self.chunk(text, options = {})
-          zones = text.to_s.split("\n")
-          zones.each do |zone|
-            zone.strip!
-            next if zone == ''
-            if false # fix
-              text << Treat::Entities::List.new(zone)
-            end
-            if zone.length < 60
-              text << Treat::Entities::Title.new(zone)
-            else
-              text << Treat::Entities::Paragraph.new(zone)
-            end
-          end
-        end
-      end
+class Treat::Processors::Chunkers::TXT
+  
+  # Separates a string into
+  # zones on the basis of newlines.
+  #
+  # Options: none.
+  def self.chunk(entity, options = {})
+    
+    entity.check_hasnt_children
+    zones = entity.to_s.split("\n")
+    
+    zones.each do |zone|
+      zone.strip!
+      next if zone == ''
+      entity << Treat::Entities::
+      Zone.from_string(zone)
     end
+    
   end
+  
 end

data/lib/treat/processors/parsers/enju.rb

@@ -1,218 +1,263 @@
-module Treat
-  module Processors
-    module Parsers
-      # The Enju class is a wrapper for the Enju syntactic
-      # parser for English. Given a file or string input,
-      # the parser formats it runs it through Enju, and
-      # parses the XML output by Enju using the Nokogiri
-      # XML reader. It creates wrappers for the sentences,
-      # syntactical phrases and  tokens that Enju identified.
-      #
-      # Original paper:
-      # Takuya Matsuzaki, Yusuke Miyao, and Jun'ichi Tsujii.
-      # 2007. Efficient HPSG Parsing with Supertagging and
-      # CFG-filtering. In Proceedings of IJCAI 2007.
-      class Enju
-        # Require the 'open13' library for interaction
-        # with the background Enju process.
-        require 'open3'
-        @@parsers = []
-        @@i = 0
-        # Require the Nokogiri XML parser.
-        require 'nokogiri'
-        # Return the process running Enju.
-        def self.proc
-          if @@parsers.size < @@options[:processes]
-            @@parsers << ::Open3.popen3("enju -xml -i")
-          end
-          @@i += 1
-          @@i = 0 if @@i == @@parsers.size
-          @@parsers[@@i-1]
-        end
-        # Parse the entity into its syntactical phrases using Enju.
-        # Calls #build to initiate XML parsing.
-        def self.parse(entity, options = {})
-          val = entity.to_s
-          entity.remove_all! if entity.has_children?
-          options[:processes] ||= 1
-          @@options = options
-          @@id_table = {}
-          @@dependencies_table = {}
-          stdin, stdout = proc
-          text, remove_last = valid_text(val)
-          stdin.puts(text + "\n")
-          parsed = build(stdout.gets, remove_last)
-          if not parsed.nil?
-            entity.remove_all!
-            parsed.children.each do |child|
-              entity << child
-            end
-            # Remove the period we added at the end.
-            if remove_last
-              last = entity.punctuations[-1]
-              entity.remove!(last)
-            end
-          else
-            warn "Couldn't parse the text '#{entity.to_s}'."
-          end
-          link_heads(entity)
-          add_dependencies(entity)
-        end
-        # Parses an Enju XML output file using the Nogoriki
-        # XML reader and converts that structure into a tree
-        # of wrappers for textual entities.
-        def self.build(xml, remove_last = false)
-          # Read in the XML file.
-          xml_reader = Nokogiri::XML::Reader.from_memory(xml)
-          current_element = nil
-          previous_depth = 0
-          # Read the XML file entity by entity.
-          while xml_reader.read
-            # The depth in the XML tree.
-            current_depth = xml_reader.depth
-            # If we are at the end of the children stack, pop up.
-            if previous_depth > current_depth
-              current_element = current_element.parent
-            end
-            # If an end element has been reached,
-            # change the depth and pop up on next
-            # iteration.
-            if xml_reader.node_type ==
-              Nokogiri::XML::Reader::TYPE_END_ELEMENT
-              previous_depth = current_depth
-              next
-            end
-            # Get and format attributes and dependencies.
-            attributes = xml_reader.attributes
-            id = attributes.delete('id')
-            new_attributes = {}; dependencies = {}
-            unless attributes.size == 0
-              new_attributes, dependencies =
-              cleanup_attributes(xml_reader.name, attributes)
-            end
-            # Create the appropriate entity for the
-            # element.
-            current_value = ''
-            case xml_reader.name
-            when 'sentence'
-              current_element = Treat::Entities::Sentence.new('')
-              @@id_table[id] = current_element.id
-              @@dependencies_table[current_element.id] = dependencies
-              current_element.features = new_attributes
-            when 'cons'
-              current_element = current_element <<
-              Treat::Entities::Phrase.new('')
-              @@id_table[id] = current_element.id
-              @@dependencies_table[current_element.id] = dependencies
-              current_element.features = new_attributes
-            when 'tok'
-              tmp_attributes = new_attributes
-              tmp_dependencies = dependencies
-            else
-              current_value = xml_reader.value.gsub(/\s+/, "")
-              unless current_value.size == 0
-                current_element = current_element <<
-                Treat::Entities::Token.from_string(current_value)
-                if current_element.is_a?(Treat::Entities::Word)
-                  current_element.features = tmp_attributes
-                  @@id_table[id] = current_element.id
-                  @@dependencies_table[current_element.id] = tmp_dependencies
-                end
-              end
-            end
-            previous_depth = current_depth
-          end
-          current_element
-        end
-        # Validate a text - Enju wants period to parse a sentence.
-        def self.valid_text(val)
-          if val.count('.') == 0
-            remove_last = true
-            text = val + '.'
+# This class is a wrapper for the Enju syntactic
+# parser for English. Given an entity's string value,
+# the parser formats it runs it through Enju, and
+# parses the XML output by Enju using the Nokogiri
+# XML reader. It creates wrappers for the sentences,
+# syntactical phrases and tokens that Enju identified.
+#
+# Original paper:
+#
+# Takuya Matsuzaki, Yusuke Miyao, and Jun'ichi Tsujii.
+# 2007. Efficient HPSG Parsing with Supertagging and
+# CFG-filtering. In Proceedings of IJCAI 2007.
+module Treat::Processors::Parsers::Enju
+
+  # Require the 'open3' library to connect
+  # with the background Enju process.
+  require 'open3'
+  
+  # Require the Nokogiri XML parser.
+  require 'nokogiri'
+  
+  # Create only one process and hold on to it.
+  @@parser = nil
+  
+  # A hash of Enju cat tags mapped to word categories.
+  Ectc = Treat::Linguistics::Tags::EnjuCatToCategory
+  
+  # A hash of Enju cat/xcat pairs mapped to PTB tags.
+  Ecxtp = Treat::Linguistics::Tags::EnjuCatXcatToPTB
+  
+  # Parse the entity into its syntactical 
+  # phrases using Enju.
+  #
+  # Options: none.
+  def self.parse(entity, options = {})
+    
+    entity.check_hasnt_children
+    val = entity.to_s
+    
+    @@id_table = {}
+    @@dependencies_table = {}
+    
+    stdin, stdout = proc
+    text, remove_last = valid_text(val)
+    stdin.puts(text + "\n")
+    
+    parsed = build(stdout.gets, remove_last)
+    
+    if parsed
+      entity.remove_all!
+      parsed.children.each do |child|
+        entity << child
+      end
+      # Remove the period we added at the end.
+      if remove_last
+        last = entity.punctuations[-1]
+        entity.remove!(last)
+      end
+    else
+      warn "Warning - Enju couldn't " +
+      "parse the text '#{entity.short_value}'."
+      return
+    end
+    
+    link_heads(entity)
+    add_dependencies(entity)
+  end
+  
+  # Return the process running Enju.
+  def self.proc
+    begin
+      @@parser = ::Open3.popen3("enju -xml -i")
+    rescue Exception => e
+      raise Treat::Exception,
+      "Couldn't initialize Enju: #{e.message}."
+    end
+    @@parser
+  end
+  
+  # Parses an Enju XML output file using the Nogoriki
+  # XML reader and converts that structure into a tree
+  # of wrappers for textual entities.
+  def self.build(xml, remove_last = false)
+    # Read in the XML file.
+    reader = Nokogiri::XML::Reader.from_memory(xml)
+    entity = nil
+    pd = 0
+    # Read the XML file entity by entity.
+    while reader.read
+      # The depth in the XML tree.
+      cd = reader.depth
+      # If we are at the end of the 
+      # children stack, pop up.
+      if pd > cd
+        entity = entity.parent
+      end
+      # If an end element has been reached,
+      # change the depth and pop up on next
+      # iteration.
+      if reader.node_type ==
+        Nokogiri::XML::Reader::TYPE_END_ELEMENT
+        pd = cd
+        next
+      end
+      # Get and format attributes and dependencies.
+      attributes = reader.attributes
+      id = attributes.delete('id')
+      new_attr = {}; dependencies = {}
+      unless attributes.size == 0
+        new_attr, dependencies =
+        cleanup_attributes(reader.name, attributes)
+      end
+      # Create the appropriate entity for the
+      # element.
+      current_value = ''
+      case reader.name
+      when 'sentence'
+        entity = Treat::Entities::Sentence.new('')
+        @@id_table[id] = entity.id
+        @@dependencies_table[entity.id] = dependencies
+        entity.features = new_attr
+      when 'cons'
+        entity = entity <<
+        Treat::Entities::Phrase.new('')
+        @@id_table[id] = entity.id
+        @@dependencies_table[entity.id] = dependencies
+        entity.features = new_attr
+      when 'tok'
+        tmp_attributes = new_attr
+        tmp_dependencies = dependencies
+      else
+        current_value = reader.value.gsub(/\s+/, "")
+        unless current_value.size == 0
+          entity = entity <<
+          Treat::Entities::Token.from_string(current_value)
+          if entity.is_a?(Treat::Entities::Word)
+            entity.features = tmp_attributes
+            @@id_table[id] = entity.id
+            @@dependencies_table[entity.id] = tmp_dependencies
           else
-            remove_last = false
-            text = val.gsub('.', '')
-            text += '.' unless ['!', '?'].include?(text[-1])
-          end
-          return text, remove_last
-        end
-        # Link the head and sem_head to their entities.
-        def self.link_heads(entity)
-          entity.each_phrase do |phrase|
-            if phrase.has?(:head)
-              phrase.link(@@id_table[phrase.head], 'head', true, -1)
-              phrase.unset(:head)
-            end
-            if phrase.has?(:sem_head)
-              phrase.link(@@id_table[phrase.sem_head], 'sem_head', true, -1)
-              phrase.unset(:sem_head)
-            end
+            # Do something useful here
+            entity.set :tag, 'SYM'
           end
         end
-        # Add dependencies a posterior to a parsed entity.
-        def self.add_dependencies(entity2)
-          entity2.each_entity(:word, :phrase) do |entity|
-            @@dependencies_table.each_pair do |id2, dependencies2|
-              # Next if there are no dependencies.
-              next if dependencies2.nil?
-              entity = entity2.root.find(id2)
-              next if entity.nil?
-              dependencies2.each_pair do |argument, type|
-                # Skip this argument if we don't know the target node.
-                next if argument == 'unk'
-                entity.link(@@id_table[argument], type.intern)
-              end
-            end
-          end
+      end
+      pd = cd
+    end
+    entity
+  end
+  
+  # Validate a text - Enju wants period to parse a sentence.
+  def self.valid_text(val)
+    if val.count('.') == 0
+      remove_last = true
+      text = val + '.'
+    else
+      remove_last = false
+      text = val.gsub('.', '')
+      text += '.' unless ['!', '?'].include?(text[-1])
+    end
+    return text, remove_last
+  end
+  
+  # Link the head and sem_head to their entities.
+  def self.link_heads(entity)
+    entity.each_phrase do |phrase|
+      if phrase.has?(:head)
+        phrase.link(
+        @@id_table[phrase.head], 
+        'head', true, -1)
+        phrase.unset(:head)
+      end
+      if phrase.has?(:sem_head)
+        phrase.link(
+        @@id_table[phrase.sem_head], 
+        'sem_head', true, -1)
+        phrase.unset(:sem_head)
+      end
+    end
+  end
+  
+  # Add dependencies a posteriori to a parsed entity.
+  def self.add_dependencies(entity2)
+    
+    entity2.each_entity(:word, :phrase) do |entity|
+      @@dependencies_table.each_pair do |id, dependencies|
+        next if dependencies.nil?
+        entity = entity2.root.find(id)
+        next if entity.nil?
+        dependencies.each_pair do |argument, type|
+          # Skip this argument if we 
+          # don't know the target node.
+          next if argument == 'unk'
+          entity.link(
+            @@id_table[argument], 
+            type.intern
+          )
         end
-        # Helper function to convert Enju attributes to Treat attributes.
-        def self.cleanup_attributes(name, attributes)
-          new_attributes = {}
-          dependencies = {}
-          pred = attributes.delete('pred')
-          attributes.each_pair do |attribute2, value|
-            attribute = attribute2.strip
-            if attribute == 'arg1' || attribute == 'arg2'
-              dependencies[value] = pred
-              next
-            end
-            if attribute == 'cat'
-              new_attributes[:cat] = value
-              if name == 'tok'
-                if value.length > 1 && ['P', 'X'].include?(value[-1]) &&
-                  value != 'PN'
-                  new_attributes[:saturated] = (value[-1] == 'P')
-                  value = value[0..-2]
-                end
-                new_attributes[:category] =
-                Treat::Languages::Tags::EnjuCatToCategory[value]
-              else
-                tags = Treat::Languages::Tags::EnjuCatXcatToPTB.select do |m|
-                  m[0] == value && m[1] == attributes['xcat']
-                end
-                tag = (tags.size == 0) ? 'FW' : tags[0][2]
-                new_attributes[:tag] = tag
-              end
-            else
-              new_attributes[:"#{attribute}"] = value
-            end
-          end
-          # Delete after iteration.
-          attributes.delete('arg1')
-          attributes.delete('arg2')
-          # Handle naming conventions.
-          if attributes.has_key?('pos')
-            new_attributes[:tag] = new_attributes[:pos]
-            new_attributes[:tag_set] = :penn
-            new_attributes.delete :pos
+      end
+    end
+    
+  end
+  
+  # Helper function to convert Enju attributes to Treat attributes.
+  def self.cleanup_attributes(name, attributes)
+    
+    new_attr = {}
+    dependencies = {}
+    pred = attributes.delete('pred')
+    
+    attributes.each_pair do |attribute2, value|
+      
+      attribute = attribute2.strip
+      
+      if attribute == 'arg1' || 
+        attribute == 'arg2'
+        dependencies[value] = pred
+        next
+      end
+      
+      if attribute == 'cat'
+        new_attr[:cat] = value
+        if name == 'tok'
+          if value.length > 1 && 
+            ['P', 'X'].include?(value[-1]) &&
+            value != 'PN'
+            new_attr[:saturated] = 
+            (value[-1] == 'P')
+            value = value[0..-2]
           end
-          if attributes.has_key?('base')
-            new_attributes[:lemma] = new_attributes[:base]
-            new_attributes.delete :base
+          new_attr[:category] = Ectc[value]
+        else
+          tags = Ecxtp.select do |m|
+            m[0] == value && m[1] == 
+            attributes['xcat']
           end
-          return new_attributes, dependencies
+          tag = (tags.size == 0) ? 
+          'FW' : tags[0][2]
+          new_attr[:tag] = tag
         end
+      else
+        new_attr[:"#{attribute}"] = value
       end
+      
+    end
+    
+    # Handle naming conventions.
+    if attributes.has_key?('pos')
+      new_attr[:tag] = new_attr[:pos]
+      new_attr[:tag_set] = :penn
+      new_attr.delete :pos
+    end
+    
+    if attributes.has_key?('base')
+      new_attr[:lemma] = new_attr[:base]
+      new_attr.delete :base
     end
+    
+    return new_attr, dependencies
+  
   end
-end
+  
+end