treat 0.2.5 → 1.0.0

data/lib/treat/formatters/readers/pdf.rb

@@ -1,28 +1,31 @@
 # encoding: utf-8
-module Treat
-  module Formatters
-    module Readers
-      # A wrapper for the Poppler pdf2text utility, which
-      # extracts the text from a PDF file.
-      class PDF
-        # Read a PDF file using the Poppler pdf2text utility.
-        # 
-        # Options: none.
-        def self.read(document, options = {})
-          create_temp_file(:txt) do |tmp|
-            `pdftotext #{document.file} #{tmp} `.strip
-            f = File.read(tmp)
-            f.gsub!("\t\r ", '')
-            f.gsub!('-­‐', '-')
-            f.gsub!("\n\n", '#keep#')
-            f.gsub!("\n", ' ')
-            f.gsub!(" ", ' ')
-            f.gsub!('#keep#', "\n\n")
-            document << Treat::Entities::Entity.from_string(f)
-          end
-          document
-        end
-      end
+# A wrapper for the Poppler pdf2text utility, which
+# extracts the text from a PDF file.
+module Treat::Formatters::Readers::PDF
+
+  # Read a PDF file using the Poppler pdf2text utility.
+  #
+  # Options: none.
+  def self.read(document, options = {})
+    
+    create_temp_file(:txt) do |tmp|
+      
+      `pdftotext #{document.file} #{tmp} `.strip
+      f = File.read(tmp)
+      f.gsub!("\t\r ", '')
+      f.gsub!('-­‐', '-')
+      f.gsub!("\n\n", '#keep#')
+      f.gsub!("\n", ' ')
+      # Fix for an incompatible space character.
+      f.gsub!(" ", ' ')  
+      f.gsub!('#keep#', "\n\n")
+      
+      document.value = f
+      document.set :format, :pdf
+      document
+      
     end
+    
   end
-end
+
+end

data/lib/treat/formatters/readers/txt.rb

@@ -1,17 +1,14 @@
-module Treat
-  module Formatters
-    module Readers
-      # This class simply reads a plain text file.
-      class Txt
-        # Build an entity from a string in plain text format.
-        # 
-        # Options: none.
-        def self.read(document, options = {})
-          f = File.read(document.file)
-          document << Treat::Entities::Entity.from_string(f)
-          document
-        end
-      end
-    end
+# This class simply reads a plain text file.
+class Treat::Formatters::Readers::TXT
+
+  # Build an entity from a string 
+  # in plain text format.
+  #
+  # Options: none.
+  def self.read(document, options = {})
+    document.value = File.read(document.file)
+    document.set :format, :txt
+    document
   end
+  
 end

data/lib/treat/formatters/readers/xml.rb

@@ -1,40 +1,77 @@
-module Treat
-  module Formatters
-    module Readers
-      class XML
-        require 'stanford-core-nlp'
-        require 'cgi'
-        # By default, backup the XML text while cleaning.
-        DefaultOptions = { :clean => true, :backup => false }
-        @@xml_cleaner = nil
-        # Read the XML document and strip it of its markup.
-        # Also splits the text into sentences and tokenizes it?
-        #
-        # Options:
-        #
-        # - (Boolean) :clean => whether to strip XML markup.
-        # - (Boolean) :backup => whether to backup the XML 
-        #   markup while cleaning.
-        def self.read(document, options = {})
-          options = DefaultOptions.merge(options)
-          document << Treat::Entities::Entity.from_string(File.read(document.file))
-          if options[:clean]
-            @@xml_cleaner ||= StanfordCoreNLP.load(:tokenize, :ssplit, :cleanxml)
-            document.each do |zone|
-              text = StanfordCoreNLP::Text.new(zone.to_s)
-              @@xml_cleaner.annotate(text)
-              sentences = []
-              text.get(:sentences) do |sentence|
-                sentences << Treat::Entities::Sentence.from_string(sentence.to_s)
-              end
-              val = sentences.join(' ')
-              zone.set :xml_value, CGI.escapeHTML(text.to_s) if options[:backup]
-              zone.value = val
-            end
-          end
-          document
+class Treat::Formatters::Readers::XML
+
+  require 'treat/loaders/stanford'
+  require 'cgi'
+
+  # By default, don't backup the XML
+  # document while cleaning it.
+  DefaultOptions = {
+    :keep_html => false
+  }
+
+  # Hold one instance of the XML cleaner.
+  @@xml_reader = nil
+
+  # Read the XML document and strip it of its markup.
+  # Also segments and tokenizes the text.
+  #
+  # Options:
+  #
+  # - (Boolean) :keep_xml => whether to backup the XML
+  #   markup while cleaning.
+  def self.read(document, options = {})
+
+    raise 'Not implemented.'
+
+    options = DefaultOptions.merge(options)
+
+    xml = File.read(document.file)
+
+    @@xml_reader ||= StanfordCoreNLP.load(
+    :tokenize, :ssplit, :cleanxml)
+
+    text = StanfordCoreNLP::Text.new(xml)
+    @@xml_reader.annotate(text)
+
+    text.get(:sentences).each do |sentence|
+
+      s = Treat::Entities::Sentence.
+      from_string(sentence.to_s, true)
+
+      sentence.get(:tokens).each do |token|
+        val = token.value.to_s.strip.gsub('\/', '/')
+        next if val =~ /^<[^>]+>$/
+
+        t = Treat::Entities::Token.
+        from_string(val)
+        c = token.get(:xml_context)
+
+        if c
+          context = []
+          c.each { |tag| context << tag.to_s }
+          t.set :xml_context, context
         end
+
+        s << t
+
       end
+
+      if Treat::Entities::Zone.from_string('')
+        section << s
+      end
+
+      if options[:backup]
+        document.set :xml_value,
+        CGI.escapeHTML(text.to_s)
+      end
+
+      document.value = ''
+
     end
+    
+    document.set :format, :xml
+    document
+    
   end
-end
+
+end

data/lib/treat/formatters/serializers/xml.rb

@@ -1,85 +1,86 @@
-module Treat
-  module Formatters
-    module Serializers
-      # This class converts an entity to a storable XML format.
-      class XML
-        # Reauire the Nokogiri XML parser.
-        require 'nokogiri'
-        # Serialize an entity tree in XML format.
-        #
-        # Options:
-        # - (String) :file => a file to write to.
-        def self.serialize(entity, options = {})
-          options = options.merge({:indent => 0}) if options[:indent].nil?
-          indent = options[:indent]
-          if options[:indent] == 0
-            enc = entity.to_s.encoding.to_s.downcase
-            string = "<?xml version=\"1.0\" encoding=\"#{enc}\" standalone=\"no\" ?>\n<treat>"
-          else
-            string = ''
-          end
-          spaces = ''
-          options[:indent].times { spaces << ' ' }
-          attributes = " id='#{entity.id}' "
-          if !entity.features.nil? && entity.features.size != 0
-            attributes << ' '
-            entity.features.each_pair do |feature, value|
-              if value.is_a? Entities::Entity
-                attributes << "#{feature}='#{value.id}' "
-              else
-                attributes << "#{feature}='#{escape(value)}' "
-              end
-            end
-            attributes << "dependencies='"
-            a = []
-            entity.dependencies.each do |dependency|
-               a << ("{target: #{dependency.target}, type: #{dependency.type}, " +
-                "directed: #{dependency.directed}, " +
-                "direction: #{dependency.direction}}" )
-            end
-            # Structs.
-            attributes << a.join('--') + "'"
-          end
-          tag = entity.class.to_s.split('::')[-1].downcase
-          unless entity.is_a?(Treat::Entities::Token)
-            string += "\n"
-          end
-          string += "#{spaces}<#{tag}#{attributes}>"
-          if entity.has_children?
-            options[:indent] += 1
-            entity.children.each do |child|
-              string =
-              string +
-              serialize(child, options)
-            end
-            options[:indent] -= 1
-          else
-            string = string + "#{escape(entity.value)}"
-          end
-          unless entity.is_a?(Treat::Entities::Token)
-            string += "\n#{spaces}"
-          end
-          string += "</#{tag}>\n"
-          if indent == 0
-            string += "\n</treat>"
-            if options[:file]
-              File.open(options[:file], 'w') { |f| f.write(string) }
-            end
-            # puts string
-          end
-          string
+# This class converts an entity to a storable XML format.
+class Treat::Formatters::Serializers::XML
+
+  # Reauire the Nokogiri XML parser.
+  require 'nokogiri'
+  # Serialize an entity tree in XML format.
+  #
+  # Options:
+  # - (String) :file => a file to write to.
+  def self.serialize(entity, options = {})
+    
+    options = options.merge({:indent => 0}) if options[:indent].nil?
+    indent = options[:indent]
+    if options[:indent] == 0
+      enc = entity.to_s.encoding.to_s.downcase
+      string = "<?xml version=\"1.0\" encoding=\"#{enc}\" standalone=\"no\" ?>\n<treat>\n"
+    else
+      string = ''
+    end
+    spaces = ''
+    options[:indent].times { spaces << ' ' }
+    attributes = " id='#{entity.id}'"
+    if !entity.features.nil? && entity.features.size != 0
+      attributes << ' '
+      entity.features.each_pair do |feature, value|
+        if value.is_a? Treat::Entities::Entity
+          attributes << "#{feature}='#{value.id}' "
+        else
+          attributes << "#{feature}='#{escape(value)}' "
+        end
+      end
+      unless entity.dependencies.empty?
+        attributes << "dependencies='"
+        a = []
+        entity.dependencies.each do |dependency|
+          a << ("{target: #{dependency.target}, type: #{dependency.type}, " +
+          "directed: #{dependency.directed}, " +
+          "direction: #{dependency.direction}}" )
         end
-        
-        def self.escape(input)
-          result = input.to_s.dup
-          result.gsub!("&", "&amp;")
-          result.gsub!("<", "&lt;")
-          result.gsub!(">", "&gt;")
-          result.gsub!("'", "&apos;")
-          result.gsub!("\"", "&quot;")
-          result
+        # Structs.
+        attributes << a.join(',') + "'"
+      end
+    end
+    tag = entity.class.to_s.split('::')[-1].downcase
+    string += "#{spaces}<#{tag}#{attributes}>"
+    unless entity.is_a?(Treat::Entities::Token)
+      string += "\n"
+    end
+    if entity.has_children?
+      options[:indent] += 1
+      entity.children.each do |child|
+        string =
+        string +
+        serialize(child, options)
+      end
+      options[:indent] -= 1
+    else
+      string = string + "#{escape(entity.value)}"
+    end
+    unless entity.is_a?(Treat::Entities::Token)
+      string += "#{spaces}"
+    end
+    string += "</#{tag}>\n"
+    if indent == 0
+      string += "\n</treat>"
+      if options[:file]
+        File.open(options[:file], 'w') do |f|
+          f.write(string)
         end
       end
+      # puts string
     end
+    string
+  end
+
+  def self.escape(input)
+    result = input.to_s.dup
+    result.gsub!("&", "&amp;")
+    result.gsub!("<", "&lt;")
+    result.gsub!(">", "&gt;")
+    result.gsub!("'", "&apos;")
+    result.gsub!("\"", "&quot;")
+    result
   end
+
 end

data/lib/treat/formatters/serializers/yaml.rb

@@ -1,22 +1,23 @@
-module Treat
-  module Formatters
-    module Serializers
-      # Require the Psych YAML serializer.
-      require 'psych'
-      # This class serializes entities in YAML format.
-      class YAML
-        # Serialize an entity in YAML format.
-        #
-        # Options:
-        # - (String) :file => a file to write to.
-        def self.serialize(entity, options = {})
-          yaml = ::Psych.dump(entity)
-          if options[:file]
-            File.open(options[:file], 'w') { |f| f.write(yaml) }
-          end
-          yaml
-        end
+# This class serializes entities in YAML format.
+class Treat::Formatters::Serializers::YAML
+  
+  silence_warnings do
+    # Require the Psych YAML serializer.
+    require 'psych'
+  end
+  
+  # Serialize an entity in YAML format.
+  #
+  # Options:
+  # - (String) :file => a file to write to.
+  def self.serialize(entity, options = {})
+    yaml = ::Psych.dump(entity)
+    if options[:file]
+      File.open(options[:file], 'w') do |f| 
+        f.write(yaml)
       end
     end
+    yaml
   end
+
 end

data/lib/treat/formatters/unserializers/autoselect.rb

@@ -1,25 +1,15 @@
-module Treat
-  module Formatters
-    module Unserializers
-      # This class doesn't perform any unserializing;
-      # it simply routes the document to an unserializer
-      # based on the file extension of the document.
-      class Autoselect
-        # Unserialize any supported file format.
-        # 
-        # Options: none.
-        def self.unserialize(document, options = {})
-          ext = document.file.split('.')[-1]
-          if ext == 'yaml' || ext == 'yml'
-            document.unserialize(:yaml)
-          elsif ext == 'xml'
-            document.unserialize(:xml)
-          else
-            raise "File #{document.file} was not recognized "+
-            "as a supported serialized format."
-          end
-        end
-      end
+class Treat::Formatters::Unserializers::Autoselect
+
+  def self.unserialize(document, options = {})
+    file = document.file
+    if file.index('yml') || file.index('yaml')
+      document.unserialize(:yaml, options)
+    elsif file.index('xml')
+      document.unserialize(:xml, options)
+    else
+      raise Treat::Exception,
+      "Unreadable serialized format for file #{file}."
     end
   end
+
 end