treat 0.1.4 → 0.2.0

data/lib/treat/formatters/readers/doc.rb

@@ -4,8 +4,10 @@ module Treat
       class Doc
         def self.read(document, options = {})
           f = `antiword #{document.file}`
+          f.gsub!("\n\n", '#keep#')
+          f.gsub!("\n", ' ')
+          f.gsub!('#keep#', "\n\n")
           document << Treat::Entities::Entity.from_string(f)
-          document
         end
       end
     end

data/lib/treat/formatters/readers/html.rb

@@ -7,7 +7,7 @@ module Treat
         # Require Hpricot.
         silence_warnings { require 'hpricot' }
         # By default, backup the HTML text while cleaning.
-        DefaultOptions = { clean: true, backup: false }
+        DefaultOptions = { :clean => true, :backup => false }
         # Read the HTML document and strip it of its markup.
         #
         # Options:

data/lib/treat/formatters/readers/image.rb

@@ -0,0 +1,43 @@
+module Treat
+  module Formatters
+    module Readers
+      # This class is a wrapper for the Google Ocropus
+      # optical character recognition (OCR) engine.
+      #
+      # "OCRopus(tm) is a state-of-the-art document
+      # analysis and OCR system, featuring pluggable
+      # layout analysis, pluggable character recognition,
+      # statistical natural language modeling, and multi-
+      # lingual capabilities."
+      #
+      # Original paper:
+      # Breuel, Thomas M. The Ocropus Open Source OCR System.
+      # DFKI and U. Kaiserslautern, Germany.
+      class Image
+        #  Read a file using the Google Ocropus reader.
+        #
+        # Options:
+        # - (Boolean) :silent => whether to silence Ocropus.
+        def self.read(document, options = {})
+          read = lambda do |doc|
+            create_temp_dir do |tmp|
+              `ocropus book2pages #{tmp}/out #{doc.file}`
+              `ocropus pages2lines #{tmp}/out`
+              `ocropus lines2fsts #{tmp}/out`
+              `ocropus buildhtml #{tmp}/out > #{tmp}/output.html`
+              f = document.file
+              doc.remove_all!
+              doc.set :file,  "#{tmp}/output.html"
+              doc.read(:html)
+              doc.set :file, f
+            end
+          end
+          options[:silent] ? 
+          silence_stdout { read.call(document) } :
+          read.call(document)
+          document
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/readers/odt.rb

@@ -17,7 +17,6 @@ module Treat
           raise "Couldn't unzip dot file #{document.file}!" unless f
           xml_h = OOXmlHandler.new
           REXML::Document.parse_stream(f, xml_h)
-          puts xml_h.plain_text
           document << Treat::Entities::Entity.from_string(xml_h.plain_text)
           document
         end
@@ -39,7 +38,7 @@ module Treat
               s = s.strip
               if s.length > 0
                 @plain_text << s
-                @plain_text << "\n"
+                @plain_text << "\n\n"
               end
             end
           end

data/lib/treat/formatters/readers/pdf.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 module Treat
   module Formatters
     module Readers
@@ -10,7 +11,14 @@ module Treat
         def self.read(document, options = {})
           create_temp_file(:txt) do |tmp|
             `pdftotext #{document.file} #{tmp} `.strip
-            document << Treat::Entities::Entity.from_string(File.read(tmp))
+            f = File.read(tmp)
+            f.gsub!("\t\r ", '')
+            f.gsub!('-­‐', '-')
+            f.gsub!("\n\n", '#keep#')
+            f.gsub!("\n", ' ')
+            f.gsub!(" ", ' ')
+            f.gsub!('#keep#', "\n\n")
+            document << Treat::Entities::Entity.from_string(f)
           end
           document
         end

data/lib/treat/formatters/readers/xml.rb

@@ -0,0 +1,40 @@
+module Treat
+  module Formatters
+    module Readers
+      class XML
+        require 'stanford-core-nlp'
+        require 'cgi'
+        # By default, backup the XML text while cleaning.
+        DefaultOptions = { :clean => true, :backup => false }
+        @@xml_cleaner = nil
+        # Read the XML document and strip it of its markup.
+        # Also splits the text into sentences and tokenizes it?
+        #
+        # Options:
+        #
+        # - (Boolean) :clean => whether to strip XML markup.
+        # - (Boolean) :backup => whether to backup the XML 
+        #   markup while cleaning.
+        def self.read(document, options = {})
+          options = DefaultOptions.merge(options)
+          document << Treat::Entities::Entity.from_string(File.read(document.file))
+          if options[:clean]
+            @@xml_cleaner ||= StanfordCoreNLP.load(:tokenize, :ssplit, :cleanxml)
+            document.each do |zone|
+              text = StanfordCoreNLP::Text.new(zone.to_s)
+              @@xml_cleaner.annotate(text)
+              sentences = []
+              text.get(:sentences) do |sentence|
+                sentences << Treat::Entities::Sentence.from_string(sentence.to_s)
+              end
+              val = sentences.join(' ')
+              zone.set :xml_value, CGI.escapeHTML(text.to_s) if options[:backup]
+              zone.value = val
+            end
+          end
+          document
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/serializers/xml.rb

@@ -6,44 +6,80 @@ module Treat
         # Reauire the Nokogiri XML parser.
         require 'nokogiri'
         # Serialize an entity tree in XML format.
+        #
+        # Options:
+        # - (String) :file => a file to write to.
         def self.serialize(entity, options = {})
-          options = {:indent => 0} if options[:indent].nil?
+          options = options.merge({:indent => 0}) if options[:indent].nil?
+          indent = options[:indent]
           if options[:indent] == 0
-            # enc = entity.encoding(:r_chardet19).to_s.gsub('_', '-').upcase
-            string = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\" ?>"  # Fix
+            enc = entity.to_s.encoding.to_s.downcase
+            string = "<?xml version=\"1.0\" encoding=\"#{enc}\" standalone=\"no\" ?>\n<treat>"
           else
             string = ''
           end
           spaces = ''
           options[:indent].times { spaces << ' ' }
-          attributes = ''
+          attributes = " id='#{entity.id}' "
           if !entity.features.nil? && entity.features.size != 0
-            attributes = ' '
+            attributes << ' '
             entity.features.each_pair do |feature, value|
               if value.is_a? Entities::Entity
                 attributes << "#{feature}='#{value.id}' "
               else
-                attributes << "#{feature}='#{value}' "
+                attributes << "#{feature}='#{escape(value)}' "
               end
             end
-            entity.edges.each_pair do |id,edge|
-              attributes << "#{edge}='#{id}' "
+            attributes << "dependencies='"
+            a = []
+            entity.dependencies.each do |dependency|
+               a << ("{target: #{dependency.target}, type: #{dependency.type}, " +
+                "directed: #{dependency.directed}, " +
+                "direction: #{dependency.direction}}" )
             end
+            # Structs.
+            attributes << a.join('--') + "'"
           end
           tag = entity.class.to_s.split('::')[-1].downcase
-          string += "\n#{spaces}<#{tag}#{attributes[0..-2]}>"
+          unless entity.is_a?(Treat::Entities::Token)
+            string += "\n"
+          end
+          string += "#{spaces}<#{tag}#{attributes}>"
           if entity.has_children?
             options[:indent] += 1
-            entity.children.each do |child| 
-              string = string + serialize(child, options)
+            entity.children.each do |child|
+              string =
+              string +
+              serialize(child, options)
             end
             options[:indent] -= 1
           else
-            string = string + "\n#{spaces}#{entity.value}"
+            string = string + "#{escape(entity.value)}"
+          end
+          unless entity.is_a?(Treat::Entities::Token)
+            string += "\n#{spaces}"
           end
-          string + "\n#{spaces}</#{tag}>"
+          string += "</#{tag}>\n"
+          if indent == 0
+            string += "\n</treat>"
+            if options[:file]
+              File.open(options[:file], 'w') { |f| f.write(string) }
+            end
+            # puts string
+          end
+          string
+        end
+        
+        def self.escape(input)
+          result = input.to_s.dup
+          result.gsub!("&", "&amp;")
+          result.gsub!("<", "&lt;")
+          result.gsub!(">", "&gt;")
+          result.gsub!("'", "&apos;")
+          result.gsub!("\"", "&quot;")
+          result
         end
       end
     end
   end
-end
+end

data/lib/treat/formatters/serializers/yaml.rb

@@ -7,9 +7,14 @@ module Treat
       class YAML
         # Serialize an entity in YAML format.
         #
-        # Options: none.
+        # Options:
+        # - (String) :file => a file to write to.
         def self.serialize(entity, options = {})
-          ::Psych.dump(entity)
+          yaml = ::Psych.dump(entity)
+          if options[:file]
+            File.open(options[:file], 'w') { |f| f.write(yaml) }
+          end
+          yaml
         end
       end
     end

data/lib/treat/formatters/unserializers/xml.rb

@@ -11,6 +11,8 @@ module Treat
         def self.unserialize(document, options = {})
           # Read in the XML file.
           xml = File.read(document.file)
+          xml.gsub!('<treat>', '')
+          xml.gsub!('</treat>', '')
           xml_reader = Nokogiri::XML::Reader.from_memory(xml)
           current_element = nil
           previous_depth = 0
@@ -33,13 +35,33 @@ module Treat
             end
 
             id = nil; value = ''
-            attributes = {}; edges = {}
-            unless xml_reader.attributes.empty?
+            attributes = {}
+            dependencies = []
+            unless xml_reader.attributes.size == 0
               xml_reader.attributes.each_pair do |k,v|
                 if k == 'id'
-                  id = v
-                elsif k == 'edges'
-                  edges = v
+                  id = v.to_i
+                elsif k == 'dependencies'
+                  a = v.split('--')
+                  a.each do |b|
+                    c = b.split(';')
+                    c.each do |dep|
+                      vals = []
+                      dep.split(',').each do |name_val|
+                        name_val = name_val[0..-2] if name_val[-1] == '}'
+                        d =  name_val.split(':')[1]
+                        vals << d.strip if d
+                      end
+                      
+                      target, type, directed, direction = *vals
+                      dependencies << [
+                          target.to_i, 
+                          type,
+                          (directed == 'true' ? true : false),
+                          direction.to_i
+                      ]
+                    end
+                  end
                 elsif k == 'value'
                   value = v
                 else
@@ -60,9 +82,13 @@ module Treat
               end
               current_element.features = attributes
               current_element.features = attributes
-              current_element.edges = edges
+              dependencies.each do |dependency|
+                target, type, directed, direction = *dependency
+                current_element.link(target, type, directed, direction)
+              end
             else
-              current_value = xml_reader.value.strip
+              current_value = xml_reader.value ?
+              xml_reader.value.strip : ''
               if current_value && current_value != ''
                 current_element.value = current_value
                 current_element.register_token(current_element)

data/lib/treat/formatters/visualizers/dot.rb

@@ -2,7 +2,16 @@ module Treat
   module Formatters
     module Visualizers
       class Dot
-        DefaultOptions = {colors: {}, :features => :all}
+        require 'date'
+        DefaultOptions = {
+          :colors => {}, 
+          :features => :all, 
+          :file => nil,
+          :remove_types => [],
+          :remove_features => [],
+          :colors => nil,
+          :first => true # For internal purposes only.
+        }
         # Create the top-most graph structure
         # and delegate the creation of the graph
         # nodes to to_dot.
@@ -11,53 +20,114 @@ module Treat
           string = "graph {"
           string << self.to_dot(entity, options)
           string << "\n}"
+          if options[:file]
+            File.open(options[:file], 'w') { |f| f.write(string) }
+          end
+          string
         end
         # dot -Tpdf test4.dot > test4.pdf
         def self.to_dot(entity, options)
+          # Filter out specified types.
+          match_types = lambda do |t1, t2s|
+            f = false
+            t2s.each { |t2| f = true if Treat::Entities.match_types[t1][t2] }
+            f
+          end
+          return '' if match_types.call(entity.type, options[:remove_types])
           # Id
           string = ''
           label = ''
+          sv = entity.short_value.inspect[1..-2]
           string = "\n#{entity.id} ["
-          # Value
-          if entity.is_a?(Treat::Entities::Token)
-            label = entity.to_s
-          else
-            label = entity.type.to_s.capitalize + " "
-            if entity.is_leaf?
-              label = entity.short_value.gsub(' [...]', " [...] \\n")
-            end
-          end
+          label = "#{entity.type.to_s.capitalize}\\n\\\"#{sv}\\\""
+          label.gsub!(' [...]', " [...] \\n")
           # Features
           if entity.has_features?
             unless options[:features] == :none
               label << "\\n"
               entity.features.each do |feature, value|
+                next if options[:remove_features].include?(feature)
                 if options[:features] == :all ||
                   options[:features].include?(feature)
                   if value.is_a?(Treat::Entities::Entity)
-                    label << "\\n#{feature}=\\\"*#{value.id}\\\","
+                    label << "\\n#{feature}:  \\\"*#{value.id}\\\""
+                  elsif value.is_a?(Struct)
+                    label << "\\n#{feature}: \\n\{ "
+                    value.members.each do |member|
+                      v = value.send(member)
+                      v = v.to_s if v.is_a?(DateTime)
+                      v = "*#{v.id}" if v.is_a?(Treat::Entities::Entity)
+                      v = v ? v.inspect : ' -- '
+                      v.gsub!('[', '\[')
+                      v.gsub!('{', '\}')
+                      v.gsub!(']', '\]')
+                      v.gsub!('}', '\}')
+                      v.gsub!('"', '\"')
+                      label <<  "#{member}: #{v},\\n"
+                    end
+                     label = label[0..-4] unless label[-2] == '{'
+                     label << "\},"
+                  elsif value.is_a?(Hash)
+                    label << "\\n#{feature}: \\n\{ "
+                    value.each do |k,v|
+                      v = v ? v.inspect : ' -- '
+                      v.gsub!('[', '\[')
+                      v.gsub!('{', '\}')
+                      v.gsub!(']', '\]')
+                      v.gsub!('}', '\}')
+                      v.gsub!('"', '\"')
+                      label <<  "#{k}: #{v},\\n"
+                    end
+                    label = label[0..-4] unless label[-2] == '{'
+                    label << "\},"
+                  elsif value.is_a?(Array)
+                    label << "\\n#{feature}: \\n\[ "
+                    value.each do |e|
+                      e = "*#{e.id}" if e.is_a?(Treat::Entities::Entity)
+                      label << "#{e},\\n"
+                    end
+                    label = label[0..-4] unless label[-2] == '['
+                    label << " \]"
                   else
-                    label << "\\n#{feature}=\\\"#{value}\\\","
+                    label << "\\n#{feature}:  \\\"#{value}\\\""
                   end
                 end
               end
             end
           end
+          color = nil
+          if options[:colors]
+            options[:colors].each do |col, lambda|
+              color = col.to_s if lambda.call(entity)
+              break if color
+            end
+          end
           label = label[0..-2] if label[-1] == ','
-          string << "label=\"#{label}\"]"
+          string << "label=\"#{label}\",color=\"#{color.to_s}\"]"
+          string.gsub!('\\\\""]', '\\""]')
+          string.gsub('\"\""]', '\""]')
           # Parent-child relationships.
           if entity.has_parent?
-            string << "\n#{entity.parent.id} -- #{entity.id};"
+            unless options[:first] == true
+              string << "\n#{entity.parent.id} -- #{entity.id};" 
+            end
           end
-          # Edges.
-          if entity.has_edges?
-            entity.edges.each_pair do |target, type|
-              string << "\n#{entity.id} -- #{target}"
-              string << "[label=#{type},dir=forward,"
-              string << "arrowhead=\"odiamond\"]"
+          # Dependencies.
+          if entity.has_dependencies?
+            entity.dependencies.each do |dependency|
+              dir = ''
+              if dependency.directed == true
+                dir = dependency.direction == 1 ? 'forward' : 'back'
+                dir = ",dir=#{dir}"
+              else
+                dir = ",dir=both"
+              end
+              string << "\n#{entity.id} -- #{dependency.target}"
+              string << "[label=#{dependency.type}#{dir}]"
             end
           end
           # Recurse.
+          options[:first] = false
           if entity.has_children?
             entity.each do |child|
               string << self.to_dot(child, options)