treat 0.1.1

data/lib/treat/feature.rb

@@ -0,0 +1,53 @@
+module Treat
+  class Feature
+    # Undefine all methods, except those that 
+    # create any problems (e.g. with serializing).
+    instance_methods.each do |meth|
+      undef_method(meth) if meth !~ 
+      /^(__|object_id|class|instance_variables|instance_variable_get)/
+    end
+    # Allows to read the probability hash,
+    # the possible values of the feature,
+    # and the best value (with highest P).
+    attr_reader :p_hash, :values, :best
+    # Initialize the feature with a hash 
+    # of features => probabilities.
+    def initialize(p_hash)
+      @p_hash = p_hash
+      normalize
+      max = @p_hash.values.max
+      @best = @p_hash.select { |i,j| j == max }.keys.sample
+      @values = @p_hash.keys
+      type = @values[0].class
+      if type == ::Symbol || type == ::NilClass
+        @object = @best
+      else
+        @object = type.new(@best)
+      end
+    end
+    # Normalize the probabilities, so that
+    # the sum of all probabilities is one,
+    # except if the sum of all probabilities
+    # is already below one (in which case we
+    # assume that the feature is intentionally
+    # incomplete).
+    def normalize
+      sum = @p_hash.inject(0.0) { |r, e| r + e[1] }
+      return if sum <= 1.0
+      p = {}
+      @p_hash.each { |k,v| p[k] =  v.to_f/sum.to_f }
+      @p_hash = p
+    end
+    # Find the probability of value x.
+    def probability(x)
+      @p_hash[x] ? @p_hash[x] : 0
+    end
+    # Alias for probability: p(x).
+    alias :p :probability
+    # Catch all other methods than the ones
+    # explicitly defined. 
+    def method_missing(sym, *args, &block)
+      @object.send(sym, *args, &block)
+    end
+  end
+end

data/lib/treat/formatters.rb

@@ -0,0 +1,44 @@
+module Treat
+  # Formatters handle conversion of Entities to and from
+  # external file formats.
+  module Formatters
+    # Readers read a document and create the top-level entity
+    # corresponding to the content of the document.
+    module Readers
+      extend Group
+      self.type = :transformer
+      self.targets = [:collection, :document]
+      self.default = :autoselect
+    end
+    # Unserializers recreate entities from a serialized format.
+    module Unserializers
+      extend Group
+      self.type = :transformer
+      self.targets = [:collection, :document]
+      self.default = :autoselect
+    end
+    # Visualizers transform entities into a visualizable format.
+    module Visualizers
+      extend Group
+      self.type = :computer
+      self.targets = [:entity]
+      self.default = :tree
+    end
+    # Serializers transform entities into a storable format.
+    module Serializers
+      extend Group
+      self.type = :computer
+      self.targets = [:entity]
+      self.default = :yaml
+    end
+    # Serializers transform entities into a storable format.
+    module Cleaners
+      extend Group
+      self.type = :annotator
+      self.targets = [:document]
+      self.default = :html
+    end
+    extend Treat::Category
+  end
+end
+

data/lib/treat/formatters/cleaners/html.rb

@@ -0,0 +1,17 @@
+module Treat
+  module Formatters
+    module Cleaners
+      class HTML
+        silently { require 'hpricot' }
+        def self.clean(document, options = {})
+          document.each_text do |text|
+            text.set :html_value, text.value
+            v = Hpricot(text.value).inner_text
+            text.value = v
+          end
+          document
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/readers/autoselect.rb

@@ -0,0 +1,35 @@
+module Treat
+  module Formatters
+    module Readers
+      # This class isn't a wrapper for anything.
+      # It simply delegates the reading task to
+      # the appropriate reader based on the file
+      # extension of the supplied document.
+      class Autoselect
+        # A list of image extensions that should be routed
+        # to the Ocropus OCR engine.
+        ImageExtensions = ['gif', 'jpg', 'jpeg', 'png']
+        # Select the appropriate reader based on the format
+        # of the filename in document.
+        # 
+        # Options:
+        # :ocr => :ocropus | :gocr (the OCR engine to use).
+        def self.read(document, options = {:ocr => :ocropus})
+          ext = document.file.split('.')[-1]
+          if ImageExtensions.include?(ext)
+            reader = 'ocropus'
+          else
+            reader = ext
+          end
+          begin
+            r = Treat::Formatters::Readers.const_get(cc(reader))
+          rescue NameError
+            raise Treat::Exception,
+            "Cannot find a default reader for format: '#{ext}'."
+          end
+          document = r.read(document, options)
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/readers/gocr.rb

@@ -0,0 +1,24 @@
+module Treat
+  module Formatters
+    module Readers
+      # A wrapper class for the GOCR engine.
+      # 
+      # "GOCR is an OCR (Optical Character Recognition) 
+      # program, developed under the GNU Public License. 
+      # It converts scanned images of text back to text files."
+      # 
+      # Project site: http://jocr.sourceforge.net
+      class GOCR
+        # Read a file using the GOCR reader.
+        def self.read(document, options = {})
+          create_temp_file(:pgm) do |tmp|
+            `convert #{document.file} #{tmp}`
+            f = `gocr #{tmp}`.strip
+            document << Treat::Entities::Entity.from_string(f)
+          end
+          document
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/readers/html.rb

@@ -0,0 +1,13 @@
+module Treat
+  module Formatters
+    module Readers
+      class HTML
+        def self.read(document, options = {})
+          f = File.read(document.file)
+          document << Treat::Entities::Entity.from_string(f)
+          document.clean(:html)
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/readers/ocropus.rb

@@ -0,0 +1,31 @@
+module Treat
+  module Formatters
+    module Readers
+      # This class is a wrapper for the Google Ocropus
+      # optical character recognition (OCR) engine.
+      # 
+      # "OCRopus(tm) is a state-of-the-art document 
+      # analysis and OCR system, featuring pluggable 
+      # layout analysis, pluggable character recognition,
+      # statistical natural language modeling, and multi-
+      # lingual capabilities."
+      # 
+      # Original paper:
+      # Breuel, Thomas M. The Ocropus Open Source OCR System. 
+      # DFKI and U. Kaiserslautern, Germany.
+      class Ocropus
+        #  Read a file using the Google Ocropus reader.
+        def self.read(document, options = {})
+          create_temp_file(:txt) do |tmp|
+            capture(:stderr) do
+              `ocropus page #{document.file} > #{tmp} -STDIO 2>/dev/null`
+            end
+            f = File.read(tmp)
+            document << Treat::Entities::Entity.from_string(f)
+          end
+          document
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/readers/pdf.rb

@@ -0,0 +1,17 @@
+module Treat
+  module Formatters
+    module Readers
+      class PDF
+        require 'fileutils'
+        # Read a file using the Poppler pdf2text utility.
+        def self.read(document, options = {})
+          create_temp_file(:txt) do |tmp|
+            `pdftotext #{document.file} #{tmp} `.strip
+            document << Treat::Entities::Entity.from_string(File.read(tmp))
+          end
+          document
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/readers/txt.rb

@@ -0,0 +1,15 @@
+module Treat
+  module Formatters
+    module Readers
+      # This class simply reads a plain text file.
+      class Txt
+        # Build an entity from a string in plain text format.
+        def self.read(document, options = {})
+          f = File.read(document.file)
+          document << Treat::Entities::Entity.from_string(f)
+          document
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/serializers/xml.rb

@@ -0,0 +1,48 @@
+module Treat
+  module Formatters
+    module Serializers
+      # This class converts an entity to XML format.
+      class XML
+        # Reauire the Nokogiri XML parser.
+        require 'nokogiri'
+        # Serialize an entity tree in XML format.
+        def self.serialize(entity, options = {})
+          options = {:indent => 0} if options[:indent].nil?
+          if options[:indent] == 0
+            string = '<?xml version="1.0" encoding="UTF-8" standalone="no" ?>'
+          else
+            string = ''
+          end
+          spaces = ''
+          options[:indent].times { spaces << ' ' }
+          attributes = ''
+          if !entity.features.nil? && entity.features.size != 0
+            attributes = ' '
+            entity.features.each_pair do |feature, value|
+              if value.is_a? Entities::Entity
+                attributes << "#{feature}='#{value.id}' "
+              else
+                attributes << "#{feature}='#{value}' "
+              end
+            end
+            entity.edges.each_pair do |id,edge|
+              attributes << "#{edge}='#{id}' "
+            end
+          end
+          tag = entity.class.to_s.split('::')[-1].downcase
+          string += "\n#{spaces}<#{tag}#{attributes[0..-2]}>"
+          if entity.has_children?
+            options[:indent] += 1
+            entity.children.each do |child| 
+              string = string + serialize(child, options)
+            end
+            options[:indent] -= 1
+          else
+            string = string + "\n#{spaces}#{entity.value}"
+          end
+          string + "\n#{spaces}</#{tag}>"
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/serializers/yaml.rb

@@ -0,0 +1,15 @@
+module Treat
+  module Formatters
+    module Serializers
+      # Require the Psych YAML serializer.
+      require 'psych'
+      # This class serializes entities in YAML format.
+      class YAML
+        # Serialize an entity in YAML format.
+        def self.serialize(entity, options = {})
+          ::Psych.dump(entity)
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/serializers/yaml/helper.rb

@@ -0,0 +1,96 @@
+require 'yaml'
+require 'set'
+
+class Class
+  def persist
+    @persist = [] if !@persist
+    @persist
+  end
+  
+  def persist= p
+    @persist = p if p.kind_of?(Array)
+  end
+  
+  def persist_with_parent
+    p = []
+    klass = self;
+    while klass
+      p.concat(klass.persist)
+      klass = klass.superclass
+    end
+    p.uniq
+  end
+end
+
+class Object
+  def self.persistent *var
+    for i in (0..var.length-1)
+      var[i] = var[i].to_s
+    end
+    self.persist.concat(var)
+    self.persist.uniq!    
+  end
+  
+  alias_method :old_to_yaml, :to_yaml
+  
+  def to_yaml ( opts = {} )       
+    p = self.class.persist_with_parent
+
+    if p && p.size > 0
+      yaml_emit opts do |map|
+        p.each do |m|
+          map.add( m, instance_variable_get( '@' + m ) )
+        end
+      end
+    else
+      old_to_yaml opts
+    end
+  end
+private
+  def yaml_emit opts
+    YAML::quick_emit( object_id, opts ) do |out|
+      out.map( taguri, to_yaml_style ) do |map|
+        yield map
+      end
+    end
+  end
+end
+
+module RHNH
+  module EnumerablePostDeserializeHelper
+    def post_deserialize
+      self.each do |e|
+        YAML.call_post_deserialize(e) if e
+      end
+    end
+  end
+end
+
+class Array
+  include RHNH::EnumerablePostDeserializeHelper
+end
+
+class Hash
+  include RHNH::EnumerablePostDeserializeHelper
+end
+
+
+module YAML
+  def YAML.call_post_deserialize obj, object_map = ::Set.new
+    if !object_map.include?(obj.object_id)
+      object_map.add(obj.object_id)
+      
+      obj.instance_variables.each do |v|
+        call_post_deserialize obj.instance_variable_get(v), object_map
+      end
+
+      obj.post_deserialize if obj.respond_to?('post_deserialize')
+    end
+  end
+  
+  def YAML.load( io )
+		yp = parser.load( io )
+		call_post_deserialize yp
+		yp
+	end
+end

data/lib/treat/formatters/unserializers/autoselect.rb

@@ -0,0 +1,19 @@
+module Treat
+  module Formatters
+    module Unserializers
+      class Autoselect
+        def self.unserialize(document, options = {})
+          ext = document.file.split('.')[-1]
+          if ext == 'yaml' || ext == 'yml'
+            document.unserialize(:yaml)
+          elsif ext == 'xml'
+            document.unserialize(:xml)
+          else
+            raise "File #{document.file} was not recognized"+
+            "as a supported serialized format."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/formatters/unserializers/xml.rb

@@ -0,0 +1,79 @@
+module Treat
+  module Formatters
+    module Unserializers
+      class XML
+        require 'nokogiri'
+        
+        def self.unserialize(document, options = {})
+          # Read in the XML file.
+          xml = File.read(document.file)
+          xml_reader = Nokogiri::XML::Reader.from_memory(xml)
+          current_element = nil
+          previous_depth = 0
+          
+          # Read the XML file entity by entity.
+          while xml_reader.read
+            # The depth in the XML tree.
+            current_depth = xml_reader.depth
+            # If we are at the end of the children stack, pop up.
+            if previous_depth > current_depth && current_depth != 0
+              current_element = current_element.parent
+            end
+            # If an end element has been reached,
+            # change the depth and pop up on next
+            # iteration.
+            if xml_reader.node_type ==
+              Nokogiri::XML::Reader::TYPE_END_ELEMENT
+              previous_depth = current_depth
+              next
+            end
+            
+            id = nil; value = ''
+            attributes = {}; edges = {}
+            xml_reader.attributes.each_pair do |k,v|
+              if k == 'id'
+                id = v
+              elsif k == 'edges'
+                edges = v
+              elsif k == 'value'
+                value = v
+              else
+                attributes[k.intern] = v
+              end
+            end
+
+            current_value = ''
+            type = xml_reader.name.intern
+
+            if Treat::Entities.list.include?(type)
+              if !current_element
+                current_element = self.revive(type, current_value, id)
+              else
+                current_element = current_element <<
+                self.revive(type, current_value, id)
+              end
+              current_element.features = attributes
+              current_element.features = attributes
+              current_element.edges = edges
+            else
+              current_value = xml_reader.value.strip
+              if current_value && current_value != ''
+                current_element.value = current_value
+              end
+            end
+  
+            previous_depth = current_depth
+          end
+          document << current_element
+          document
+        end
+        
+        def self.revive(type, value, id)
+          klass = Treat::Entities.const_get(cc(type))
+          klass.new(value, id)
+        end
+        
+      end
+    end
+  end
+end