treat 0.2.5 → 1.0.0

data/lib/treat/formatters.rb

@@ -1,37 +1,41 @@
-module Treat
-  # Formatters handle conversion of Entities to and from
-  # external file formats.
-  module Formatters
-    # Readers read a document and create the top-level entity
-    # corresponding to the content of the document.
-    module Readers
-      extend Group
-      self.type = :transformer
-      self.targets = [:collection, :document]
-      self.default = :autoselect
-    end
-    # Serializers transform entities into a storable format.
-    module Serializers
-      extend Group
-      self.type = :computer
-      self.targets = [:entity]
-      self.default = :yaml
-    end
-    # Unserializers recreate entities from a serialized format.
-    module Unserializers
-      extend Group
-      self.type = :transformer
-      self.targets = [:collection, :document]
-      self.default = :autoselect
-    end
-    # Visualizers transform entities into a visualizable format.
-    module Visualizers
-      extend Group
-      self.type = :computer
-      self.targets = [:entity]
-      self.default = :tree
-    end
-    extend Treat::Category
+# Formatters handle conversion of Entities to and from
+# external file formats.
+module Treat::Formatters
+  
+  # Readers read a document's content.
+  module Readers
+    extend Treat::Groupable
+    self.type = :computer
+    self.targets = [:document]
   end
+  
+  # Unserializers recreate entities 
+  # from a serialized format.
+  module Unserializers
+    extend Treat::Groupable
+    self.type = :computer
+    self.targets = [:entity]
+  end
+  
+  # Serializers transform entities 
+  # into a storable format.
+  module Serializers
+    extend Treat::Groupable
+    self.type = :computer
+    self.targets = [:entity]
+    self.default = :yaml
+  end
+  
+  # Visualizers transform entities 
+  # into a visualizable format.
+  module Visualizers
+    extend Treat::Groupable
+    self.type = :computer
+    self.targets = [:entity]
+    self.default = :tree
+  end
+  
+  # Make Formatters categorizable.
+  extend Treat::Categorizable
+  
 end
-

data/lib/treat/formatters/readers/abw.rb

@@ -1,33 +1,53 @@
-module Treat
-  module Formatters
-    module Readers
-      class Abw
-        require 'rexml/document'
-        require 'rexml/streamlistener'
-        def self.read(document, options = {})
-          xml_h = AbiWordXmlHandler.new
-          REXML::Document.parse_stream(IO.read(document.file), xml_h)
-          document << Treat::Entities::Entity.from_string(xml_h.plain_text)
-          document
-        end
-        class AbiWordXmlHandler
-          include REXML::StreamListener
-          attr_reader :plain_text
-          def initialize
-            @plain_text = ""
-          end
-          def text(s)
-            if s != 'AbiWord' && s != 'application/x-abiword'
-              s.strip!
-              if s.length > 0
-                s += ' ' 
-                s += "\n\n" if s.length < 60
-              end
-              @plain_text << s
-            end
-          end
+# A wrapper for a small utility written
+# by Mark Watson to read AbiWord files.
+# Released under the GPL.
+#
+# Original project website:
+# http://www.markwatson.com/opensource/
+#
+# Todo: reimplement with Nokogiri and use
+# XML node information to better translate
+# the format of the text.
+class Treat::Formatters::Readers::ABW
+
+  silence_warnings do
+    require 'rexml/document'
+    require 'rexml/streamlistener'
+  end
+  
+  # Extract the readable text from an AbiWord file.
+  #
+  # Options: none.
+  def self.read(document, options = {})
+
+    xml_h = ABWXmlHandler.new
+    REXML::Document.parse_stream(
+    IO.read(document.file), xml_h)
+
+    document.value = xml_h.plain_text
+    document.set :format, :abw_word
+    document
+
+  end
+
+  # Helper class to parse the AbiWord file.
+  class ABWXmlHandler
+    include REXML::StreamListener
+    attr_reader :plain_text
+    def initialize
+      @plain_text = ""
+    end
+    def text(s)
+      if s != 'AbiWord' && s !=
+        'application/x-abiword'
+        s.strip!
+        if s.length > 0
+          s += ' '
+          s += "\n\n" if s.length < 45
         end
+        @plain_text << s
       end
     end
   end
-end
+
+end

data/lib/treat/formatters/readers/autoselect.rb

@@ -1,35 +1,39 @@
-module Treat
-  module Formatters
-    module Readers
-      # This class isn't a wrapper for anything.
-      # It simply delegates the reading task to
-      # the appropriate reader based on the file
-      # extension of the supplied document.
-      class Autoselect
-        # A list of image extensions that should be routed to Ocropus.
-        ImageExtensions = ['gif', 'jpg', 'jpeg', 'png']
-        # Select the appropriate reader based on the format
-        # of the filename in document.
-        # 
-        # Options:
-        #
-        # - :ocr_engine => :ocropus or :gocr (the OCR engine to use).
-        def self.read(document, options)
-          ext = document.file.split('.')[-1]
-          reader = ImageExtensions.include?(ext) ? 'image' : ext
-          reader = 'html' if reader == 'htm'
-          reader = 'yaml' if reader == 'yml'
-          begin
-            r = Treat::Formatters::Readers.const_get(cc(reader))
-          rescue NameError
-            raise Treat::Exception,
-            "Cannot find a reader for format: '#{ext}'."
-          end
-          document = r.read(document, options)
-          document.set :encoding, document.to_s.encoding.to_s.downcase
-          document
-        end
-      end
-    end
+class Treat::Formatters::Readers::Autoselect
+  
+  ExtensionRegexp = /^.*?\.([a-zA-Z0-9]{2,5})$/
+  ImageExtensions = ['gif', 'jpg', 'jpeg', 'png']
+  DefaultOptions = {
+    :default_to => :txt
+  }
+  
+  # Choose a reader to use.
+  #
+  # Options:
+  #  - (Symbol) :default_to => format to default to.
+  def self.read(document, options = {})
+    options = DefaultOptions.merge(options)
+    document.read(detect_format(document.file, options[:default_to]))
   end
+  
+  def self.detect_format(filename, default_to = DefaultOptions[:default_to])
+
+    ext = filename.scan(ExtensionRegexp)
+    ext = (ext.is_a?(Array) && ext[0] && ext[0][0]) ?
+    ext[0][0] : ''
+
+    format =
+    ImageExtensions.include?(ext) ?
+    'image' : ext
+
+    # Humanize extensions.
+    format = 'html' if format == 'htm'
+    format = 'yaml' if format == 'yml'
+
+    format = default_to if format == ''
+
+    format.intern
+
+  end
+
+
 end

data/lib/treat/formatters/readers/doc.rb

@@ -1,15 +1,21 @@
-module Treat
-  module Formatters
-    module Readers
-      class Doc
-        def self.read(document, options = {})
-          f = `antiword #{document.file}`
-          f.gsub!("\n\n", '#keep#')
-          f.gsub!("\n", ' ')
-          f.gsub!('#keep#', "\n\n")
-          document << Treat::Entities::Entity.from_string(f)
-        end
-      end
-    end
+# A wrapper for the 'antiword' command-line utility.
+class Treat::Formatters::Readers::DOC
+  
+  # Extract the readable text from a DOC file
+  # using the antiword command-line utility.
+  #
+  # Options: none.
+  def self.read(document, options = {})
+    
+    f = `antiword #{document.file}`
+    f.gsub!("\n\n", '#keep#')
+    f.gsub!("\n", ' ')
+    f.gsub!('#keep#', "\n\n")
+    
+    document.value = f
+    document.set :format, :doc
+    document
+    
   end
+  
 end

data/lib/treat/formatters/readers/html.rb

@@ -1,33 +1,55 @@
-module Treat
-  module Formatters
-    module Readers
-      # A temporary HTML reader; simply strips the
-      # document of all of its markup.
-      class HTML
-        # Require Hpricot.
-        silence_warnings { require 'hpricot' }
-        # By default, backup the HTML text while cleaning.
-        DefaultOptions = { :clean => true, :backup => false }
-        # Read the HTML document and strip it of its markup.
-        #
-        # Options:
-        #
-        # - (Boolean) :clean => whether to strip HTML markup.
-        # - (Boolean) :backup => whether to backup the HTML 
-        #   markup while cleaning.
-        def self.read(document, options = {})
-          options = DefaultOptions.merge(options)
-          f = File.read(document.file)
-          document << Treat::Entities::Entity.from_string(f)
-          if options[:clean]
-            document.each do |section|
-              section.set :html_value, section.value if options[:backup]
-              section.value = Hpricot(section.value).inner_text
-            end
-          end
-          document
-        end
-      end
+# This class is a wrapper for the 'ruby-readability'
+# gem, which extracts the primary readable content
+# of a web page by using set of handwritten rules.
+#
+# Project homepage:
+# https://github.com/iterationlabs/ruby-readability
+class Treat::Formatters::Readers::HTML
+
+  silence_warnings { require 'ruby-readability' }
+
+  # By default, don't backup the original HTML
+  DefaultOptions = {
+    :keep_html => false,
+    :tags => %w[p div h1 h2 h3 ul ol dl dt li]
+  }
+
+  # Read the HTML document and strip it of its markup.
+  #
+  # Options:
+  #
+  #   text when cleaning the document (default: false).
+  # - (Boolean) :remove_empty_nodes => remove <p> tags
+  #   that have no text content
+  # - (String) :encoding => if the page is of a known
+  #   encoding, you can specify it; if left unspecified,
+  #   the encoding will be guessed (only in Ruby 1.9.x)
+  # - (String) :html_headers => in Ruby 1.9.x these will
+  #   be passed to the guess_html_encoding gem to aid with
+  #   guessing the HTML encoding.
+  # - (Array of String) :tags  => the base whitelist of
+  #   tags to sanitize, defaults to %w[div p].
+  #   also removes p tags that contain only images
+  # - (Array of String) :attributes => list allowed attributes
+  # - (Array of String) :ignore_image_format => for use with images.
+  # - (Numeric) :min_image_height => minimum image height for images.
+  # - (Numeric) :min_image_width => minimum image width for images.
+  def self.read(document, options = {})
+
+    # set encoding with the guess_html_encoding
+    options = DefaultOptions.merge(options)
+    html = File.read(document.file)
+
+    silence_warnings do
+      # Strip comments
+      html.gsub!(/<!--[^>]*-->/m, '')
+      d = Readability::Document.new(html, options)
+      document.value = "<h1>#{d.title}</h1>\n" + d.content
+      document.set :format, :html
     end
+    
+    document
+
   end
+
 end

data/lib/treat/formatters/readers/image.rb

@@ -1,43 +1,44 @@
-module Treat
-  module Formatters
-    module Readers
-      # This class is a wrapper for the Google Ocropus
-      # optical character recognition (OCR) engine.
-      #
-      # "OCRopus(tm) is a state-of-the-art document
-      # analysis and OCR system, featuring pluggable
-      # layout analysis, pluggable character recognition,
-      # statistical natural language modeling, and multi-
-      # lingual capabilities."
-      #
-      # Original paper:
-      # Breuel, Thomas M. The Ocropus Open Source OCR System.
-      # DFKI and U. Kaiserslautern, Germany.
-      class Image
-        #  Read a file using the Google Ocropus reader.
-        #
-        # Options:
-        # - (Boolean) :silent => whether to silence Ocropus.
-        def self.read(document, options = {})
-          read = lambda do |doc|
-            create_temp_dir do |tmp|
-              `ocropus book2pages #{tmp}/out #{doc.file}`
-              `ocropus pages2lines #{tmp}/out`
-              `ocropus lines2fsts #{tmp}/out`
-              `ocropus buildhtml #{tmp}/out > #{tmp}/output.html`
-              f = document.file
-              doc.remove_all!
-              doc.set :file,  "#{tmp}/output.html"
-              doc.read(:html)
-              doc.set :file, f
-            end
-          end
-          options[:silent] ? 
-          silence_stdout { read.call(document) } :
-          read.call(document)
-          document
-        end
+# This class is a wrapper for the Google Ocropus
+# optical character recognition (OCR) engine.
+#
+# "OCRopus(tm) is a state-of-the-art document
+# analysis and OCR system, featuring pluggable
+# layout analysis, pluggable character recognition,
+# statistical natural language modeling, and multi-
+# lingual capabilities."
+#
+# Original paper:
+#
+# Breuel, Thomas M. The Ocropus Open Source OCR System.
+# DFKI and U. Kaiserslautern, Germany.
+class Treat::Formatters::Readers::Image
+
+  #  Read a file using the Google Ocropus reader.
+  #
+  # Options:
+  #
+  # - (Boolean) :silent => whether to silence Ocropus.
+  def self.read(document, options = {})
+    
+    read = lambda do |doc|
+      create_temp_dir do |tmp|
+        `ocropus book2pages #{tmp}/out #{doc.file}`
+        `ocropus pages2lines #{tmp}/out`
+        `ocropus lines2fsts #{tmp}/out`
+        `ocropus buildhtml #{tmp}/out > #{tmp}/output.html`
+        doc.set :file,  "#{tmp}/output.html"
+        doc = doc.read(:html)
+        doc.set :file, f
+        doc.set :format, :image
       end
     end
+    
+    options[:silent] ?
+    silence_stdout { read.call(document) } :
+    read.call(document)
+    
+    document
+    
   end
-end
+  
+end

data/lib/treat/formatters/readers/odt.rb

@@ -1,50 +1,64 @@
-module Treat
-  module Formatters
-    module Readers
-      # A reader for the ODT (Open Office) document format.
-      #
-      # Based on work by Mark Watson, licensed under the GPL.
-      # Original project website: http://www.markwatson.com/opensource/
-      class Odt
-        # Require the 'zip' gem to unarchive the ODT files
-        silence_warnings { require 'zip' }
-        # Build an entity from an ODT file.
-        def self.read(document, options = {})
-          f = nil
-          Zip::ZipFile.open(document.file, Zip::ZipFile::CREATE) do |zipfile|
-            f = zipfile.read('content.xml')
-          end
-          raise "Couldn't unzip dot file #{document.file}!" unless f
-          xml_h = OOXmlHandler.new
-          REXML::Document.parse_stream(f, xml_h)
-          document << Treat::Entities::Entity.from_string(xml_h.plain_text)
-          document
-        end
-        # Xml listener for the parsing of the ODT file.
-        class OOXmlHandler
-          require 'rexml/document'
-          require 'rexml/streamlistener'
-          include REXML::StreamListener
-          attr_reader :plain_text
-          def initialize
-            @plain_text = ""
-            @last_name = ""
-          end
-          def tag_start(name, attrs)
-            @last_name = name
-          end
-          def text(s)
-            if @last_name.index('text')
-              s = s.strip
-              if s.length > 0
-                @plain_text << s
-                @plain_text << "\n\n"
-              end
-            end
-          end
+# A reader for the ODT (Open Office)
+# document format.
+#
+# Based on work by Mark Watson,
+# licensed under the GPL.
+#
+# Original project website:
+# http://www.markwatson.com/opensource/
+#
+# Todo: reimplement with Nokogiri and use
+# XML node information to better translate
+# the format of the text.
+class Treat::Formatters::Readers::ODT
+
+  # Require the 'zip' gem to unarchive the ODT files
+  silence_warnings { require 'zip' }
+
+  # Extract the readable text from an ODT file.
+  #
+  # Options: none.
+  def self.read(document, options = {})
+    f = nil
+    Zip::ZipFile.open(document.file,
+    Zip::ZipFile::CREATE) do |zipfile|
+      f = zipfile.read('content.xml')
+    end
+    raise "Couldn't unzip dot file " +
+    "#{document.file}!" unless f
+    xml_h = ODTXmlHandler.new
+    REXML::Document.parse_stream(f, xml_h)
+
+    document.value = xml_h.plain_text
+    document.set :format, :odt_office
+    document
+
+  end
+
+  # Xml listener for the parsing of the ODT file.
+  class ODTXmlHandler
+    silence_warnings do
+      require 'rexml/document'
+      require 'rexml/streamlistener'
+    end
+    include REXML::StreamListener
+    attr_reader :plain_text
+    def initialize
+      @plain_text = ""
+      @last_name = ""
+    end
+    def tag_start(name, attrs)
+      @last_name = name
+    end
+    def text(s)
+      if @last_name.index('text')
+        s = s.strip
+        if s.length > 0
+          @plain_text << s
+          @plain_text << "\n\n"
         end
       end
-
     end
   end
+
 end