documentalist 0.1.0 → 0.1.1

data/lib/backends/open_office/bridges/jodconverter-2.2.2/lib/DEPENDENCIES.txt

@@ -0,0 +1,17 @@
+To use the library in your own Java app you need
+
+ * commons-io
+ * jodconverter
+ * juh
+ * jurt
+ * ridl
+ * slf4j-api
+ * slf4j-jdk14 or another slf4j implementation - see http://slf4j.org
+ * unoil
+ * xstream - only if you use XmlDocumentFormatRegistry
+
+The command line interface additionally requires
+
+ * commons-cli
+ * jodconverter-cli
+

data/lib/{open_office → backends/open_office}/server.rb

@@ -1,9 +1,10 @@
-require 'timeout'
+require 'system_timer'
 require 'tmpdir'
 
 module OpenOffice
   module Server
     # Path to the Python executable
+    # TODO : Eww, use config instead
     PYTHON_PATH = "/usr/bin/python"
 
     # Server can convert from the following file formats
@@ -13,6 +14,7 @@ module OpenOffice
     CONVERT_TO = [:odt, :doc, :rtf, :pdf, :txt, :html, :htm, :wps]
 
     # Python conversion script path
+    # TODO : Wrong
     PY_OD_CONVERTER = File.join(File.dirname(__FILE__), "../DocumentConverter.py")
 
     # Maximum allowed CPU usage for an OpenOffice process
@@ -22,6 +24,7 @@ module OpenOffice
     SERVER_START_DELAY = 4
 
     # Log file
+    # TODO : Wrong
     LOG_FILE = Object.const_defined?(:RAILS_ROOT) ? File.join(RAILS_ROOT, "log", "openoffice.log") : ""
 
     def self.convert(origin, options = {:to => :txt})
@@ -37,7 +40,7 @@ module OpenOffice
           raise "Can't convert #{origin} to #{options[:to]}"
         end
 
-        timeout(10, :attempts => 2) do
+        Documentalist.timeout(10, :attempts => 2) do
           system("#{PYTHON_PATH} #{PY_OD_CONVERTER} #{origin} #{destination} > /dev/null 2>&1")
 
           # HACK : sometimes text files get saved in ISO-8859-1 instead of regular UTF-8, so we force
@@ -70,7 +73,7 @@ module OpenOffice
       system("/usr/bin/soffice -headless -accept=\"socket,host=127.0.0.1,port=8100;urp;\" -nofirststartwizard -nologo -nocrashreport -norestore -nolockcheck -nodefault #{">>" unless LOG_FILE.empty?} #{LOG_FILE} 2>&1 &")
 
       begin
-        timeout(2) do
+        SystemTimer.timeout(2.seconds) do
           while !running?
             print "."
           end
@@ -90,7 +93,7 @@ module OpenOffice
       raise "Not running!" unless running?
 
       begin
-        timeout(3, :attempts => 2) do
+        Documentalist.timeout(3, :attempts => 2) do
           while(running?)
             system("pkill -9 office")
           end
@@ -129,7 +132,7 @@ module OpenOffice
       if block_given?
         attempts = options[:attempts] || 1
         begin
-          Timeout::timeout(max_time) do
+          SystemTimer.timeout(max_time) do
             yield
           end
         rescue Timeout::Error

data/lib/backends/pdf_tools.rb

@@ -0,0 +1,14 @@
+# To change this template, choose Tools | Templates
+# and open the template in the editor.
+
+module Documentalist
+  module PdfTools
+    def self.convert(origin, options)
+      if system("pdftotext #{origin} #{options[:destination]} > /dev/null 2>&1")
+        options[:destination]
+      else
+        raise "PdfTools failed"
+      end
+    end
+  end
+end

data/lib/documentalist.rb

@@ -1,63 +1,151 @@
 require 'rubygems'
-require 'erb'
-require 'fileutils'
-require 'tmpdir'
-require 'zip/zip'
-require 'open_office/server'
+require 'yaml'
+require 'system_timer'
+require 'logger'
+
+# Require all backends
+Dir.glob(File.join(File.dirname(__FILE__), 'backends', '*.rb')).each do |backend|
+  require backend
+end
 
 module Documentalist
-  def self.merge(str, options = {})
-    locals = options[:locals]
+  @@config = {}
+  @@logger = nil
 
-    if locals and locals.is_a? Hash
-      locals.each do |k,v|
-        instance_variable_set("@#{k.to_s}".to_sym, v)
-      end
-    end
+  def self.config
+    default_config! unless config?
+    @@config
+  end
 
-    ERB.new(str).result(binding)
+  def self.config=(hash)
+    # We want to symbolize keys ourselves since we're not depending on Active Support
+    @@config = symbolize hash
   end
 
-  def self.get_contents(odt_file)
-    contents = ""
-    Zip::ZipFile.open(odt_file) { |zip| contents = zip.read("content.xml") }
-    contents.gsub("&lt;%", "<%").gsub("%&gt;", "%>")
+  def self.config?
+    @@config != {}
   end
 
-  def self.merge_template(template, options = {})
-    # Get template contents
-    tmp_contents= Tempfile.new("officer-contents")
-    tmp_contents.write(merge(get_contents(template), :locals => options[:locals]))
-    tmp_contents.close
+  def self.default_config!
+    config_from_yaml! File.join(File.dirname(__FILE__), %w{.. config default.yml})
+  end
+
+  def self.config_from_yaml!(file, options = {})
+    self.config = YAML::load(File.open(file))
+    self.config = config[options[:section].to_sym] if options[:section]
+  end
+
+  BACKENDS = {
+    OpenOffice => {[:odt, :doc, :rtf, :docx, :txt, :html, :htm, :wps] => [:odt, :doc, :rtf, :pdf, :txt, :html, :htm, :wps]},
+    NetPBM => {:ppm => [:jpg, :jpeg]},
+    PdfTools => {:pdf => :txt},
+    
+    # Find a better pattern to pick backend, this one smells pretty bad
+    # WkHTML2PDF => {[:html, :htm] => :pdf}
+  }
+
+  # Finds the relevant server to perform the conversion
+  def self.backend_for_conversion(origin, destination)
+    origin = origin.to_s.gsub(/.*\./, "").to_sym
+    destination = destination.to_s.gsub(/.*\./, "").to_sym
+
+    BACKENDS.detect do |s, conversions|
+      conversions.keys.flatten.include?(origin) and conversions.values.flatten.include?(destination)
+    end.to_a.first
+  end
+
+  # Takes all conversion requests and dispatches them appropriately
+  def self.convert(file, options={})
+    raise "#{file} does not exist !" unless File.exist?(file)
+
+    unless options[:to] or options[:to_format]
+      raise Documentalist::Error.new("No destination or format was given")
+    end
+    
+    # Convert to plain text by default
+    options[:to_format] = options[:to_format] ? options[:to_format].to_sym : :txt
+
+    unless options[:to]
+      options[:to] = file.gsub(/#{"\\" + File.extname(file)}$/, ".#{options[:to_format].to_s}")
+    end
+
+    options[:from_format] = File.extname(file).gsub(/\./, "").to_sym
 
-    # Copy the template so we can merge the data into the copy
-    tmp_merged_template = File.join(Dir.tmpdir, "merged-template-#{rand(10**9)}#{File.extname(template)}")
-    FileUtils.cp(template, tmp_merged_template)
+    backend = backend_for_conversion(options[:from_format], options[:to_format])
+    converted = backend.convert(file, options)
 
-    # Stuff the merged contents.xml into the OpenDocument zip
-    Zip::ZipFile.open(tmp_merged_template) do |zip|
-      zip.replace("content.xml", tmp_contents.path)
-      zip.commit
+    yield(converted) if block_given?
+    converted
+  end
+
+  def self.extract_text(file)
+    converted = convert(file, :to => :txt)
+    if converted and File.exist?(converted)
+      text = File.open(converted).read.toutf8
+      FileUtils.rm(converted)
+
+      yield(extracted_text) if block_given?
+      text
     end
+  end
+
+  def self.extract_images(file)
+    temp_dir = File.join(CONVERSIONS_PATH, (Time.new.to_f*100_000).to_i.to_s)
+    
+    if File.extname(file) == '.pdf'
+      temp_file = File.join(temp_dir, File.basename(file))
 
-    # Remove the merged contents.xml
-    tmp_contents.unlink
+      system "mkdir #{temp_dir} && cp #{file} #{temp_file}"
+      system "cd #{temp_dir} && pdfimages #{temp_file} 'img'"
 
-    # Manages the converted file depending on the context
-    if options[:to]
-      if File.extname(options[:to]) == File.extname(template)
-        FileUtils.mv(tmp_merged_template, options[:to])
-      else
-        OpenOffice::Server.convert(tmp_merged_template, options[:to])
-        FileUtils.rm(tmp_merged_template)
+      Dir.glob(File.join(temp_dir, "*.ppm")).each do |ppm_image|
+        Documentalist.convert(ppm_image, :to => :jpeg)
       end
     else
-      FileUtils.rm(template)
-      FileUtils.mv(tmp_merged_template, template)
+      convert file, :to => :html, :directory => temp_dir
     end
+
+    image_file_names = Dir.glob(File.join(temp_dir, "*.{jpg,jpeg,bmp,tif,tiff,gif,png}"))
+
+    yield(image_file_names) if block_given?
+    image_file_names
   end
 
-  def self.convert(from, to)
-    OpenOffice::Server.convert(from, :to => to)
+  # Runs a block with a system-enforced timeout and optionally retry with an
+  # optional sleep between attempts of running the given block.
+  # All times are in seconds.
+  def self.timeout(time_limit = 0, options = {:attempts => 1, :sleep => nil})
+    if block_given?
+      attempts = options[:attempts] || 1
+      begin
+        SystemTimer.timeout time_limit do
+          yield
+        end
+      rescue Timeout::Error
+        attempts -= 1
+        sleep(options[:sleep]) if options[:sleep]
+        retry unless attempts.zero?
+        raise
+      end
+    end
   end
+
+  def self.logger
+    unless @@logger
+      @@logger = Logger.new(Documentalist.config[:log_file])
+      @@logger.level = Logger.const_get(config[:log_level] ? config[:log_level].upcase : "WARN")
+    end
+
+    @@logger
+  end
+
+  # Returns a new hash with recursively symbolized keys
+  def self.symbolize(hash)
+    hash.each_key do |key|
+      hash[key.to_sym] = hash.delete key
+      hash[key.to_sym] = symbolize(hash[key.to_sym]) if hash[key.to_sym].is_a?(Hash)
+    end
+  end
+
+  class Error < RuntimeError; end
 end  

data/lib/tasks/tasks.rb

@@ -0,0 +1,6 @@
+namespace :documentalist do
+  desc "This task checks that the required binaries and libraries are available on the target system"
+  task :check_install do
+    puts "Checking install..."
+  end
+end

data/rails/config/documentalist.yml.tpl

@@ -0,0 +1,67 @@
+# Sample Rails configuration file
+# Optional settings are :
+#  * log_path : Allows you to override the default log file [Rail.root/log/documentalist.log]
+
+development:
+  # Python configuration
+  python:
+    path: /usr/bin/python
+
+  # Java configuration
+  java:
+    path: /usr/bin/java
+
+  # OpenOffice configuration
+  open_office:
+    # Path to the OpenOpffice binary
+    path: /usr/bin/soffice
+
+    # Select desired bridge between PYOD and JOD
+    bridge: JOD
+
+    # Maximum allowed CPU usage before the process is considered stalled
+    max_cpu: 80
+
+    # OpenOffice server allowed startup time (seconds)
+    max_startup_time: 4
+
+    # OpenOffice server allowed waking up time (between startup and actual processing)
+    wakeup_time: 3
+
+    # Conversion tries before giving up
+    max_conversion_attempts: 3
+
+    # Maximum allowed time for converting a document
+    max_conversion_time: 6
+
+test:
+  python:
+    path: /usr/bin/python
+
+  java:
+    path: /usr/bin/java
+
+  open_office:
+    path: /usr/bin/soffice
+    bridge: JOD
+    max_cpu: 80
+    max_startup_time: 4
+    wakeup_time: 3
+    max_conversion_attempts: 3
+    max_conversion_time: 6
+
+production:
+  python:
+    path: /usr/bin/python
+
+  java:
+    path: /usr/bin/java
+
+  open_office:
+    path: /usr/bin/soffice
+    bridge: JOD
+    max_cpu: 80
+    max_startup_time: 4
+    wakeup_time: 3
+    max_conversion_attempts: 3
+    max_conversion_time: 6

data/rails/init.rb

@@ -1 +1,12 @@
-require 'documentalist'
+require 'yaml'
+
+require File.join(File.dirname(__FILE__), %w{initialize_configuration})
+require File.join(File.dirname(__FILE__), %w{.. lib documentalist})
+
+# Load configuration from Rails.root/config/documentalist.yml
+Documentalist.config_from_yaml! File.join(RAILS_ROOT, %w{config documentalist.yml}), :section => RAILS_ENV
+
+# Set a default for the logfile if it hasn't been provided by the configuration file
+unless Documentalist.config[:logfile]
+  Documentalist.config[:logfile] = File.join(RAILS_ROOT, %w{log documentalist-#{RAILS_ENV}.log})
+end

data/rails/initialize_configuration.rb

@@ -0,0 +1,6 @@
+# Initializes a standard configuration file in Rails.root/config/documentalist.yml
+unless File.exists?(File.join(RAILS_ROOT , %w{config documentalist.yml}))
+  FileUtils.cp(File.join(File.dirname(__FILE__), %w{config documentalist.yml.tpl}), File.join(RAILS_ROOT, %w{config documentalist.yml}))
+end
+
+

data/test/documentalist_test.rb

@@ -1,40 +1,63 @@
-require 'test/unit'
-require 'documentalist'
+require 'test_helper'
+require 'system_timer'
+require 'tmpdir'
 
 class DocumentalistTest < Test::Unit::TestCase
-  @@odt_fixture = File.join(File.dirname(__FILE__), "fixtures/fixture.odt")
+  include FlexMock::TestCase
 
-  def test_merge
-    template = "<%= @var1 %><%= 1.upto(3).map{ |n| n.to_s }.join %><%= @var2 %>"
+  # Test the custom symbolize method used as a replacement for the Active Support version
+  def test_symbolize
+    hash = { "a" => "b",
+      "c" => {
+        "d" => "e"
+      }
+    }
 
-    merged = Documentalist.merge(template, :locals => {
-        :var1 => "test",
-        :var2 => "working?"
+    symbolized = { :a => "b",
+      :c => {
+        :d => "e"
       }
-    )
+    }
+    
+    assert_equal Documentalist.send(:symbolize, hash), 
+      symbolized,
+      "Hash wasn't properly symbolized"
+  end
 
-    assert_equal "test123working?", merged, "Merge wasn't performed correctly"
+  # Test that we use a system timeout and not a green thread based timeout that
+  # could possibly not work on some long external system calls
+  def test_timeout_uses_system_timeout
+    flexmock(SystemTimer).should_receive(:timeout).once
+    Documentalist.timeout(0.1) { }
   end
 
-  def test_read_zipped_odt
-    contents = Documentalist.get_contents(@@odt_fixture)
+  # Test that we have a default configuration for Documentalist even if
+  def test_default_config
+    # Check that we did not get some Rails context from other tests
+    assert !Object.const_defined?(:RAILS_ENV)
 
-    assert_match /Hello/, contents
-    assert_match /thing/, contents
-    assert !(contents =~ /%&gt;/)
-    assert !(contents =~ /&lt;%=/)
+    # Check that at least a configuration key has been magically set
+    assert Documentalist.config[:open_office]
   end
 
-  def test_odt_merge
-    template = "#{File.join(File.dirname(__FILE__), "fixtures/fixture.odt")}"
-    result = "#{File.join(File.dirname(__FILE__), "fixtures/result.odt")}"
+  def test_logger
+    log_file = File.join(Dir.tmpdir, "#{rand(10 ** 9).to_s}.log")
+
+    Documentalist.config[:log_file] = log_file
+    assert !File.exists?(log_file), "Log file already exists"
+
+    Documentalist.logger
+    assert File.exists?(log_file), "Log file should have been created"
+
+    assert_no_difference("File.size(\"#{log_file}\")", "Nothing should have been written") do
+      Documentalist.logger.debug("This message should go nowhere")
+    end
 
-    Documentalist.merge_template(template,
-      :locals => {:thing => "world"},
-      :to => result
-    )
+    assert_difference("File.size(\"#{log_file}\")", nil, "Nothing should have been written") do
+      Documentalist.logger.warn("This message should be written !")
+    end
     
-    assert /world/, Documentalist.get_contents(result)
-    File.delete(result)
+    FileUtils.rm(log_file)
+    assert !File.exists?(log_file), "Log file hasn't been removed properly"
   end
 end