proselytism 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+spec/tmp
+test/version_tmp
+.idea
+spec/tmp/*

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in proselytism.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Itkin
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,94 @@
+# Proselytism
+
+Document converter, text and image extractor using OpenOffice headless server, pdf_tools and net_pbm
+
+## Note
+
+This gem has been originally written for as a RoR 3.2 engine running on Ruby 1.8.7.
+It should be framework agnostic and has been tested on Ubuntu and MacOSX.
+
+Due to its dependency to system_timer it doesn't work with ruby 1.9.x
+
+## Installation
+
+Install the required external librairies :
+
+    # aptitude install netpbm
+    # aptitude install xpdf
+    # aptitude install libreoffice
+
+Add this line to your application's Gemfile:
+
+    gem 'proselytism', :git => "git://github.com/itkin/proselytism.git"
+
+And then execute:
+
+    $ bundle
+
+Generate the config file or / and an initializer
+
+    $ rails g proselytism:config
+    $ rails g proselytism:initializer
+
+As an engine, Proselytism automatically load and autoconfig with /config/proselytism.yml if it exists
+You can override these configurations params with an initializer. This is especially usefull when you want a custom log file
+    
+```ruby
+#/config/initializers/proselytism.rb
+Proselytism.config do |config|
+  config.logger = ActiveSupport::BufferedLogger.new(File.join(Rails.root, 'log', 'proselytism.log'))
+end
+```
+
+## Usage
+
+```ruby
+Proselytism.convert source_file_path, :to => :pdf do |converted_file_path|
+
+end
+Proselytism.extract_text source_file_path do |extracted_text|
+
+end
+Proselytism.extract_images source_file_path do |image_files_paths|
+
+end
+```
+
+Proselytism create its converted files in temporary folders.
+  - If you pass a block to the method the folders are automatically deleted after the block is yield, so use or copy the file content within the block
+  - If you don't pass a block, don't forget to safely remove the temp folder
+
+```ruby
+pdf_file_path = Proselytism.convert source_file_path, :to => :pdf
+FileUtils.remove_entry_secure File.dirname(pdf_file_path)
+```
+    
+## Add your own converter
+
+Add your own converter by extending Proselytism::Converters::Base
+  - Your converter will be automatically selected and used related to the form and to extensions list
+  - Add a perform method which
+    - define a text command
+    - call execute
+    - return the converted file(s) path
+
+```ruby
+class MyConverter < Proselytism::Converters::Base
+  form :ext1, :ext2
+  to :ext3, :ext4
+
+  def perform(origin, options={})
+    destination = destination_file_path(origin, options)
+    command = "pdftotext #{origin} #{destination} 2>&1"
+    execute command
+    destination
+  end
+end
+```
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task :default => :spec
+task :test => :spec
+

data/lib/generators/proselytism/config_generator.rb

@@ -0,0 +1,14 @@
+require 'rails/generators'
+
+module Proselytism
+  class ConfigGenerator < Rails::Generators::Base
+
+    self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+
+    desc "Generate Proselytism config file load before rails initialize"
+    def create_config_file
+      template 'config.yml', 'config/proselytism.yml'
+    end
+
+  end
+end

data/lib/generators/proselytism/initializer_generator.rb

@@ -0,0 +1,14 @@
+require 'rails/generators'
+
+module Proselytism
+  class InitializerGenerator < Rails::Generators::Base
+
+    self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+
+
+    desc "Generate an initializer file to override the configuration file"
+    def create_initializer_file
+      template 'initializer.rb', 'config/initializers/proselytism.rb'
+    end
+  end
+end

data/lib/generators/proselytism/templates/config.yml

@@ -0,0 +1,29 @@
+test:
+  open_office_path: "/Applications/OpenOffice.org.app/Contents/MacOS/soffice"
+  oo_server_bridge: "JOD"
+  oo_server_max_cpu: 95  #percent
+  oo_server_max_cpu_delay: 2 #seconds
+  oo_server_availability_delay: 6 # seconds
+  oo_server_start_delay: 2 #seconds
+  oo_conversion_max_tries: 2
+  oo_conversion_max_time: 4 #seconds
+
+development:
+  open_office_path: "/Applications/OpenOffice.org.app/Contents/MacOS/soffice"
+  oo_server_bridge: "JOD"
+  oo_server_max_cpu: 95  #percent
+  oo_server_max_cpu_delay: 2 #seconds
+  oo_server_availability_delay: 6 # seconds
+  oo_server_start_delay: 2 #seconds
+  oo_conversion_max_tries: 2
+  oo_conversion_max_time: 4 #seconds
+
+production:
+  open_office_path: "/usr/bin/soffice"
+  oo_server_bridge: "PYOD"
+  oo_server_max_cpu: 98  #percent
+  oo_server_max_cpu_delay: 2 #seconds
+  oo_server_availability_delay: 7 #seconds
+  oo_server_start_delay: 2 #seconds
+  oo_conversion_max_tries: 2
+  oo_conversion_max_time: 5 #seconds

data/lib/generators/proselytism/templates/initializer.rb

@@ -0,0 +1,30 @@
+Proselytism.config do |config|
+  #Open Office binary path  /usr/bin/soffice for unix or /Applications/OpenOffice.org.app/Contents/MacOS/soffice for mac OSX
+  config.open_office_path = "/Applications/OpenOffice.org.app/Contents/MacOS/soffice"
+
+  #Bridge PYOD or JOD (PYOD dones't work as is on mac)
+  config.oo_server_bridge = "JOD"
+
+  #When ensuring server availability (before converting a doc),
+  #Restart the server if all processs are above max cpu during max_cpu_delay
+  config.oo_server_max_cpu = 95  #percent
+  config.oo_server_max_cpu_delay = 2 #seconds
+
+  #Time the server waits for availability before converting a doc
+  config.oo_server_availability_delay = 6 # seconds
+
+  # Wait time after server start
+  config.oo_server_start_delay = 2 #seconds
+
+
+  config.oo_conversion_max_tries = 2
+
+  #max time for performing a conversion (then restart an attempt)
+  config.oo_conversion_max_time = 5 #seconds
+
+  #Path where conversion are done by default system temp dir
+  #config.tmp_path = File.expand_path("../tmp",  __FILE__)
+
+  #Logger (otherwhise rails logger)
+  #config.logger = ActiveSupport::BufferedLogger.new("your/log/path")
+end

data/lib/proselytism.rb

@@ -0,0 +1,14 @@
+require "active_support/core_ext"
+
+require "proselytism/version"
+require "proselytism/shared"
+require "proselytism/proselytism"
+require "proselytism/converter"
+
+require "proselytism/converters/open_office"
+require "proselytism/converters/pdf_to_text"
+require "proselytism/converters/pdf_images"
+require "proselytism/converters/ppm_to_jpeg"
+
+
+require "proselytism/engine" if defined? Rails

data/lib/proselytism/converter.rb

@@ -0,0 +1,74 @@
+require 'singleton'
+require 'active_support/core_ext/class/attribute'
+require 'active_support/core_ext/module/aliasing'
+
+module Proselytism
+  module Converters
+    class Base
+      include ::Singleton
+      include Proselytism::Shared
+      class_attribute :from, :to, :subclasses
+
+      class Error < Exception; end
+
+      def config
+        Proselytism.config
+      end
+
+      def destination_file_path(origin, options={})
+        if options[:dest]
+          options[:dest]
+        else
+          File.join options[:dir], File.basename(origin).gsub(/\..*$/, options[:folder] ? '' : ".#{options[:to]}")
+        end
+      end
+
+      #call perform logging duration and potential errors
+      def convert(file_path, options={})
+        log :debug, "convert #{file_path} to :#{options[:to]}" do
+          begin
+            perform(file_path, options)
+          rescue Error => e
+            log :error, e.message
+            raise e
+          end
+        end
+      end
+
+      #execute a command and raise error with the command output if it fails
+      def execute(command)
+        output = `#{command}`
+        if $?.exitstatus != 0
+          raise self.class::Error, ["#{self.class.name} unable to exec command: #{command}",'--', output,'--'].join("\n")
+        end
+        $?.exitstatus == 0
+      end
+
+      singleton_class.class_eval do
+
+        def inherited_with_registering(subclass)
+          self.subclasses ||= []
+          self.subclasses << subclass
+          inherited_without_registering(subclass)
+          subclass
+        end
+
+        alias_method_chain :inherited, :registering
+
+        [:from, :to].each do |attr|
+          define_method "#{attr}_with_default" do |*formats|
+            if formats.length
+              self.send "#{attr}=", formats.map(&:to_sym)
+            else
+              self.send "#{attr}_without_default" || []
+            end
+          end
+          alias_method_chain attr, :default
+        end
+      end
+
+    end
+
+  end
+end
+

data/lib/proselytism/converters/open_office.rb

@@ -0,0 +1,183 @@
+require 'system_timer'
+
+class Proselytism::Converters::OpenOffice < Proselytism::Converters::Base
+
+  class Error < parent::Base::Error; end
+
+  from  :odt, :doc, :rtf, :sxw, :docx, :txt, :html, :htm, :wps
+  to    :odt, :doc, :rtf, :pdf, :txt, :html, :htm, :wps
+
+  module Bridges
+    module JOD
+      def self.command
+        "java -jar #{File.expand_path('open_office/odconverters/jodconverter-2.2.2/lib/jodconverter-cli-2.2.2.jar', File.dirname(__FILE__))}"
+      end
+    end
+    module PYOD
+      def self.command
+        "python #{File.expand_path('open_office/odconverters/pyodconverter.py', File.dirname(__FILE__))}"
+      end
+    end
+  end
+
+  # Converts documents
+  def perform(origin, options={})
+    destination = destination_file_path(origin, options)
+    command = "#{Proselytism::Converters::OpenOffice}::Bridges::#{config.oo_server_bridge}".constantize.command + " '#{origin}' '#{destination}' 2>&1"
+    server.perform { execute(command) }
+    destination
+  end
+
+
+  # HACK pour contourner un comportement ?trange d'OpenOffice, normalement les enregistrements
+  # se font en UTF-8, mais parfois pour une raison obscure les fichiers texte sont en ISO-8859-1
+  # donc on rajoute un test pour re-convertir dans l'encodage qu'on attend
+  def convert_txt_to_utf8(file_path)
+    if `file #{file_path}` =~ /ISO/
+      system("iconv --from-code ISO-8859-1 --to-code UTF-8 #{file_path} > tmp_iconv.txt && mv tmp_iconv.txt #{file_path}")
+    end
+  end
+
+  def server
+    Server.instance
+  end
+
+
+  class Server
+    include Singleton
+    include Proselytism::Shared
+    class Error < Proselytism::Converters::OpenOffice::Error; end
+
+    def config
+      Proselytism.config
+    end
+
+    # Run a block with a timeout and retry if the first execution fails
+    def perform(&block)
+      attempts = 1
+      begin
+        ensure_available
+        Timeout::timeout(config.oo_conversion_max_time,&block)
+      rescue Timeout::Error, Proselytism::Converters::OpenOffice::Error
+        attempts += 1
+        restart!
+        retry unless attempts > config.oo_conversion_max_tries
+        raise Error, "OpenOffice server perform timeout"
+      end
+    end
+
+    # Restart if running or start new instance
+    def restart!
+      stop! if running?
+      start!
+    end
+
+    # Start new instance
+    def start!
+      log :debug, "OpenOffice server started" do
+        system "#{config.open_office_path} -headless -accept=\"socket,host=127.0.0.1,port=8100\;urp\;\" -nofirststartwizard -nologo -nocrashreport -norestore -nolockcheck -nodefault &"
+        begin
+          SystemTimer.timeout_after(3) do
+            while !running?
+              log :debug, ". Waiting OpenOffice server to run"
+              sleep(0.1)
+            end
+          end
+        rescue
+          raise Error, "Could not start OpenOffice"
+        end
+        # OpenOffice needs some time to wake up
+        sleep(config.oo_server_start_delay)
+      end
+      nil
+    end
+
+    def start_with_running_control!
+      if running?
+        log :debug, "OpenOffice server is allready running"
+      else
+        start_without_running_control!
+      end
+    end
+    alias_method_chain :start!, :running_control
+
+    # Kill running instance
+    def stop!
+      #operating_system = `uname -s`
+      #command = "killall -u `whoami` -#{operating_system == "Linux" ? 'q' : 'm'} soffice"
+      begin
+        Timeout::timeout(3) do
+          loop do
+            system("killall -9 soffice && killall -9 soffice.bin > /dev/null 2>&1")
+            break unless running?
+            sleep(0.2)
+          end
+        end
+      rescue Timeout::Error
+        raise Error, "Could not kill OpenOffice !!"
+      ensure
+        # Remove user profile
+        system("rm -rf ~/openoffice.org*")
+        log :debug, "OpenOffice server stopped"
+      end
+    end
+
+    def stop_with_running_control!
+      if !running?
+        log :debug, "OpenOffice server is allready stoped"
+      else
+        stop_without_running_control!
+      end
+    end
+    alias_method_chain :stop!, :running_control
+
+    # Is OpenOffice server running?
+    def running?
+      !`pgrep soffice`.blank?
+    end
+
+
+    # Is the current instance stuck ?
+    def stalled?
+      begin
+        SystemTimer.timeout_after config.oo_server_max_cpu_delay do
+          loop do
+            cpu_usage = `ps -Ao pcpu,pid,comm= | grep soffice`.split(/\n/).map{|usage| /^\s*\d+/.match(usage)[0].strip.to_i}
+            break unless cpu_usage.all?{|usage| usage > config.oo_server_max_cpu }
+            sleep(0.2)
+          end
+        end
+        false
+      rescue
+        log :error, "OpenOffice server stalled : \n---\n" + `ps -Ao pcpu,pid,comm | grep soffice` + "\n---"
+        true
+      end
+    end
+
+    def available?
+      `ps -o pid,stat,command |grep soffice`.match(/\d+\s(\w)/i)[1] == "S"
+    end
+
+    # Make sure there will be an available instance
+    def ensure_available
+      start! unless running?
+      restart! if stalled?
+      begin
+        SystemTimer.timeout_after config.oo_server_availability_delay do
+          while !available?
+            log :debug, ". Waiting OpenOffice server availability"
+            sleep(0.5)
+          end
+        end
+      rescue Timeout::Error
+        raise Error, "OpenOffice Server unavailable"
+      end
+      true
+    end
+
+  end
+
+
+
+end
+