documentalist 0.1.0

data/Manifest

@@ -0,0 +1,12 @@
+README.rdoc
+Rakefile
+documentalist.gemspec
+init.rb
+lib/DocumentConverter.py
+lib/documentalist.rb
+lib/open_office/server.rb
+rails/init.rb
+test/documentalist_test.rb
+test/fixtures/fixture.odt
+test/open_office_test.rb
+Manifest

data/README.rdoc

@@ -0,0 +1,3 @@
+= Documentalist
+
+Rails gem for talking to OpenOffice.

data/Rakefile

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'rake'
+require 'echoe'
+
+Echoe.new('documentalist', '0.1.0') do |p|
+  p.description     = "Ruby interface that talks to OpenOffice"
+  p.url             = "http://github.com/davout/documentalist"
+  p.author          = "David FRANCOIS"
+  p.email           = "david.francois@webflows.fr"
+  p.ignore_pattern  = ["tmp/*", "script/*"]
+  p.test_pattern    = "test/**/*.rb"
+  p.development_dependencies = []
+  p.runtime_dependencies = ['zip >=2.0.2']
+end  
+
+Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }

data/documentalist.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{documentalist}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["David FRANCOIS"]
+  s.date = %q{2010-06-08}
+  s.description = %q{Ruby interface that talks to OpenOffice}
+  s.email = %q{david.francois@webflows.fr}
+  s.extra_rdoc_files = ["README.rdoc", "lib/DocumentConverter.py", "lib/documentalist.rb", "lib/open_office/server.rb"]
+  s.files = ["README.rdoc", "Rakefile", "documentalist.gemspec", "init.rb", "lib/DocumentConverter.py", "lib/documentalist.rb", "lib/open_office/server.rb", "rails/init.rb", "test/documentalist_test.rb", "test/fixtures/fixture.odt", "test/open_office_test.rb", "Manifest"]
+  s.homepage = %q{http://github.com/davout/documentalist}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Documentalist", "--main", "README.rdoc"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{documentalist}
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Ruby interface that talks to OpenOffice}
+  s.test_files = ["test/documentalist_test.rb", "test/open_office_test.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<zip>, [">= 2.0.2"])
+    else
+      s.add_dependency(%q<zip>, [">= 2.0.2"])
+    end
+  else
+    s.add_dependency(%q<zip>, [">= 2.0.2"])
+  end
+end

data/lib/DocumentConverter.py

@@ -0,0 +1,151 @@
+#!/usr/bin/python
+#
+# PyODConverter (Python OpenDocument Converter) v1.0.0 - 2008-05-05
+#
+# This script converts a document from one office format to another by
+# connecting to an OpenOffice.org instance via Python-UNO bridge.
+#
+# Copyright (C) 2008 Mirko Nasato <mirko@artofsolving.com>
+# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl-2.1.html
+# - or any later version.
+#
+DEFAULT_OPENOFFICE_PORT = 8100
+
+import uno
+from os.path import abspath, isfile, splitext
+from com.sun.star.beans import PropertyValue
+from com.sun.star.task import ErrorCodeIOException
+from com.sun.star.connection import NoConnectException
+
+FAMILY_TEXT = "Text"
+FAMILY_SPREADSHEET = "Spreadsheet"
+FAMILY_PRESENTATION = "Presentation"
+FAMILY_DRAWING = "Drawing"
+
+FILTER_MAP = {
+    "pdf": {
+        FAMILY_TEXT: "writer_pdf_Export",
+        FAMILY_SPREADSHEET: "calc_pdf_Export",
+        FAMILY_PRESENTATION: "impress_pdf_Export",
+        FAMILY_DRAWING: "draw_pdf_Export"
+    },
+    "html": {
+        FAMILY_TEXT: "HTML (StarWriter)",
+        FAMILY_SPREADSHEET: "HTML (StarCalc)",
+        FAMILY_PRESENTATION: "impress_html_Export"
+    },
+    "odt": { FAMILY_TEXT: "writer8" },
+    "doc": { FAMILY_TEXT: "MS Word 97" },
+    "rtf": { FAMILY_TEXT: "Rich Text Format" },
+    "txt": { FAMILY_TEXT: "Text" },
+    "ods": { FAMILY_SPREADSHEET: "calc8" },
+    "xls": { FAMILY_SPREADSHEET: "MS Excel 97" },
+    "odp": { FAMILY_PRESENTATION: "impress8" },
+    "ppt": { FAMILY_PRESENTATION: "MS PowerPoint 97" },
+    "swf": { FAMILY_PRESENTATION: "impress_flash_Export" }
+}
+# see http://wiki.services.openoffice.org/wiki/Framework/Article/Filter
+# for more available filters
+
+
+class DocumentConversionException(Exception):
+
+    def __init__(self, message):
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class DocumentConverter:
+    
+    def __init__(self, port=DEFAULT_OPENOFFICE_PORT):
+        localContext = uno.getComponentContext()
+        resolver = localContext.ServiceManager.createInstanceWithContext("com.sun.star.bridge.UnoUrlResolver", localContext)
+        try:
+            context = resolver.resolve("uno:socket,host=localhost,port=%s;urp;StarOffice.ComponentContext" % port)
+        except NoConnectException:
+            raise DocumentConversionException, "failed to connect to OpenOffice.org on port %s" % port
+        self.desktop = context.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", context)
+
+    def convert(self, inputFile, outputFile):
+
+        inputUrl = self._toFileUrl(inputFile)
+        outputUrl = self._toFileUrl(outputFile)
+        
+        document = self.desktop.loadComponentFromURL(inputUrl, "_blank", 0, self._toProperties(Hidden=True))
+        try:
+          document.refresh()
+        except AttributeError:
+          pass
+        
+        outputExt = self._getFileExt(outputFile)
+        filterName = self._filterName(document, outputExt)
+
+        try:
+            document.storeToURL(outputUrl, self._toProperties(FilterName=filterName))
+        finally:
+            document.close(True)
+
+    def _filterName(self, document, outputExt):
+        family = self._detectFamily(document)
+        try:
+            filterByFamily = FILTER_MAP[outputExt]
+        except KeyError:
+            raise DocumentConversionException, "unknown output format: '%s'" % outputExt
+        try:
+            return filterByFamily[family]
+        except KeyError:
+            raise DocumentConversionException, "unsupported conversion: from '%s' to '%s'" % (family, outputExt)
+    
+    def _detectFamily(self, document):
+        if document.supportsService("com.sun.star.text.GenericTextDocument"):
+            # NOTE: a GenericTextDocument is either a TextDocument, a WebDocument, or a GlobalDocument
+            # but this further distinction doesn't seem to matter for conversions
+            return FAMILY_TEXT
+        if document.supportsService("com.sun.star.sheet.SpreadsheetDocument"):
+            return FAMILY_SPREADSHEET
+        if document.supportsService("com.sun.star.presentation.PresentationDocument"):
+            return FAMILY_PRESENTATION
+        if document.supportsService("com.sun.star.drawing.DrawingDocument"):
+            return FAMILY_DRAWING
+        raise DocumentConversionException, "unknown document family: %s" % document
+
+    def _getFileExt(self, path):
+        ext = splitext(path)[1]
+        if ext is not None:
+            return ext[1:].lower()
+
+    def _toFileUrl(self, path):
+        return uno.systemPathToFileUrl(abspath(path))
+
+    def _toProperties(self, **args):
+        props = []
+        for key in args:
+	    prop = PropertyValue()
+	    prop.Name = key
+	    prop.Value = args[key]
+	    props.append(prop)
+        return tuple(props)
+
+
+if __name__ == "__main__":
+    from sys import argv, exit
+    
+    if len(argv) < 3:
+        print "USAGE: python %s <input-file> <output-file>" % argv[0]
+        exit(255)
+    if not isfile(argv[1]):
+        print "no such input file: %s" % argv[1]
+        exit(1)
+
+    try:
+        converter = DocumentConverter()    
+        converter.convert(argv[1], argv[2])
+    except DocumentConversionException, exception:
+        print "ERROR!" + str(exception)
+        exit(1)
+    except ErrorCodeIOException, exception:
+        print "ERROR! ErrorCodeIOException %d" % exception.ErrCode
+        exit(1)
+

data/lib/documentalist.rb

@@ -0,0 +1,63 @@
+require 'rubygems'
+require 'erb'
+require 'fileutils'
+require 'tmpdir'
+require 'zip/zip'
+require 'open_office/server'
+
+module Documentalist
+  def self.merge(str, options = {})
+    locals = options[:locals]
+
+    if locals and locals.is_a? Hash
+      locals.each do |k,v|
+        instance_variable_set("@#{k.to_s}".to_sym, v)
+      end
+    end
+
+    ERB.new(str).result(binding)
+  end
+
+  def self.get_contents(odt_file)
+    contents = ""
+    Zip::ZipFile.open(odt_file) { |zip| contents = zip.read("content.xml") }
+    contents.gsub("&lt;%", "<%").gsub("%&gt;", "%>")
+  end
+
+  def self.merge_template(template, options = {})
+    # Get template contents
+    tmp_contents= Tempfile.new("officer-contents")
+    tmp_contents.write(merge(get_contents(template), :locals => options[:locals]))
+    tmp_contents.close
+
+    # Copy the template so we can merge the data into the copy
+    tmp_merged_template = File.join(Dir.tmpdir, "merged-template-#{rand(10**9)}#{File.extname(template)}")
+    FileUtils.cp(template, tmp_merged_template)
+
+    # Stuff the merged contents.xml into the OpenDocument zip
+    Zip::ZipFile.open(tmp_merged_template) do |zip|
+      zip.replace("content.xml", tmp_contents.path)
+      zip.commit
+    end
+
+    # Remove the merged contents.xml
+    tmp_contents.unlink
+
+    # Manages the converted file depending on the context
+    if options[:to]
+      if File.extname(options[:to]) == File.extname(template)
+        FileUtils.mv(tmp_merged_template, options[:to])
+      else
+        OpenOffice::Server.convert(tmp_merged_template, options[:to])
+        FileUtils.rm(tmp_merged_template)
+      end
+    else
+      FileUtils.rm(template)
+      FileUtils.mv(tmp_merged_template, template)
+    end
+  end
+
+  def self.convert(from, to)
+    OpenOffice::Server.convert(from, :to => to)
+  end
+end  

data/lib/open_office/server.rb

@@ -0,0 +1,148 @@
+require 'timeout'
+require 'tmpdir'
+
+module OpenOffice
+  module Server
+    # Path to the Python executable
+    PYTHON_PATH = "/usr/bin/python"
+
+    # Server can convert from the following file formats
+    CONVERT_FROM = [:odt, :doc, :rtf, :docx, :txt, :html, :htm, :wps]
+
+    # To the following formats
+    CONVERT_TO = [:odt, :doc, :rtf, :pdf, :txt, :html, :htm, :wps]
+
+    # Python conversion script path
+    PY_OD_CONVERTER = File.join(File.dirname(__FILE__), "../DocumentConverter.py")
+
+    # Maximum allowed CPU usage for an OpenOffice process
+    MAX_CPU = 80
+
+    # Server start grace time
+    SERVER_START_DELAY = 4
+
+    # Log file
+    LOG_FILE = Object.const_defined?(:RAILS_ROOT) ? File.join(RAILS_ROOT, "log", "openoffice.log") : ""
+
+    def self.convert(origin, options = {:to => :txt})
+      if options and options[:to]
+        raise "#{origin} does not exist !" unless File.exist?(origin)
+        ensure_available
+
+        if options[:to].is_a? Symbol
+          destination = "#{origin.gsub(/[^\.]*$/, "")}#{options[:to].to_s}"
+        elsif options[:to].is_a? String
+          destination = options[:to]
+        else
+          raise "Can't convert #{origin} to #{options[:to]}"
+        end
+
+        timeout(10, :attempts => 2) do
+          system("#{PYTHON_PATH} #{PY_OD_CONVERTER} #{origin} #{destination} > /dev/null 2>&1")
+
+          # HACK : sometimes text files get saved in ISO-8859-1 instead of regular UTF-8, so we force
+          # a conversion if it's the case
+          if `file #{destination}` =~ /ISO/ and destination =~ /\.txt$/
+            temp_file = File.join(Dir.tmpdir, "tmp_iconv_#{rand(10**9)}.txt")
+            system("iconv --from-code ISO-8859-1 --to-code UTF-8 #{destination} >  && #{temp_file} mv #{temp_file} #{destination}")
+          end
+        end
+
+        destination
+      end
+    end
+
+    private
+    # Is OpenOffice server running?
+    def self.running?
+      !`pgrep office`.empty?
+    end
+
+    # Restart if running or start new instance
+    def self.restart!
+      kill! if running?
+      start!
+    end
+
+    # Start new instance
+    def self.start!
+      raise "Already running!" if running?
+      system("/usr/bin/soffice -headless -accept=\"socket,host=127.0.0.1,port=8100;urp;\" -nofirststartwizard -nologo -nocrashreport -norestore -nolockcheck -nodefault #{">>" unless LOG_FILE.empty?} #{LOG_FILE} 2>&1 &")
+
+      begin
+        timeout(2) do
+          while !running?
+            print "."
+          end
+        end
+      rescue
+        raise "Could not start OpenOffice"
+      end
+
+      # OpenOffice needs some time to wake up
+      sleep(SERVER_START_DELAY)
+
+      nil
+    end
+
+    # Kill running instance
+    def self.kill!
+      raise "Not running!" unless running?
+
+      begin
+        timeout(3, :attempts => 2) do
+          while(running?)
+            system("pkill -9 office")
+          end
+        end
+      rescue Timeout::Error
+        raise "Mayday, mayday ! Could not kill OpenOffice !!"
+      ensure
+        # Remove user profile
+        system("rm -rf ~/openoffice.org*")
+      end
+    end
+
+    # Is the current instance stuck ?
+    def self.stalled?
+      if running?
+        cpu_usage = `ps -Ao pcpu,pid,cmd | grep office`
+        cpu_usage = cpu_usage.split(/\n/).select{|line| /#{pids.join("|")}/.match(line) }
+
+        cpu_usage.any?{|usage| /^\s*\d+/.match(usage)[0].strip.to_i > MAX_CPU}
+      end
+    end
+
+    # Make sure there will be an available instance
+    def self.ensure_available
+      start! unless running?
+      restart! if stalled?
+    end
+
+    # Get OO processes pids
+    def self.pids
+      `pgrep office`.split.map{|pid| pid.to_i } unless `pgrep office`.empty?
+    end
+
+    # Run a block with a timeout and retry if the first execution fails
+    def self.timeout(max_time = 0, options = {:attempts => 1, :sleep => nil})
+      if block_given?
+        attempts = options[:attempts] || 1
+        begin
+          Timeout::timeout(max_time) do
+            yield
+          end
+        rescue Timeout::Error
+          attempts -= 1
+          sleep(options[:sleep]) if options[:sleep]
+          retry unless attempts.zero?
+          raise
+        end
+      end
+    end
+
+    def self.convertible?(origin, destination)
+      CONVERT_FROM.include?(File.extname(origin)) && CONVERT_TO.include?(File.extname(destination))
+    end
+  end
+end

data/rails/init.rb

@@ -0,0 +1 @@
+require 'documentalist'

data/test/documentalist_test.rb

@@ -0,0 +1,40 @@
+require 'test/unit'
+require 'documentalist'
+
+class DocumentalistTest < Test::Unit::TestCase
+  @@odt_fixture = File.join(File.dirname(__FILE__), "fixtures/fixture.odt")
+
+  def test_merge
+    template = "<%= @var1 %><%= 1.upto(3).map{ |n| n.to_s }.join %><%= @var2 %>"
+
+    merged = Documentalist.merge(template, :locals => {
+        :var1 => "test",
+        :var2 => "working?"
+      }
+    )
+
+    assert_equal "test123working?", merged, "Merge wasn't performed correctly"
+  end
+
+  def test_read_zipped_odt
+    contents = Documentalist.get_contents(@@odt_fixture)
+
+    assert_match /Hello/, contents
+    assert_match /thing/, contents
+    assert !(contents =~ /%&gt;/)
+    assert !(contents =~ /&lt;%=/)
+  end
+
+  def test_odt_merge
+    template = "#{File.join(File.dirname(__FILE__), "fixtures/fixture.odt")}"
+    result = "#{File.join(File.dirname(__FILE__), "fixtures/result.odt")}"
+
+    Documentalist.merge_template(template,
+      :locals => {:thing => "world"},
+      :to => result
+    )
+    
+    assert /world/, Documentalist.get_contents(result)
+    File.delete(result)
+  end
+end

data/test/open_office_test.rb

@@ -0,0 +1,21 @@
+require 'test/unit'
+require 'documentalist'
+require 'fileutils'
+require 'tmpdir'
+
+class OpenOfficeTest < Test::Unit::TestCase
+  def test_open_office
+    destination = File.join(Dir.tmpdir, "fixture#{rand(10**9)}.pdf")
+
+    Documentalist.convert(
+      File.join(File.dirname(__FILE__), "fixtures", "fixture.odt"),
+      destination
+    )
+
+    assert File.exist?(destination)
+
+    FileUtils.rm(destination)
+
+    assert !File.exist?(destination)
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification 
+name: documentalist
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- David FRANCOIS
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-08 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: zip
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 0
+        - 2
+        version: 2.0.2
+  type: :runtime
+  version_requirements: *id001
+description: Ruby interface that talks to OpenOffice
+email: david.francois@webflows.fr
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- lib/DocumentConverter.py
+- lib/documentalist.rb
+- lib/open_office/server.rb
+files: 
+- README.rdoc
+- Rakefile
+- documentalist.gemspec
+- init.rb
+- lib/DocumentConverter.py
+- lib/documentalist.rb
+- lib/open_office/server.rb
+- rails/init.rb
+- test/documentalist_test.rb
+- test/fixtures/fixture.odt
+- test/open_office_test.rb
+- Manifest
+has_rdoc: true
+homepage: http://github.com/davout/documentalist
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Documentalist
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 11
+      segments: 
+      - 1
+      - 2
+      version: "1.2"
+requirements: []
+
+rubyforge_project: documentalist
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Ruby interface that talks to OpenOffice
+test_files: 
+- test/documentalist_test.rb
+- test/open_office_test.rb