chupa-text-decomposer-libreoffice 1.0.0

data/README.md

@@ -0,0 +1,45 @@
+# README
+
+## Name
+
+chupa-text-decomposer-libreoffice
+
+## Description
+
+This is a ChupaText decomposer plugin for to extract text and
+meta-data from office files such as Microsoft Word file, Microsoft
+Excel file and OpenDocument Format file. It uses
+[LibreOffice](https://www.libreoffice.org/).
+
+You can use `libreoffice` decomposer.
+
+It depends on `pdf` decomposer. Because it converts a office file to
+PDF file and extracts text and meta-data by `pdf` decomposer.
+
+## Install
+
+Install chupa-text-decomposer-libreoffice gem:
+
+```
+% gem install chupa-text-decomposer-libreoffice
+```
+
+Install
+[LibreOffice from download page](http://www.libreoffice.org/download).
+
+Now, you can extract text and meta-data from office files:
+
+```
+% chupa-text document.doc
+```
+
+## Author
+
+  * Kouhei Sutou `<kou@clear-code.com>`
+
+## License
+
+LGPL 2.1 or later.
+
+(Kouhei Sutou has a right to change the license including contributed
+patches.)

data/Rakefile

@@ -0,0 +1,46 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+task :default => :test
+
+require "pathname"
+
+require "rubygems"
+require "bundler/gem_helper"
+require "packnga"
+
+base_dir = Pathname(__FILE__).dirname
+
+helper = Bundler::GemHelper.new(base_dir.to_s)
+def helper.version_tag
+  version
+end
+
+helper.install
+spec = helper.gemspec
+
+Packnga::DocumentTask.new(spec) do
+end
+
+Packnga::ReleaseTask.new(spec) do
+end
+
+desc "Run tests"
+task :test do
+  ruby("test/run-test.rb")
+end

data/chupa-text-decomposer-libreoffice.gemspec

@@ -0,0 +1,51 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+clean_white_space = lambda do |entry|
+  entry.gsub(/(\A\n+|\n+\z)/, '') + "\n"
+end
+
+Gem::Specification.new do |spec|
+  spec.name = "chupa-text-decomposer-libreoffice"
+  spec.version = "1.0.0"
+  spec.homepage = "https://github.com/ranguba/chupa-text-decomposer-libreoffice"
+  spec.authors = ["Kouhei Sutou"]
+  spec.email = ["kou@clear-code.com"]
+  readme = File.read("README.md", :encoding => "UTF-8")
+  entries = readme.split(/^\#\#\s(.*)$/)
+  description = clean_white_space.call(entries[entries.index("Description") + 1])
+  spec.summary = description.split(/\n\n+/, 2).first
+  spec.description = description
+  spec.license = "LGPLv2.1 or later"
+  spec.files = ["#{spec.name}.gemspec"]
+  spec.files += ["README.md", "LICENSE.txt", "Rakefile", "Gemfile"]
+  spec.files += [".yardopts"]
+  spec.files += Dir.glob("lib/**/*.rb")
+  spec.files += Dir.glob("doc/text/*")
+  spec.files += Dir.glob("test/**/*")
+
+  spec.requirements << "LibreOffice"
+
+  spec.add_runtime_dependency("chupa-text-decomposer-pdf")
+
+  spec.add_development_dependency("bundler")
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("test-unit")
+  spec.add_development_dependency("packnga")
+  spec.add_development_dependency("redcarpet")
+end

data/doc/text/news.md

@@ -0,0 +1,5 @@
+# News
+
+## 1.0.0: 2014-02-16
+
+The first release!!!

data/lib/chupa-text/decomposers/libreoffice.rb

@@ -0,0 +1,131 @@
+# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "tempfile"
+require "tmpdir"
+
+module ChupaText
+  module Decomposers
+    class LibreOffice < Decomposer
+      registry.register("libreoffice", self)
+
+      def initialize(options)
+        super
+        @command = find_command
+      end
+
+      TARGET_EXTENSIONS = [
+        "odt",
+        "ods",
+        "odp",
+        "doc",
+        "xls",
+        "ppt",
+        "docx",
+        "xlsx",
+        "pptx",
+      ]
+      TARGET_MIME_TYPES = [
+        "application/vnd.oasis.opendocument.text",
+        "application/vnd.oasis.opendocument.presentation",
+        "application/vnd.oasis.opendocument.spreadsheet",
+        "application/msword",
+        "application/vnd.ms-excel",
+        "application/vnd.ms-powerpoint",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+      ]
+      def target?(data)
+        return false if @command.nil?
+        TARGET_EXTENSIONS.include?(data.extension) or
+          TARGET_MIME_TYPES.include?(data.mime_type)
+      end
+
+      def decompose(data)
+        pdf_data = convert_to_pdf(data)
+        return if pdf_data.nil?
+        yield(pdf_data)
+      end
+
+      private
+      def find_command
+        candidates = [
+          @options[:libreoffice],
+          ENV["LIBREOFFICE"],
+          "libreoffice",
+          "soffice",
+        ]
+        candidates.each do |candidate|
+          next if candidate.nil?
+          command = ExternalCommand.new(candidate)
+          return command if command.exist?
+          expanded_candidate = expand_candidate(candidate)
+          next if expanded_candidate.nil?
+          command = ExternalCommand.new(expanded_candidate)
+          return command if command.exist?
+        end
+        nil
+      end
+
+      def expand_candidate(candidate)
+        Dir.glob("/opt/libreoffice*/program/#{candidate}").first
+      end
+
+      def convert_to_pdf(data)
+        Dir.mktmpdir do |temporary_directory|
+          output = Tempfile.new("chupa-text-decomposer-libreoffice-output")
+          error = Tempfile.new("chupa-text-decomposer-libreoffice-error")
+          succeeded = @command.run("--headless",
+                                   "--nologo",
+                                   "--convert-to", "pdf",
+                                   "--outdir", temporary_directory,
+                                   data.path,
+                                   {
+                                     :spawn_options => {
+                                       :out => output.path,
+                                       :err => error.path,
+                                     },
+                                   })
+          unless succeeded
+            tag = "[decomposer][libreoffice][convert][exited][abnormally]"
+            message = [
+              tag,
+              "output: <#{output.read}>",
+              "error: <#{error.read}>",
+            ].join("\n")
+            ChupaText.logger.error(message)
+            return nil
+          end
+          pdf_path, = Dir.glob("#{temporary_directory}/*.pdf")
+          if pdf_path.nil?
+            tag = "[decomposer][libreoffice][convert][failed]"
+            message = [
+              "#{tag}: LibreOffice may be running",
+              "output: <#{output.read}>",
+              "error: <#{error.read}>",
+            ].join("\n")
+            ChupaText.logger.error(message)
+            return nil
+          end
+          File.open(pdf_path, "rb") do |pdf|
+            ChupaText::VirtualFileData.new(pdf_path, pdf)
+          end
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,45 @@
+# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module FixtureHelper
+  def setup_decomposer
+    @decomposer = ChupaText::Decomposers::LibreOffice.new({})
+  end
+
+  def fixture_path(*components)
+    base_path = Pathname(__FILE__).dirname + "fixture"
+    base_path.join(*components)
+  end
+end
+
+module DecomposeHelper
+  def decompose(path)
+    data = ChupaText::InputData.new(path)
+
+    pdf_decomposer = ChupaText::Decomposers::PDF.new({})
+    decomposed = []
+    @decomposer.decompose(data) do |decomposed_data|
+      if pdf_decomposer.target?(decomposed_data)
+        pdf_decomposer.decompose(decomposed_data) do |pdf_decomposed_data|
+          decomposed << pdf_decomposed_data
+        end
+      else
+        decomposed << decomposed_data
+      end
+    end
+    decomposed
+  end
+end

data/test/run-test.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+$VERBOSE = true
+
+require "bundler/setup"
+
+require "test-unit"
+
+require "chupa-text"
+
+ChupaText::Decomposers.load
+
+require_relative "helper"
+
+exit(Test::Unit::AutoRunner.run(true))

data/test/test-doc.rb

@@ -0,0 +1,123 @@
+# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "pathname"
+
+class TestDoc < Test::Unit::TestCase
+  include FixtureHelper
+
+  def setup
+    setup_decomposer
+  end
+
+  def fixture_path(*components)
+    super("doc", *components)
+  end
+
+  sub_test_case("target?") do
+    sub_test_case("extension") do
+      def create_data(uri)
+        data = ChupaText::Data.new
+        data.body = ""
+        data.uri = uri
+        data
+      end
+
+      def test_doc
+        assert_true(@decomposer.target?(create_data("document.doc")))
+      end
+    end
+
+    sub_test_case("mime-type") do
+      def create_data(mime_type)
+        data = ChupaText::Data.new
+        data.mime_type = mime_type
+        data
+      end
+
+      def test_ms_word
+        mime_type = "application/msword"
+        assert_true(@decomposer.target?(create_data(mime_type)))
+      end
+    end
+  end
+
+  sub_test_case("decompose") do
+    include DecomposeHelper
+
+    sub_test_case("attributes") do
+      def test_title
+        assert_equal(["Title"], decompose("title"))
+      end
+
+      def test_author
+        assert_equal([nil], decompose("author"))
+      end
+
+      def test_subject
+        assert_equal(["Subject"], decompose("subject"))
+      end
+
+      def test_keywords
+        assert_equal(["Keyword1, Keyword2"], decompose("keywords"))
+      end
+
+      def test_creator
+        assert_equal(["Writer"], decompose("creator"))
+      end
+
+      def test_producer
+        assert_equal(["LibreOffice 4.1"], decompose("producer"))
+      end
+
+      def test_creation_date
+        assert_equal([nil], decompose("creation_date"))
+      end
+
+      private
+      def decompose(attribute_name)
+        super(fixture_path("attributes.doc")).collect do |data|
+          data[attribute_name]
+        end
+      end
+    end
+
+    sub_test_case("one page") do
+      def test_body
+        assert_equal(["Page1"], decompose.collect(&:body))
+      end
+
+      private
+      def decompose
+        super(fixture_path("one-page.doc"))
+      end
+    end
+
+    sub_test_case("multi pages") do
+      def test_body
+        assert_equal([<<-BODY.chomp], decompose.collect(&:body))
+Page1
+Page2
+        BODY
+      end
+
+      private
+      def decompose
+        super(fixture_path("multi-pages.doc"))
+      end
+    end
+  end
+end