chupa-text 1.1.3 → 1.1.4

data/lib/chupa-text/decomposers/opendocument-presentation.rb

@@ -0,0 +1,105 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "chupa-text/decomposers/opendocument"
+
+module ChupaText
+  module Decomposers
+    class OpenDocumentPresentation < OpenDocument
+      registry.register("opendocument-presentation", self)
+
+      def initialize(options={})
+        super
+        @extension = "odp"
+        @mime_type = "application/vnd.oasis.opendocument.presentation"
+      end
+
+      private
+      def process_content(entry, context, &block)
+        context[:slides] = []
+        listener = SlidesListener.new(context[:slides])
+        parse(entry.file_data, listener)
+      end
+
+      def finish_decompose(context, &block)
+        metadata = TextData.new("", source_data: context[:data])
+        context[:attributes].each do |name, value|
+          metadata[name] = value
+        end
+        yield(metadata)
+
+        (context[:slides] || []).each_with_index do |slide, i|
+          text = slide[:text]
+          text_data = TextData.new(text, source_data: context[:data])
+          text_data["index"] = i
+          yield(text_data)
+        end
+      end
+
+      class SlidesListener
+        include REXML::SAX2Listener
+
+        TEXT_URI = "urn:oasis:names:tc:opendocument:xmlns:text:1.0"
+        DRAW_URI = "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"
+
+        def initialize(slides)
+          @slides = slides
+          @in_p = false
+        end
+
+        def start_element(uri, local_name, qname, attributes)
+          case uri
+          when TEXT_URI
+            case local_name
+            when "p"
+              @in_p = true
+            end
+          when DRAW_URI
+            case local_name
+            when "page"
+              @slides << {text: ""}
+            end
+          end
+        end
+
+        def end_element(uri, local_name, qname)
+          @in_p = false
+          case uri
+          when TEXT_URI
+            case local_name
+            when "p"
+              @slides.last[:text] << "\n"
+            end
+          end
+        end
+
+        def characters(text)
+          add_text(text)
+        end
+
+        def cdata(content)
+          add_text(content)
+        end
+
+        private
+        def add_text(text)
+          return unless @in_p
+          @slides.last[:text] << CGI.unescapeHTML(text)
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/opendocument-spreadsheet.rb

@@ -0,0 +1,134 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "chupa-text/decomposers/opendocument"
+
+module ChupaText
+  module Decomposers
+    class OpenDocumentSpreadsheet < OpenDocument
+      registry.register("opendocument-spreadsheet", self)
+
+      def initialize(options={})
+        super
+        @extension = "ods"
+        @mime_type = "application/vnd.oasis.opendocument.spreadsheet"
+      end
+
+      private
+      def process_content(entry, context, &block)
+        context[:sheets] = []
+        listener = SheetsListener.new(context[:sheets])
+        parse(entry.file_data, listener)
+      end
+
+      def finish_decompose(context, &block)
+        metadata = TextData.new("", source_data: context[:data])
+        context[:attributes].each do |name, value|
+          metadata[name] = value
+        end
+        yield(metadata)
+
+        (context[:sheets] || []).each_with_index do |sheet, i|
+          text = sheet[:text]
+          text_data = TextData.new(text, source_data: context[:data])
+          text_data["index"] = i
+          name = sheet[:name]
+          text_data["name"] = name if name
+          yield(text_data)
+        end
+      end
+
+      class SheetsListener
+        include REXML::SAX2Listener
+
+        TEXT_URI = "urn:oasis:names:tc:opendocument:xmlns:text:1.0"
+        TABLE_URI = "urn:oasis:names:tc:opendocument:xmlns:table:1.0"
+
+        def initialize(sheets)
+          @sheets = sheets
+          @prefix_to_uri = {}
+          @uri_to_prefix = {}
+          @in_p = false
+        end
+
+        def start_prefix_mapping(prefix, uri)
+          @prefix_to_uri[prefix] = uri
+          @uri_to_prefix[uri] = prefix
+        end
+
+        def end_prefix_mapping(prefix)
+          uri = @prefix_to_uri.delete(prefix)
+          @uri_to_prefix.delete(uri)
+        end
+
+        def start_element(uri, local_name, qname, attributes)
+          case uri
+          when TEXT_URI
+            case local_name
+            when "p"
+              @in_p = true
+            end
+          when TABLE_URI
+            table_prefix = @uri_to_prefix[TABLE_URI]
+            case local_name
+            when "table"
+              @sheets << {
+                name: attributes["#{table_prefix}:name"],
+                rows: [],
+              }
+            when "table-row"
+              @sheets.last[:rows] << []
+            when "table-cell"
+              @sheets.last[:rows].last << {text: ""}
+            end
+          end
+        end
+
+        def end_element(uri, local_name, qname)
+          @in_p = false
+          case uri
+          when TABLE_URI
+            case local_name
+            when "table"
+              sheet = @sheets.last
+              text = ""
+              sheet[:rows].each do |row|
+                cell_texts = row.collect {|cell| cell[:text]}
+                next if cell_texts.all?(&:empty?)
+                text << cell_texts.join("\t") << "\n"
+              end
+              sheet[:text] = text
+            end
+          end
+        end
+
+        def characters(text)
+          add_text(text)
+        end
+
+        def cdata(content)
+          add_text(content)
+        end
+
+        private
+        def add_text(text)
+          return unless @in_p
+          @sheets.last[:rows].last.last[:text] << CGI.unescapeHTML(text)
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/opendocument-text.rb

@@ -0,0 +1,89 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "chupa-text/decomposers/opendocument"
+
+module ChupaText
+  module Decomposers
+    class OpenDocumentText < OpenDocument
+      registry.register("opendocument-text", self)
+
+      def initialize(options={})
+        super
+        @extension = "odt"
+        @mime_type = "application/vnd.oasis.opendocument.text"
+      end
+
+      private
+      def process_content(entry, context, &block)
+        context[:text] = ""
+        listener = TextListener.new(context[:text])
+        parse(entry.file_data, listener)
+      end
+
+      def finish_decompose(context, &block)
+        text_data = TextData.new(context[:text] || "",
+                                 source_data: context[:data])
+        context[:attributes].each do |name, value|
+          text_data[name] = value
+        end
+        yield(text_data)
+      end
+
+      class TextListener
+        include REXML::SAX2Listener
+
+        TEXT_URI = "urn:oasis:names:tc:opendocument:xmlns:text:1.0"
+        def initialize(output)
+          @output = output
+          @in_p = false
+        end
+
+        def start_element(uri, local_name, qname, attributes)
+          return unless uri == TEXT_URI
+          case local_name
+          when "p"
+            @in_p = true
+          end
+        end
+
+        def end_element(uri, local_name, qname)
+          @in_p = false
+
+          return unless uri == TEXT_URI
+          case local_name
+          when "p"
+            @output << "\n"
+          end
+        end
+
+        def characters(text)
+          add_text(text)
+        end
+
+        def cdata(content)
+          add_text(content)
+        end
+
+        private
+        def add_text(text)
+          return unless @in_p
+          @output << CGI.unescapeHTML(text)
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/opendocument.rb

@@ -0,0 +1,139 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "cgi/util"
+require "rexml/parsers/sax2parser"
+require "rexml/sax2listener"
+
+require "archive/zip"
+
+module ChupaText
+  module Decomposers
+    class OpenDocument < Decomposer
+      def target?(data)
+        data.extension == @extension or
+          data.mime_type == @mime_type
+      end
+
+      def target_score(data)
+        if target?(data)
+          -1
+        else
+          nil
+        end
+      end
+
+      def decompose(data, &block)
+        context = {
+          data: data,
+          attributes: {},
+        }
+        data.open do |input|
+          Archive::Zip.open(input) do |zip|
+            zip.each do |entry|
+              next unless entry.file?
+              case entry.zip_path
+              when "content.xml"
+                process_content(entry, context, &block)
+              when "meta.xml"
+                process_meta(entry, context, &block)
+              end
+            end
+          end
+        end
+        finish_decompose(context, &block)
+      end
+
+      private
+      def parse(io, listener)
+        source = REXML::Source.new(io.read)
+        parser = REXML::Parsers::SAX2Parser.new(source)
+        parser.listen(listener)
+        parser.parse
+      end
+
+      def process_meta(entry, context, &block)
+        listener = AttributesListener.new(context[:attributes])
+        parse(entry.file_data, listener)
+      end
+
+      class AttributesListener
+        include REXML::SAX2Listener
+
+        META_URI = "urn:oasis:names:tc:opendocument:xmlns:meta:1.0"
+        DUBLIN_CORE_URI = "http://purl.org/dc/elements/1.1/"
+
+        def initialize(attributes)
+          @attributes = attributes
+          @name = nil
+          @type = nil
+        end
+
+        def start_element(uri, local_name, qname, attributes)
+          case uri
+          when META_URI
+            case local_name
+            when "creation-date"
+              @name = "created_time"
+              @type = :w3cdtf
+            when "keyword"
+              @name = "keywords"
+              @type = :array
+            when "generator"
+              @name = local_name
+            end
+          when DUBLIN_CORE_URI
+            case local_name
+            when "date"
+              @name = "modified_time"
+              @type = :w3cdtf
+            when "description", "title", "subject"
+              @name = local_name
+            end
+          end
+        end
+
+        def end_element(uri, local_name, qname)
+          @name = nil
+          @type = nil
+        end
+
+        def characters(text)
+          set_attribute(text)
+        end
+
+        def cdata(content)
+          set_attribute(content)
+        end
+
+        def set_attribute(value)
+          return if @name.nil?
+
+          value = CGI.unescapeHTML(value)
+          case @type
+          when :w3cdtf
+            value = Time.xmlschema(value)
+          when :array
+            values = @attributes[@name] || []
+            values << value
+            value = values
+          end
+          @attributes[@name] = value
+        end
+      end
+    end
+  end
+end