chupa-text 1.1.3 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38219bcd127b3e4b7ee11a46d51e7e6762b2e5ad637239518c9ec64cdb4906fc
-  data.tar.gz: 279ac9fa242ec20c8c7d7710d5a26bb58bd73cbe0d3a6473e4b714d766743825
+  metadata.gz: ce1def40525d7278aa45cdbe3af69cd95656fa58d1d02e6ddb3e2677940ed7d6
+  data.tar.gz: 4ec184c4bd0f61508d4b1908e7c89abd8aeb01026c8b3590b404ca672887c6a1
 SHA512:
-  metadata.gz: 776f7004429ff9a95bdbac88a50029cfc1cebdbf783ded8f91ddd6e29f79fc5fc8d113a616bfdfb5684b940a5cbb9ef47d4a9bff85cc6a9f69e1300997293207
-  data.tar.gz: b0c5299e11c13377a68f3871bb1a673ec3216577f04430f3cb30352999ac3f8a5b7d9d9d2dabe5a30da053904662610b193d7364f42f414acd72c3d2b18df126
+  metadata.gz: ae43e4354761a953f61cda5348524f44346996133e4f77a310bfbfd07295b4548d41847f8132387ec9ade30c44a9c2bdd7936c7633085534ce03bb5db6f9061f
+  data.tar.gz: f9174e01e21a2dbbc1647d191084969e9fd0cf4dc6f24a469ac7c6c4f2378d7a11946e6b2350e8df86ad5d007c6f048c804d4c1c2b79e95628511383c76c6061

data/doc/text/news.md

@@ -1,5 +1,17 @@
 # News
 
+## 1.1.4: 2019-02-26
+
+### Improvements
+
+  * Added support for decomposer selection by score.
+
+  * Added support for Office Open XML.
+
+  * Added support for OpenDocument.
+
+  * `chupa-text`: Added `--mime-boundary` option.
+
 ## 1.1.3: 2018-07-18
 
 ### Improvements

data/lib/chupa-text/command/chupa-text.rb

@@ -46,6 +46,7 @@ module ChupaText
         @uri = nil
         @mime_type = nil
         @format = :json
+        @mime_formatter_options = {}
         @need_screenshot = true
         @expected_screenshot_size = [200, 200]
       end
@@ -127,6 +128,11 @@ module ChupaText
                   "(default: #{@format})") do |format|
           @format = format
         end
+        parser.on("--mime-boundary=BOUNDARY",
+                  "Use BOUNDARY for MIME boundary.",
+                  "(default: Use SHA1 digest of URI)") do |boundary|
+          @mime_formatter_options[:boundary] = boundary
+        end
         parser.on("--[no-]need-screenshot",
                   "Generate screenshot if available.",
                   "(default: #{@need_screenshot})") do |boolean|
@@ -220,7 +226,7 @@ module ChupaText
         when :text
           Formatters::Text.new($stdout)
         when :mime
-          Formatters::MIME.new($stdout)
+          Formatters::MIME.new($stdout, @mime_formatter_options)
         end
       end
     end

data/lib/chupa-text/decomposer.rb

@@ -30,6 +30,14 @@ module ChupaText
       raise NotImplementedError, "must implement #{self.class}\##{__method__}"
     end
 
+    def target_score(data)
+      if target?(data)
+        0
+      else
+        nil
+      end
+    end
+
     def decompose(data)
       raise NotImplementedError, "must implement #{self.class}\##{__method__}"
     end

data/lib/chupa-text/decomposers/office-open-xml-document.rb

@@ -0,0 +1,51 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "chupa-text/decomposers/office-open-xml"
+
+module ChupaText
+  module Decomposers
+    class OfficeOpenXMLDocument < OfficeOpenXML
+      registry.register("office-open-xml-document", self)
+
+      def initialize(options={})
+        super
+        @extensions = [
+          "docx",
+          "docm",
+          "dotx",
+          "dotm",
+        ]
+        @mime_types = [
+          "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+          "application/vnd.ms-word.document.macroEnabled.12",
+          "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+          "application/vnd.ms-word.template.macroEnabled.12",
+        ]
+        @namespace_uri =
+          "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+      end
+
+      private
+      def process_entry(entry, context)
+        case entry.zip_path
+        when "word/document.xml"
+          extract_text(entry, context[:text])
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/office-open-xml-presentation.rb

@@ -0,0 +1,67 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "chupa-text/decomposers/office-open-xml"
+
+module ChupaText
+  module Decomposers
+    class OfficeOpenXMLPresentation < OfficeOpenXML
+      registry.register("office-open-xml-presentation", self)
+
+      def initialize(options={})
+        super
+        @extensions = [
+          "pptx",
+          "pptm",
+          "ppsx",
+          "ppsm",
+          "potx",
+          "potm",
+          "sldx",
+          "sldm",
+        ]
+        @mime_types = [
+          "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+          "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+          "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+          "application/vnd.ms-powerpoint.slideshow.macroEnabled.12",
+          "application/vnd.openxmlformats-officedocument.presentationml.template",
+          "application/vnd.ms-powerpoint.template.macroEnabled.12",
+          "application/vnd.openxmlformats-officedocument.presentationml.slide",
+          "application/vnd.ms-powerpoint.slide.macroEnabled.12",
+        ]
+        @namespace_uri =
+          "http://schemas.openxmlformats.org/drawingml/2006/main"
+      end
+
+      private
+      def process_entry(entry, context)
+        case entry.zip_path
+        when /\Appt\/slides\/slide(\d+)\.xml/
+          nth_slide = Integer($1, 10)
+          slide_text = ""
+          extract_text(entry, slide_text)
+          context[:slides] ||= []
+          context[:slides] << [nth_slide, slide_text]
+        end
+      end
+
+      def accumulate_text(context)
+        context[:slides].sort_by(&:first).collect(&:last).join("\n")
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/office-open-xml-workbook.rb

@@ -0,0 +1,114 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "chupa-text/decomposers/office-open-xml"
+
+module ChupaText
+  module Decomposers
+    class OfficeOpenXMLWorkbook < OfficeOpenXML
+      registry.register("office-open-xml-workbook", self)
+
+      def initialize(options={})
+        super
+        @extensions = [
+          "xlsx",
+          "xlsm",
+          "xltx",
+          "xltm",
+        ]
+        @mime_types = [
+          "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+          "application/vnd.ms-excel.sheet.macroEnabled.12",
+          "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+          "application/vnd.ms-excel.template.macroEnabled.12",
+        ]
+        @namespace_uri =
+          "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
+      end
+
+      private
+      def process_entry(entry, context)
+        case entry.zip_path
+        when "xl/sharedStrings.xml"
+          context[:shared_strings] = []
+          extract_text(entry, context[:shared_strings])
+        when /\Axl\/worksheets\/sheet(\d+)\.xml\z/
+          nth_sheet = Integer($1, 10)
+          sheet = []
+          listener = SheetListener.new(sheet)
+          parse(entry.file_data, listener)
+          context[:sheets] ||= []
+          context[:sheets] << [nth_sheet, sheet]
+        end
+      end
+
+      def accumulate_text(context)
+        shared_strings = context[:shared_strings]
+        sheets = context[:sheets].sort_by(&:first).collect(&:last)
+        sheet_texts = sheets.collect do |sheet|
+          sheet_text = ""
+          sheet.each do |row|
+            row_texts = row.collect do |index|
+              shared_strings[index]
+            end
+            sheet_text << row_texts.join("\t") << "\n"
+          end
+          sheet_text
+        end
+        sheet_texts.join("\n")
+      end
+
+      class SheetListener
+        include REXML::SAX2Listener
+
+        URI = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
+
+        def initialize(sheet)
+          @sheet = sheet
+          @in_v = false
+        end
+
+        def start_element(uri, local_name, qname, attributes)
+          return unless uri == URI
+          case local_name
+          when "row"
+            @sheet << []
+          when "v"
+            @in_v = true
+          end
+        end
+
+        def end_element(uri, local_name, qname)
+          @in_v = false
+        end
+
+        def characters(text)
+          add_column(text)
+        end
+
+        def cdata(content)
+          add_column(content)
+        end
+
+        private
+        def add_column(text)
+          return unless @in_v
+          @sheet.last << Integer(text, 10)
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/office-open-xml.rb

@@ -0,0 +1,196 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "cgi/util"
+require "rexml/parsers/sax2parser"
+require "rexml/sax2listener"
+
+require "archive/zip"
+
+module ChupaText
+  module Decomposers
+    class OfficeOpenXML < Decomposer
+      def target?(data)
+        @extensions.include?(data.extension) or
+          @mime_types.include?(data.mime_type)
+      end
+
+      def target_score(data)
+        if target?(data)
+          -1
+        else
+          nil
+        end
+      end
+
+      def decompose(data)
+        context = {
+          text: "",
+          attributes: {},
+        }
+        data.open do |input|
+          Archive::Zip.open(input) do |zip|
+            zip.each do |entry|
+              next unless entry.file?
+              case entry.zip_path
+              when "docProps/app.xml"
+                listener = AttributesListener.new(context[:attributes])
+                parse(entry.file_data, listener)
+              when "docProps/core.xml"
+                listener = AttributesListener.new(context[:attributes])
+                parse(entry.file_data, listener)
+              else
+                process_entry(entry, context)
+              end
+            end
+          end
+        end
+        text = accumulate_text(context)
+        text_data = TextData.new(text, source_data: data)
+        context[:attributes].each do |name, value|
+          text_data[name] = value
+        end
+        yield(text_data)
+      end
+
+      private
+      def parse(io, listener)
+        source = REXML::Source.new(io.read)
+        parser = REXML::Parsers::SAX2Parser.new(source)
+        parser.listen(listener)
+        parser.parse
+      end
+
+      def extract_text(entry, texts)
+        listener = TextListener.new(texts, @namespace_uri)
+        parse(entry.file_data, listener)
+      end
+
+      def accumulate_text(context)
+        context[:text]
+      end
+
+      class TextListener
+        include REXML::SAX2Listener
+
+        def initialize(output, target_uri)
+          @output = output
+          @target_uri = target_uri
+          @in_target = false
+        end
+
+        def start_element(uri, local_name, qname, attributes)
+          return unless uri == @target_uri
+          case local_name
+          when "t"
+            @in_target = true
+          end
+        end
+
+        def end_element(uri, local_name, qname)
+          @in_target = false
+
+          return unless uri == @target_uri
+          case local_name
+          when "p", "br"
+            @output << "\n"
+          end
+        end
+
+        def characters(text)
+          add_text(text)
+        end
+
+        def cdata(content)
+          add_text(content)
+        end
+
+        private
+        def add_text(text)
+          return unless @in_target
+          @output << CGI.unescapeHTML(text)
+        end
+      end
+
+      class AttributesListener
+        include REXML::SAX2Listener
+
+        CORE_PROPERTIES_URI =
+          "http://schemas.openxmlformats.org/package/2006/metadata/core-properties"
+        EXTENDED_PROPERTIES_URI =
+          "http://schemas.openxmlformats.org/officeDocument/2006/extended-properties"
+        DUBLIN_CORE_URI = "http://purl.org/dc/elements/1.1/"
+        DUBLIN_CORE_TERMS_URI = "http://purl.org/dc/terms/"
+
+        def initialize(attributes)
+          @attributes = attributes
+          @name = nil
+          @type = nil
+        end
+
+        def start_element(uri, local_name, qname, attributes)
+          case uri
+          when CORE_PROPERTIES_URI
+            case local_name
+            when "keywords"
+              @name = local_name
+            end
+          when EXTENDED_PROPERTIES_URI
+            case local_name
+            when "Application"
+              @name = local_name.downcase
+            end
+          when DUBLIN_CORE_URI
+            case local_name
+            when "description", "title", "subject"
+              @name = local_name
+            end
+          when DUBLIN_CORE_TERMS_URI
+            case local_name
+            when "created", "modified"
+              @name = "#{local_name}_time"
+              @type = :w3cdtf
+            end
+          end
+        end
+
+        def end_element(uri, local_name, qname)
+          @name = nil
+          @type = nil
+        end
+
+        def characters(text)
+          set_attribute(text)
+        end
+
+        def cdata(content)
+          set_attribute(content)
+        end
+
+        def set_attribute(value)
+          return if @name.nil?
+
+          value = CGI.unescapeHTML(value)
+          case @type
+          when :w3cdtf
+            value = Time.xmlschema(value)
+          end
+          @attributes[@name] = value
+        end
+      end
+    end
+  end
+end