chupa-text 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8e70028837d0c6abebb4d01432f3467e8c5dd42129b9dffa3dd16a939594894
-  data.tar.gz: 4a445af645995552c82e8339f1fd2d5f71dd8a47be669ee674ede6e635585c4f
+  metadata.gz: 99f896b313c052ecad3ea86ac6009c161cb2b3216b828aaec67da9dc886c5fbe
+  data.tar.gz: 986e35e1c19faa63652d0d8f43c9bd73e527e1d7af2593eea4525ee473414aa1
 SHA512:
-  metadata.gz: 1052a2c7148c1bc184633b1ed365f422584bcdb4bc1952534e24f2ebf1445a0f286cdf4d47241a7aeefcfce80e4bdfb7c5bf6e9e74fba30a5a1f8d333fde62b3
-  data.tar.gz: c2b355f73f09004214037069ffbdeafddd4e786c2e2498a905c9b651947f7508d1fc268052404e37297d2e816b7e115fa35ef959e98f22369f1e355220138b91
+  metadata.gz: 962932aed44748ca3c4809434d3e943ab5a113a6f57f058d29f7c60e6d1090f07d272d85221a5f62e9459f8a84f05725c7d5d33b15564d224dadb20215c155ae
+  data.tar.gz: 65face75476f34016b17b64cd0d843e305b87052192d944f2e60f5f06ea91b218e6d4fce1271eacf8f4de7152113fb87de86665b12b9fc479e80789dd0e428dd

data/doc/text/news.md

@@ -1,5 +1,25 @@
 # News
 
+## 1.1.8: 2019-03-03
+
+### Improvements
+
+  * `gzip`: Added error checks.
+
+  * `xml`:
+
+     * Added error checks.
+
+     * Added support for Nokogiri as an alternative backend.
+
+  * Reduced memory usage.
+
+  * Added support for body size limitation.
+
+  * `opendocument`: Added error checks.
+
+  * `office-open-xml`: Added error checks.
+
 ## 1.1.7: 2019-03-01
 
 ### Improvements

data/lib/chupa-text.rb

@@ -23,6 +23,7 @@ require "chupa-text/default-logger"
 require "chupa-text/logger"
 
 require "chupa-text/loggable"
+require "chupa-text/unzippable"
 
 require "chupa-text/configuration"
 require "chupa-text/configuration-loader"

data/lib/chupa-text/command/chupa-text.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -49,6 +49,7 @@ module ChupaText
         @mime_formatter_options = {}
         @need_screenshot = true
         @expected_screenshot_size = [200, 200]
+        @max_body_size = nil
       end
 
       def run(*arguments)
@@ -143,6 +144,11 @@ module ChupaText
                   "(default: #{@expected_screenshot_size.join("x")})") do |size|
           @expected_screenshot_size = size
         end
+        parser.on("--max-body-size=BYTE", Integer,
+                  "The max byte of extracted body.",
+                  "(default: no limit)") do |size|
+          @max_body_size = size
+        end
 
         parser.separator("")
         parser.separator("Log related options:")
@@ -190,7 +196,7 @@ module ChupaText
       end
 
       def create_extractor
-        extractor = Extractor.new
+        extractor = Extractor.new(max_body_size: @max_body_size)
         extractor.apply_configuration(@configuration)
         extractor
       end

data/lib/chupa-text/data.rb

@@ -141,7 +141,9 @@ module ChupaText
     end
 
     def peek_body(size)
-      body[0, size]
+      _body = body
+      return nil if _body.nil?
+      _body[0, size]
     end
 
     def [](name)
@@ -196,12 +198,20 @@ module ChupaText
       @need_screenshot
     end
 
-    def to_utf8_body_data
-      b = body
+    def to_utf8_body_data(max_body_size: nil)
+      b = nil
+      if max_body_size
+        open do |input|
+          b = input.read(max_body_size)
+        end
+      else
+        b = body
+      end
       return self if b.nil?
+
       converter = UTF8Converter.new(b)
       utf8_body = converter.convert
-      if b.equal?(utf8_body)
+      if max_body_size.nil? and b.equal?(utf8_body)
         self
       else
         TextData.new(utf8_body, source_data: self)

data/lib/chupa-text/decomposers/gzip.rb

@@ -19,6 +19,8 @@ require "zlib"
 module ChupaText
   module Decomposers
     class Gzip < Decomposer
+      include Loggable
+
       registry.register("gzip", self)
 
       TARGET_EXTENSIONS = ["gz", "tgz"]
@@ -33,8 +35,7 @@ module ChupaText
       end
 
       def decompose(data)
-        data.open do |input|
-          reader = Zlib::GzipReader.new(input)
+        open_reader(data) do |reader|
           uri = nil
           case data.extension
           when "gz"
@@ -46,6 +47,26 @@ module ChupaText
           yield(extracted)
         end
       end
+
+      private
+      def open_reader(data)
+        data.open do |input|
+          begin
+            yield(Zlib::GzipReader.new(input))
+          rescue Zlib::Error => zlib_error
+            error do
+              message = "#{log_tag} Failed to uncompress: "
+              message << "#{zlib_error.class}: #{zlib_error.message}\n"
+              message << zlib_error.backtrace.join("\n")
+              message
+            end
+          end
+        end
+      end
+
+      def log_tag
+        "[decomposer][gzip]"
+      end
     end
   end
 end

data/lib/chupa-text/decomposers/office-open-xml-document.rb

@@ -58,6 +58,10 @@ module ChupaText
         end
         yield(text_data)
       end
+
+      def log_tag
+        "#{super}[document]"
+      end
     end
   end
 end

data/lib/chupa-text/decomposers/office-open-xml-presentation.rb

@@ -76,6 +76,10 @@ module ChupaText
           yield(text_data)
         end
       end
+
+      def log_tag
+        "#{super}[presentation]"
+      end
     end
   end
 end

data/lib/chupa-text/decomposers/office-open-xml-workbook.rb

@@ -93,6 +93,10 @@ module ChupaText
         end
       end
 
+      def log_tag
+        "#{super}[workbook]"
+      end
+
       class WorkbookListener < SAXListener
         URI = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
 

data/lib/chupa-text/decomposers/office-open-xml.rb

@@ -14,13 +14,14 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "archive/zip"
-
 require "chupa-text/sax-parser"
 
 module ChupaText
   module Decomposers
     class OfficeOpenXML < Decomposer
+      include Loggable
+      include Unzippable
+
       def target?(data)
         @extensions.include?(data.extension) or
           @mime_types.include?(data.mime_type)
@@ -35,29 +36,27 @@ module ChupaText
       end
 
       def decompose(data, &block)
-        context = {
-          data: data,
-          attributes: {},
-        }
-        start_decompose(context)
-        data.open do |input|
-          Archive::Zip.open(input) do |zip|
-            zip.each do |entry|
-              next unless entry.file?
-              case entry.zip_path
-              when "docProps/app.xml"
-                listener = AttributesListener.new(context[:attributes])
-                parse(entry.file_data, listener)
-              when "docProps/core.xml"
-                listener = AttributesListener.new(context[:attributes])
-                parse(entry.file_data, listener)
-              else
-                process_entry(entry, context)
-              end
+        unzip(data) do |zip|
+          context = {
+            data: data,
+            attributes: {},
+          }
+          start_decompose(context)
+          zip.each do |entry|
+            next unless entry.file?
+            case entry.zip_path
+            when "docProps/app.xml"
+              listener = AttributesListener.new(context[:attributes])
+              parse(entry.file_data, listener)
+            when "docProps/core.xml"
+              listener = AttributesListener.new(context[:attributes])
+              parse(entry.file_data, listener)
+            else
+              process_entry(entry, context)
             end
           end
+          finish_decompose(context, &block)
         end
-        finish_decompose(context, &block)
       end
 
       private
@@ -71,8 +70,8 @@ module ChupaText
         parse(entry.file_data, listener)
       end
 
-      def accumulate_text(context)
-        context[:text]
+      def log_tag
+        "[decomposer][office-open-xml]"
       end
 
       class TextListener < SAXListener

data/lib/chupa-text/decomposers/opendocument-presentation.rb

@@ -49,6 +49,10 @@ module ChupaText
         end
       end
 
+      def log_tag
+        "#{super}[presentation]"
+      end
+
       class SlidesListener < SAXListener
         TEXT_URI = "urn:oasis:names:tc:opendocument:xmlns:text:1.0"
         DRAW_URI = "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"

data/lib/chupa-text/decomposers/opendocument-spreadsheet.rb

@@ -51,6 +51,10 @@ module ChupaText
         end
       end
 
+      def log_tag
+        "#{super}[spreadsheet]"
+      end
+
       class SheetsListener < SAXListener
         TEXT_URI = "urn:oasis:names:tc:opendocument:xmlns:text:1.0"
         TABLE_URI = "urn:oasis:names:tc:opendocument:xmlns:table:1.0"

data/lib/chupa-text/decomposers/opendocument-text.rb

@@ -43,6 +43,10 @@ module ChupaText
         yield(text_data)
       end
 
+      def log_tag
+        "#{super}[text]"
+      end
+
       class TextListener < SAXListener
         TEXT_URI = "urn:oasis:names:tc:opendocument:xmlns:text:1.0"
         def initialize(output)

data/lib/chupa-text/decomposers/opendocument.rb

@@ -14,13 +14,14 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "archive/zip"
-
 require "chupa-text/sax-parser"
 
 module ChupaText
   module Decomposers
     class OpenDocument < Decomposer
+      include Loggable
+      include Unzippable
+
       def target?(data)
         data.extension == @extension or
           data.mime_type == @mime_type
@@ -35,24 +36,22 @@ module ChupaText
       end
 
       def decompose(data, &block)
-        context = {
-          data: data,
-          attributes: {},
-        }
-        data.open do |input|
-          Archive::Zip.open(input) do |zip|
-            zip.each do |entry|
-              next unless entry.file?
-              case entry.zip_path
-              when "content.xml"
-                process_content(entry, context, &block)
-              when "meta.xml"
-                process_meta(entry, context, &block)
-              end
+        unzip(data) do |zip|
+          context = {
+            data: data,
+            attributes: {},
+          }
+          zip.each do |entry|
+            next unless entry.file?
+            case entry.zip_path
+            when "content.xml"
+              process_content(entry, context, &block)
+            when "meta.xml"
+              process_meta(entry, context, &block)
             end
           end
+          finish_decompose(context, &block)
         end
-        finish_decompose(context, &block)
       end
 
       private
@@ -66,6 +65,10 @@ module ChupaText
         parse(entry.file_data, listener)
       end
 
+      def log_tag
+        "[decomposer][opendocument]"
+      end
+
       class AttributesListener < SAXListener
         META_URI = "urn:oasis:names:tc:opendocument:xmlns:meta:1.0"
         DUBLIN_CORE_URI = "http://purl.org/dc/elements/1.1/"

data/lib/chupa-text/decomposers/xml.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,12 +14,13 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "rexml/document"
-require "rexml/streamlistener"
+require "chupa-text/sax-parser"
 
 module ChupaText
   module Decomposers
     class XML < Decomposer
+      include Loggable
+
       registry.register("xml", self)
 
       def target?(data)
@@ -31,22 +32,48 @@ module ChupaText
         text = ""
         listener = Listener.new(text)
         data.open do |input|
-          parser = REXML::Parsers::StreamParser.new(input, listener)
-          parser.parse
+          begin
+            parser = SAXParser.new(input, listener)
+            parser.parse
+          rescue SAXParser::ParseError => xml_error
+            error do
+              message = "#{log_tag} Failed to parse XML: "
+              message << "#{xml_error.class}: #{xml_error.message}\n"
+              message << xml_error.backtrace.join("\n")
+              message
+            end
+            return
+          end
         end
         text_data = TextData.new(text, :source_data => data)
         yield(text_data)
       end
 
-      class Listener
-        include REXML::StreamListener
+      private
+      def log_tag
+        "[decomposer][xml]"
+      end
 
+      class Listener < SAXListener
         def initialize(output)
           @output = output
+          @level = 0
+        end
+
+        def start_element(*args)
+          @level += 1
+        end
+
+        def end_element(*args)
+          @level -= 1
+        end
+
+        def characters(text)
+          @output << text if @level > 0
         end
 
-        def text(text)
-          @output << text
+        def cdata(content)
+          @output << content if @level > 0
         end
       end
     end

data/lib/chupa-text/decomposers/zip.rb

@@ -14,14 +14,13 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "archive/zip"
-
 require "chupa-text/path-converter"
 
 module ChupaText
   module Decomposers
     class Zip < Decomposer
       include Loggable
+      include Unzippable
 
       registry.register("zip", self)
 
@@ -33,7 +32,7 @@ module ChupaText
       end
 
       def decompose(data)
-        open_zip(data) do |zip|
+        unzip(data) do |zip|
           zip.each do |entry|
             next unless entry.file?
 
@@ -68,23 +67,6 @@ module ChupaText
       end
 
       private
-      def open_zip(data)
-        begin
-          data.open do |input|
-            Archive::Zip.open(input) do |zip|
-              yield(zip)
-            end
-          end
-        rescue Archive::Zip::Error => zip_error
-          error do
-            message = "#{log_tag} Failed to process zip: "
-            message << "#{zip_error.class}: #{zip_error.message}\n"
-            message << zip_error.backtrace.join("\n")
-            message
-          end
-        end
-      end
-
       def log_tag
         "[decomposer][zip]"
       end

data/lib/chupa-text/extractor.rb

@@ -21,8 +21,9 @@ module ChupaText
   class Extractor
     include Loggable
 
-    def initialize
+    def initialize(max_body_size: nil)
       @decomposers = []
+      @max_body_size = max_body_size
     end
 
     # Sets the extractor up by the configuration. It adds decomposers
@@ -57,38 +58,8 @@ module ChupaText
     #   You can get text data by `text_data.body`.
     #
     # @return [void]
-    def extract(input)
-      targets = [ensure_data(input)]
-      until targets.empty?
-        target = targets.shift
-        debug do
-          "#{log_tag}[extract][target] <#{target.uri}>:<#{target.mime_type}>"
-        end
-        decomposer = find_decomposer(target)
-        if decomposer.nil?
-          if target.text_plain?
-            debug {"#{log_tag}[extract][text-plain]"}
-            yield(target.to_utf8_body_data)
-            next
-          else
-            debug {"#{log_tag}[extract][decomposer] not found"}
-            if target.text?
-              yield(target.to_utf8_body_data)
-            end
-            next
-          end
-        end
-        debug {"#{log_tag}[extract][decomposer] #{decomposer.class}"}
-        decomposer.decompose(target) do |decomposed|
-          debug do
-            "#{log_tag}[extract][decomposed] " +
-              "#{decomposer.class}: " +
-              "<#{target.uri}>: " +
-              "<#{target.mime_type}> -> <#{decomposed.mime_type}>"
-          end
-          targets.push(decomposed)
-        end
-      end
+    def extract(input, &block)
+      extract_recursive(ensure_data(input), &block)
     end
 
     private
@@ -112,6 +83,35 @@ module ChupaText
       candidate[1]
     end
 
+    def extract_recursive(target, &block)
+      debug do
+        "#{log_tag}[extract][target] <#{target.uri}>:<#{target.mime_type}>"
+      end
+      decomposer = find_decomposer(target)
+      if decomposer.nil?
+        if target.text_plain?
+          debug {"#{log_tag}[extract][text-plain]"}
+          yield(target.to_utf8_body_data(max_body_size: @max_body_size))
+        else
+          debug {"#{log_tag}[extract][decomposer] not found"}
+          if target.text?
+            yield(target.to_utf8_body_data(max_body_size: @max_body_size))
+          end
+        end
+      else
+        debug {"#{log_tag}[extract][decomposer] #{decomposer.class}"}
+        decomposer.decompose(target) do |decomposed|
+          debug do
+            "#{log_tag}[extract][decomposed] " +
+              "#{decomposer.class}: " +
+              "<#{target.uri}>: " +
+              "<#{target.mime_type}> -> <#{decomposed.mime_type}>"
+          end
+          extract_recursive(decomposed, &block)
+        end
+      end
+    end
+
     def log_tag
       "[extractor]"
     end

data/lib/chupa-text/input-data.rb

@@ -36,6 +36,10 @@ module ChupaText
       @content.body
     end
 
+    def peek_body(size)
+      @content.peek_body(size)
+    end
+
     def size
       @content.size
     end

data/lib/chupa-text/sax-parser.rb

@@ -25,6 +25,9 @@ end
 
 module ChupaText
   class SAXParser
+    class ParseError < Error
+    end
+
     class << self
       def backend
         case ENV["CHUPA_TEXT_SAX_PARSER_BACKEND"]
@@ -94,6 +97,10 @@ module ChupaText
           @listener.cdata(content)
         end
 
+        def error(detail)
+          raise ParseError, detail
+        end
+
         private
         def build_qname(prefix, local_name)
           if prefix
@@ -105,10 +112,25 @@ module ChupaText
       end
     else
       def parse
-        source = REXML::Source.new(@input.read)
+        source = @input
+        if source.is_a?(Archive::Zip::Codec::Deflate::Decompress)
+          source = source.read
+        end
         parser = REXML::Parsers::SAX2Parser.new(source)
         parser.listen(Listener.new(@listener))
-        parser.parse
+        begin
+          parser.parse
+        rescue REXML::ParseException => error
+          message = "#{error.class}: #{error.message}"
+          raise ParseError, message
+        rescue ArgumentError => error
+          if error.message.start_with?("invalid byte sequence")
+            message = "#{error.class}: #{error.message}"
+            raise ParseError, message
+          else
+            raise
+          end
+        end
       end
 
       class Listener

data/lib/chupa-text/unzippable.rb

@@ -0,0 +1,39 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "archive/zip"
+
+module ChupaText
+  module Unzippable
+    private
+    def unzip(data)
+      data.open do |input|
+        begin
+          Archive::Zip.open(input) do |zip|
+            yield(zip)
+          end
+        rescue Archive::Zip::Error => zip_error
+          error do
+            message = "#{log_tag} Failed to process zip: "
+            message << "#{zip_error.class}: #{zip_error.message}\n"
+            message << zip_error.backtrace.join("\n")
+            message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/utf8-converter.rb

@@ -16,8 +16,9 @@
 
 module ChupaText
   class UTF8Converter
-    def initialize(string)
+    def initialize(string, max_size: nil)
       @string = string
+      @max_size = max_size
     end
 
     def convert
@@ -26,44 +27,51 @@ module ChupaText
       when Encoding::UTF_8
         bom_size, bom_encoding = detect_bom
         if bom_size
-          return @string.byteslice(bom_size,
-                                   @string.bytesize - bom_size)
+          utf8_string = @string.byteslice(bom_size,
+                                          @string.bytesize - bom_size)
         else
-          return @string
+          utf8_string = @string
         end
+        return truncate(utf8_string)
       when Encoding::ASCII_8BIT
-        return @string if @string.ascii_only?
+        return truncate(@string) if @string.ascii_only?
       else
-        return @string.encode(Encoding::UTF_8,
-                              invalid: :replace,
-                              undef: :replace,
-                              replace: "")
+        utf8_string = @string.encode(Encoding::UTF_8,
+                                     invalid: :replace,
+                                     undef: :replace,
+                                     replace: "")
+        return truncate(utf8_string)
       end
 
       bom_size, bom_encoding = detect_bom
       if bom_encoding
         string_without_bom = @string.byteslice(bom_size,
                                                @string.bytesize - bom_size)
-        return string_without_bom.encode(Encoding::UTF_8,
-                                         bom_encoding,
-                                         invalid: :replace,
-                                         undef: :replace,
-                                         replace: "")
+        utf8_string = string_without_bom.encode(Encoding::UTF_8,
+                                                bom_encoding,
+                                                invalid: :replace,
+                                                undef: :replace,
+                                                replace: "")
+        return truncate(utf8_string)
       end
 
       guessed_encoding = guess_encoding
       if guessed_encoding
-        @string.encode(Encoding::UTF_8,
-                       guessed_encoding,
-                       invalid: :replace,
-                       undef: :replace,
-                       replace: "")
+        truncate(@string.encode(Encoding::UTF_8,
+                                guessed_encoding,
+                                invalid: :replace,
+                                undef: :replace,
+                                replace: ""))
       else
-        utf8_body = @string.dup
-        utf8_body.force_encoding(Encoding::UTF_8)
-        utf8_body.scrub!("")
-        utf8_body.gsub!(/\p{Control}+/, "")
-        utf8_body
+        if @max_size
+          utf8_string = @string.byteslice(0, @max_size)
+        else
+          utf8_string = @string.dup
+        end
+        utf8_string.force_encoding(Encoding::UTF_8)
+        utf8_string.scrub!("")
+        utf8_string.gsub!(/\p{Control}+/, "")
+        utf8_string
       end
     end
 
@@ -113,5 +121,15 @@ module ChupaText
         @string.force_encoding(original_encoding)
       end
     end
+
+    def truncate(string)
+      if @max_size and string.bytesize > @max_size
+        truncated = string.byteslice(0, @max_size)
+        truncated.scrub!("")
+        truncated
+      else
+        string
+      end
+    end
   end
 end

data/lib/chupa-text/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module ChupaText
-  VERSION = "1.1.7"
+  VERSION = "1.1.8"
 end

data/test/decomposers/test-gzip.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -83,7 +83,6 @@ class TestDecomposersGzip < Test::Unit::TestCase
       end
     end
 
-
     sub_test_case("tgz") do
       def setup
         super
@@ -109,5 +108,23 @@ class TestDecomposersGzip < Test::Unit::TestCase
                      decompose(@data).collect(&:source))
       end
     end
+
+    def test_invalid
+      data = ChupaText::Data.new
+      data.body = "Hello"
+      data.size = data.body.bytesize
+      data.mime_type = "application/gzip"
+      messages = capture_log do
+        assert_equal([], decompose(data).collect(&:body))
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][gzip] Failed to uncompress: " +
+                       "Zlib::GzipFile::Error: not in gzip format",
+                     ],
+                   ],
+                   messages)
+    end
   end
 end

data/test/decomposers/test-office-open-xml-document.rb

@@ -140,5 +140,23 @@ Single quote: ''
         BODY
       end
     end
+
+    sub_test_case("invalid") do
+      def test_empty
+        messages = capture_log do
+          assert_equal([], decompose(fixture_path("docx", "empty.docx")))
+        end
+        assert_equal([
+                       [
+                         :error,
+                         "[decomposer][office-open-xml][document] " +
+                         "Failed to process zip: " +
+                         "Archive::Zip::UnzipError: " +
+                         "unable to locate end-of-central-directory record",
+                       ],
+                     ],
+                     messages)
+      end
+    end
   end
 end

data/test/decomposers/test-office-open-xml-presentation.rb

@@ -129,5 +129,23 @@ class TestDecomposersOfficeOpenXMLPresentation < Test::Unit::TestCase
                      decompose(fixture_path("pptx", "multi-slides.pptx")))
       end
     end
+
+    sub_test_case("invalid") do
+      def test_empty
+        messages = capture_log do
+          assert_equal([], decompose(fixture_path("pptx", "empty.pptx")))
+        end
+        assert_equal([
+                       [
+                         :error,
+                         "[decomposer][office-open-xml][presentation] " +
+                         "Failed to process zip: " +
+                         "Archive::Zip::UnzipError: " +
+                         "unable to locate end-of-central-directory record",
+                       ],
+                     ],
+                     messages)
+      end
+    end
   end
 end

data/test/decomposers/test-office-open-xml-workbook.rb

@@ -152,5 +152,23 @@ class TestDecomposersOfficeOpenXMLWorkbook < Test::Unit::TestCase
                      decompose(fixture_path("xlsx", "multi-sheets.xlsx")))
       end
     end
+
+    sub_test_case("invalid") do
+      def test_empty
+        messages = capture_log do
+          assert_equal([], decompose(fixture_path("xlsx", "empty.xlsx")))
+        end
+        assert_equal([
+                       [
+                         :error,
+                         "[decomposer][office-open-xml][workbook] " +
+                         "Failed to process zip: " +
+                         "Archive::Zip::UnzipError: " +
+                         "unable to locate end-of-central-directory record",
+                       ],
+                     ],
+                     messages)
+      end
+    end
   end
 end

data/test/decomposers/test-opendocument-presentation.rb

@@ -132,5 +132,23 @@ class TestDecomposersOpenDocumentPresentation < Test::Unit::TestCase
                      decompose.collect {|data| [data["index"], data.body]})
       end
     end
+
+    sub_test_case("invalid") do
+      def test_empty
+        messages = capture_log do
+          assert_equal([], decompose(fixture_path("odp", "empty.odp")))
+        end
+        assert_equal([
+                       [
+                         :error,
+                         "[decomposer][opendocument][presentation] " +
+                         "Failed to process zip: " +
+                         "Archive::Zip::UnzipError: " +
+                         "unable to locate end-of-central-directory record",
+                       ],
+                     ],
+                     messages)
+      end
+    end
   end
 end

data/test/decomposers/test-opendocument-spreadsheet.rb

@@ -164,5 +164,23 @@ class TestDecomposersOpenDocumentSpreadsheet < Test::Unit::TestCase
                      decompose(fixture_path("ods", "shapes.ods")))
       end
     end
+
+    sub_test_case("invalid") do
+      def test_empty
+        messages = capture_log do
+          assert_equal([], decompose(fixture_path("ods", "empty.ods")))
+        end
+        assert_equal([
+                       [
+                         :error,
+                         "[decomposer][opendocument][spreadsheet] " +
+                         "Failed to process zip: " +
+                         "Archive::Zip::UnzipError: " +
+                         "unable to locate end-of-central-directory record",
+                       ],
+                     ],
+                     messages)
+      end
+    end
   end
 end

data/test/decomposers/test-opendocument-text.rb

@@ -140,5 +140,23 @@ Single quote: ''
         BODY
       end
     end
+
+    sub_test_case("invalid") do
+      def test_empty
+        messages = capture_log do
+          assert_equal([], decompose(fixture_path("odt", "empty.odt")))
+        end
+        assert_equal([
+                       [
+                         :error,
+                         "[decomposer][opendocument][text] " +
+                         "Failed to process zip: " +
+                         "Archive::Zip::UnzipError: " +
+                         "unable to locate end-of-central-directory record",
+                       ],
+                     ],
+                     messages)
+      end
+    end
   end
 end

data/test/decomposers/test-xml.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -35,12 +35,40 @@ class TestDecomposersXML < Test::Unit::TestCase
   Hello
   &
   World
-
       TEXT
       assert_equal([text],
                    decompose(xml).collect(&:body))
     end
 
+    def test_invalid_xml
+      messages = capture_log do
+        assert_equal([], decompose("<root x=/>"))
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][xml] Failed to parse XML: " +
+                       "ChupaText::SAXParser::ParseError: ...",
+                     ],
+                   ],
+                   messages)
+    end
+
+    def test_invalid_encoding
+      messages = capture_log do
+        assert_equal([],
+                     decompose("\x00\x05\a\xA6"))
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][xml] Failed to parse XML: " +
+                       "ChupaText::SAXParser::ParseError: ...",
+                     ],
+                   ],
+                   messages)
+    end
+
     private
     def decompose(xml)
       data = ChupaText::Data.new
@@ -54,5 +82,16 @@ class TestDecomposersXML < Test::Unit::TestCase
       end
       decomposed
     end
+
+    def capture_log
+      messages = super
+      messages.collect do |level, message|
+        [
+          level,
+          message.gsub(/(ChupaText::SAXParser::ParseError:) .*/,
+                       "\\1 ...")
+        ]
+      end
+    end
   end
 end

data/test/test-extractor.rb

@@ -228,5 +228,15 @@ class TestExtractor < Test::Unit::TestCase
         assert_equal(["こんにちは"], extract(data))
       end
     end
+
+    sub_test_case("max body size") do
+      def test_last_invalid
+        @extractor = ChupaText::Extractor.new(max_body_size: 5)
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こん"
+        assert_equal(["こ"], extract(data))
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-01 00:00:00.000000000 Z
+date: 2019-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: archive-zip
@@ -189,6 +189,7 @@ files:
 - lib/chupa-text/screenshot.rb
 - lib/chupa-text/size-parser.rb
 - lib/chupa-text/text-data.rb
+- lib/chupa-text/unzippable.rb
 - lib/chupa-text/utf8-converter.rb
 - lib/chupa-text/version.rb
 - lib/chupa-text/virtual-content.rb
@@ -210,6 +211,7 @@ files:
 - test/fixture/command/chupa-text/no-decomposer.conf
 - test/fixture/command/chupa-text/numbers.csv
 - test/fixture/docx/attributes.docx
+- test/fixture/docx/empty.docx
 - test/fixture/docx/multi-pages.docx
 - test/fixture/docx/one-page.docx
 - test/fixture/docx/special-characters.docx
@@ -218,24 +220,29 @@ files:
 - test/fixture/gzip/hello.tgz
 - test/fixture/gzip/hello.txt.gz
 - test/fixture/odp/attributes.odp
+- test/fixture/odp/empty.odp
 - test/fixture/odp/multi-slides.odp
 - test/fixture/odp/one-slide.odp
 - test/fixture/ods/attributes.ods
 - test/fixture/ods/covered-table-cell.ods
+- test/fixture/ods/empty.ods
 - test/fixture/ods/multi-sheets.ods
 - test/fixture/ods/one-sheet.ods
 - test/fixture/ods/shapes.ods
 - test/fixture/odt/attributes.odt
+- test/fixture/odt/empty.odt
 - test/fixture/odt/multi-pages.odt
 - test/fixture/odt/one-page.odt
 - test/fixture/odt/special-characters.odt
 - test/fixture/pptx/attributes.pptx
+- test/fixture/pptx/empty.pptx
 - test/fixture/pptx/multi-slides.pptx
 - test/fixture/pptx/one-slide.pptx
 - test/fixture/tar/directory.tar
 - test/fixture/tar/top-level.tar
 - test/fixture/tar/utf-8.tar
 - test/fixture/xlsx/attributes.xlsx
+- test/fixture/xlsx/empty.xlsx
 - test/fixture/xlsx/multi-sheets.xlsx
 - test/fixture/xlsx/not-shared-cell.xlsx
 - test/fixture/xlsx/one-sheet.xlsx