chupa-text 1.1.5 → 1.1.6

data/lib/chupa-text/decomposers/tar.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,6 +17,8 @@
 require "stringio"
 require "rubygems/package"
 
+require "chupa-text/path-converter"
+
 module ChupaText
   module Decomposers
     class Tar < Decomposer
@@ -28,18 +30,22 @@ module ChupaText
       end
 
       def decompose(data)
-        Gem::Package::TarReader.new(StringIO.new(data.body)) do |reader|
-          reader.each do |entry|
-            next unless entry.file?
+        data.open do |input|
+          Gem::Package::TarReader.new(input) do |reader|
+            reader.each do |entry|
+              next unless entry.file?
 
-            entry.extend(CopyStreamable)
-            entry_uri = data.uri.dup
-            base_path = entry_uri.path.gsub(/\.tar\z/i, "")
-            entry_uri.path = "#{base_path}/#{entry.full_name}"
-            extracted = VirtualFileData.new(entry_uri,
-                                            entry,
-                                            :source_data => data)
-            yield(extracted)
+              entry.extend(CopyStreamable)
+              entry_uri = data.uri.dup
+              base_path = entry_uri.path.gsub(/\.tar\z/i, "")
+              path_converter = PathConverter.new(entry.full_name,
+                                                 uri_escape: true)
+              entry_uri.path = "#{base_path}/#{path_converter.convert}"
+              extracted = VirtualFileData.new(entry_uri,
+                                              entry,
+                                              :source_data => data)
+              yield(extracted)
+            end
           end
         end
       end

data/lib/chupa-text/decomposers/zip.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2017-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,13 +15,16 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "stringio"
-require "tmpdir"
 
 require "archive/zip"
 
+require "chupa-text/path-converter"
+
 module ChupaText
   module Decomposers
     class Zip < Decomposer
+      include Loggable
+
       registry.register("zip", self)
 
       def target?(data)
@@ -32,7 +35,7 @@ module ChupaText
       end
 
       def decompose(data)
-        Archive::Zip.open(StringIO.new(data.body)) do |zip|
+        open_zip(data) do |zip|
           zip.each do |entry|
             next unless entry.file?
 
@@ -45,7 +48,10 @@ module ChupaText
             end
             entry_uri = data.uri.dup
             base_path = entry_uri.path.gsub(/\.zip\z/i, "")
-            entry_uri.path = "#{base_path}/#{entry.zip_path}"
+            path_converter = PathConverter.new(entry.zip_path,
+                                               encoding: base_path.encoding,
+                                               uri_escape: true)
+            entry_uri.path = "#{base_path}/#{path_converter.convert}"
             entry_data = VirtualFileData.new(entry_uri,
                                              entry.file_data,
                                              source_data: data)
@@ -53,6 +59,26 @@ module ChupaText
           end
         end
       end
+
+      private
+      def open_zip(data)
+        begin
+          Archive::Zip.open(StringIO.new(data.body)) do |zip|
+            yield(zip)
+          end
+        rescue Archive::Zip::Error => zip_error
+          error do
+            message = "#{log_tag} Failed to process zip: "
+            message << "#{zip_error.class}: #{zip_error.message}\n"
+            message << zip_error.backtrace.join("\n")
+            message
+          end
+        end
+      end
+
+      def log_tag
+        "[decomposer][zip]"
+      end
     end
   end
 end

data/lib/chupa-text/extractor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -68,11 +68,13 @@ module ChupaText
         if decomposer.nil?
           if target.text_plain?
             debug {"#{log_tag}[extract][text-plain]"}
-            yield(target)
+            yield(target.to_utf8_body_data)
             next
           else
             debug {"#{log_tag}[extract][decomposer] not found"}
-            yield(target) if target.text?
+            if target.text?
+              yield(target.to_utf8_body_data)
+            end
             next
           end
         end

data/lib/chupa-text/path-converter.rb

@@ -0,0 +1,70 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "cgi/util"
+
+module ChupaText
+  class PathConverter
+    def initialize(path, options={})
+      @path = path
+      @options = options
+    end
+
+    def convert
+      path = @path
+      encoding = @options[:encoding]
+      path = convert_encoding(path, encoding) if encoding
+      path = convert_to_uri_path(path) if @options[:uri_escape]
+      path
+    end
+
+    private
+    def convert_encoding(path, encoding)
+      case path.encoding
+      when Encoding::ASCII_8BIT
+        if path.ascii_only?
+          path.force_encoding(Encoding::UTF_8)
+        else
+          candidates = [
+            Encoding::UTF_8,
+            Encoding::EUC_JP,
+            Encoding::Windows_31J,
+          ]
+          found = false
+          candidates.find do |candidate|
+            path.force_encoding(candidate)
+            if path.valid_encoding?
+              found = true
+              break
+            end
+          end
+          path.force_encoding(Encoding::ASCII_8BIT) unless found
+        end
+      end
+      path.encode(encoding,
+                  invalid: :replace,
+                  undef: :replace,
+                  replace: "")
+    end
+
+    def convert_to_uri_path(path)
+      converted_components = path.split("/").collect do |component|
+        CGI.escape(component)
+      end
+      converted_components.join("/")
+    end
+  end
+end

data/lib/chupa-text/utf8-converter.rb

@@ -0,0 +1,117 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module ChupaText
+  class UTF8Converter
+    def initialize(string)
+      @string = string
+    end
+
+    def convert
+      encoding = @string.encoding
+      case encoding
+      when Encoding::UTF_8
+        bom_size, bom_encoding = detect_bom
+        if bom_size
+          return @string.byteslice(bom_size,
+                                   @string.bytesize - bom_size)
+        else
+          return @string
+        end
+      when Encoding::ASCII_8BIT
+        return @string if @string.ascii_only?
+      else
+        return @string.encode(Encoding::UTF_8,
+                              invalid: :replace,
+                              undef: :replace,
+                              replace: "")
+      end
+
+      bom_size, bom_encoding = detect_bom
+      if bom_encoding
+        string_without_bom = @string.byteslice(bom_size,
+                                               @string.bytesize - bom_size)
+        return string_without_bom.encode(Encoding::UTF_8,
+                                         bom_encoding,
+                                         invalid: :replace,
+                                         undef: :replace,
+                                         replace: "")
+      end
+
+      guessed_encoding = guess_encoding
+      if guessed_encoding
+        @string.encode(Encoding::UTF_8,
+                       guessed_encoding,
+                       invalid: :replace,
+                       undef: :replace,
+                       replace: "")
+      else
+        utf8_body = @string.dup
+        utf8_body.force_encoding(Encoding::UTF_8)
+        utf8_body.scrub!("")
+        utf8_body.gsub!(/\p{Control}+/, "")
+        utf8_body
+      end
+    end
+
+    private
+    UTF_8_BOM = "\xef\xbb\xbf".b
+    UTF_16BE_BOM = "\xfe\xff".b
+    UTF_16LE_BOM = "\xff\xfe".b
+    UTF_32BE_BOM = "\x00\x00\xfe\xff".b
+    UTF_32LE_BOM = "\xff\xfe\x00\x00".b
+    def detect_bom
+      case @string.byteslice(0, 4).b
+      when UTF_32BE_BOM
+        return 4, Encoding::UTF_32BE
+      when UTF_32LE_BOM
+        return 4, Encoding::UTF_32LE
+      end
+
+      case @string.byteslice(0, 3).b
+      when UTF_8_BOM
+        return 3, Encoding::UTF_8
+      end
+
+      case @string.byteslice(0, 2).b
+      when UTF_16BE_BOM
+        return 2, Encoding::UTF_16BE
+      when UTF_16LE_BOM
+        return 2, Encoding::UTF_16LE
+      end
+
+      nil
+    end
+
+    def guess_encoding
+      original_encoding = @string.encoding
+      begin
+        candidates = [
+          Encoding::UTF_8,
+          Encoding::EUC_JP,
+          Encoding::Windows_31J,
+        ]
+        candidates.each do |candidate|
+          @string.force_encoding(candidate)
+          return candidate if @string.valid_encoding?
+        end
+        nil
+      ensure
+        @string.force_encoding(original_encoding)
+      end
+    end
+  end
+end

data/lib/chupa-text/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module ChupaText
-  VERSION = "1.1.5"
+  VERSION = "1.1.6"
 end

data/test/command/test-chupa-text.rb

@@ -289,7 +289,7 @@ class TestCommandChupaText < Test::Unit::TestCase
                            "path"      => path.sub_ext(".txt").to_s,
                            "mime-type" => "text/plain",
                            "source-mime-types" => ["text/csv"],
-                           "body"      => "1 2 3\n4 5 6\n7 8 9\n",
+                           "body"      => "1\t2\t3\n4\t5\t6\n7\t8\t9\n",
                            "size"      => 18,
                            "screenshot" => {
                              "mime-type" => "image/svg+xml",
@@ -304,9 +304,9 @@ class TestCommandChupaText < Test::Unit::TestCase
     x="0"
     y="20"
     style="font-size: 20px; white-space: pre-wrap;"
-    xml:space="preserve">1 2 3
-4 5 6
-7 8 9
+    xml:space="preserve">1\t2\t3
+4\t5\t6
+7\t8\t9
 </text>
 </svg>
                              SVG

data/test/decomposers/test-csv.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -22,15 +22,30 @@ class TestDecomposersCSV< Test::Unit::TestCase
   end
 
   sub_test_case("decompose") do
-    def test_body
+    def test_valid
       csv = <<-CSV
 Hello,World
 Ruby,ChupaText
       CSV
-      assert_equal([csv.gsub(/,/, " ")],
+      assert_equal([csv.gsub(/,/, "\t")],
                    decompose(csv).collect(&:body))
     end
 
+    def test_invalid
+      messages = capture_log do
+        assert_equal([], decompose("He\x82\x00llo").collect(&:body))
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][csv] Failed to parse CSV: " +
+                       "CSV::MalformedCSVError: " +
+                       "Invalid byte sequence in UTF-8 in line 1.",
+                     ],
+                   ],
+                   messages)
+    end
+
     private
     def decompose(csv)
       data = ChupaText::Data.new

data/test/decomposers/test-office-open-xml-presentation.rb

@@ -48,41 +48,32 @@ class TestDecomposersOfficeOpenXMLPresentation < Test::Unit::TestCase
   sub_test_case("#decompose") do
     sub_test_case("attributes") do
       def decompose(attribute_name)
-        super(fixture_path("pptx", "attributes.pptx")).collect do |data|
-          data[attribute_name]
-        end
+        super(fixture_path("pptx", "attributes.pptx")).first[attribute_name]
       end
 
       def test_title
-        assert_equal(["Title"], decompose("title"))
+        assert_equal("Title", decompose("title"))
       end
 
       def test_author
-        assert_equal([nil], decompose("author"))
+        assert_equal(nil, decompose("author"))
       end
 
       def test_subject
-        assert_equal(["Subject"], decompose("subject"))
+        assert_equal("Subject", decompose("subject"))
       end
 
       def test_keywords
-        assert_equal(["Keyword1 Keyword2"], decompose("keywords"))
+        assert_equal("Keyword1 Keyword2", decompose("keywords"))
       end
 
       def test_modified_time
-        assert_equal([Time],
-                     decompose("modified_time").collect(&:class))
+        assert_equal(Time, decompose("modified_time").class)
       end
 
       def test_application
-        assert_equal(["LibreOffice"],
-                     normalize_applications(decompose("application")))
-      end
-
-      def normalize_applications(applications)
-        applications.collect do |application|
-          normalize_application(application)
-        end
+        assert_equal("LibreOffice",
+                     normalize_application(decompose("application")))
       end
 
       def normalize_application(application)
@@ -92,41 +83,50 @@ class TestDecomposersOfficeOpenXMLPresentation < Test::Unit::TestCase
           application
         end
       end
-
-      def test_creation_date
-        assert_equal([nil], decompose("creation_date"))
-      end
     end
 
-    sub_test_case("one slide") do
-      def decompose
-        super(fixture_path("pptx", "one-slide.pptx"))
-      end
-
-      def test_body
-        assert_equal([<<-BODY], decompose.collect(&:body))
-Slide1 title
-Slide1 content
-        BODY
+    sub_test_case("slides") do
+      def decompose(path)
+        super(path).collect do |data|
+          [
+            data["index"],
+            data.body,
+          ]
+        end
       end
-    end
 
-    sub_test_case("multi slides") do
-      def decompose
-        super(fixture_path("pptx", "multi-slides.pptx"))
+      def test_one_slide
+        assert_equal([
+                       [nil, ""],
+                       [
+                         0,
+                         "Slide1 title\n" +
+                         "Slide1 content\n",
+                       ],
+                     ],
+                     decompose(fixture_path("pptx", "one-slide.pptx")))
       end
 
-      def test_body
-        assert_equal([<<-BODY], decompose.collect(&:body))
-Slide1 title
-Slide1 content
-
-Slide2 title
-Slide2 content
-
-Slide3 title
-Slide3 content
-        BODY
+      def test_multi_slides
+        assert_equal([
+                       [nil, ""],
+                       [
+                         0,
+                         "Slide1 title\n" +
+                         "Slide1 content\n",
+                       ],
+                       [
+                         1,
+                         "Slide2 title\n" +
+                         "Slide2 content\n",
+                       ],
+                       [
+                         2,
+                         "Slide3 title\n" +
+                         "Slide3 content\n",
+                       ],
+                     ],
+                     decompose(fixture_path("pptx", "multi-slides.pptx")))
       end
     end
   end