chupa-text 1.1.5 → 1.1.6

data/test/decomposers/test-office-open-xml-workbook.rb

@@ -48,46 +48,36 @@ class TestDecomposersOfficeOpenXMLWorkbook < Test::Unit::TestCase
   sub_test_case("#decompose") do
     sub_test_case("attributes") do
       def decompose(attribute_name)
-        super(fixture_path("xlsx", "attributes.xlsx")).collect do |data|
-          data[attribute_name]
-        end
+        super(fixture_path("xlsx", "attributes.xlsx")).first[attribute_name]
       end
 
       def test_title
-        assert_equal(["Title"], decompose("title"))
+        assert_equal("Title", decompose("title"))
       end
 
       def test_author
-        assert_equal([nil], decompose("author"))
+        assert_equal(nil, decompose("author"))
       end
 
       def test_subject
-        assert_equal(["Subject"], decompose("subject"))
+        assert_equal("Subject", decompose("subject"))
       end
 
       def test_keywords
-        assert_equal(["Keyword1 Keyword2"], decompose("keywords"))
+        assert_equal("Keyword1 Keyword2", decompose("keywords"))
       end
 
       def test_created_time
-        assert_equal([Time],
-                     decompose("created_time").collect(&:class))
+        assert_equal(Time, decompose("created_time").class)
       end
 
       def test_modified_time
-        assert_equal([Time],
-                     decompose("modified_time").collect(&:class))
+        assert_equal(Time, decompose("modified_time").class)
       end
 
       def test_application
-        assert_equal(["LibreOffice"],
-                     normalize_applications(decompose("application")))
-      end
-
-      def normalize_applications(applications)
-        applications.collect do |application|
-          normalize_application(application)
-        end
+        assert_equal("LibreOffice",
+                     normalize_application(decompose("application")))
       end
 
       def normalize_application(application)
@@ -97,41 +87,69 @@ class TestDecomposersOfficeOpenXMLWorkbook < Test::Unit::TestCase
           application
         end
       end
-
-      def test_creation_date
-        assert_equal([nil], decompose("creation_date"))
-      end
     end
 
-    sub_test_case("one sheet") do
-      def decompose
-        super(fixture_path("xlsx", "one-sheet.xlsx"))
+    sub_test_case("sheets") do
+      def decompose(path)
+        super(path).collect do |data|
+          [
+            data["index"],
+            data["name"],
+            data.body,
+          ]
+        end
       end
 
-      def test_body
-        assert_equal([<<-BODY], decompose.collect(&:body))
-Sheet1 - A1\tSheet1 - B1
-Sheet1 - A2\tSheet1 - B2
-        BODY
+      def test_one_sheet
+        assert_equal([
+                       [nil, nil, ""],
+                       [
+                         0,
+                         "Sheet1",
+                         "Sheet1 - A1\tSheet1 - B1\n" +
+                         "Sheet1 - A2\tSheet1 - B2\n",
+                       ],
+                     ],
+                     decompose(fixture_path("xlsx", "one-sheet.xlsx")))
       end
-    end
 
-    sub_test_case("multi sheets") do
-      def decompose
-        super(fixture_path("xlsx", "multi-sheets.xlsx"))
+      def test_no_shared_cell
+        assert_equal([
+                       [nil, nil, ""],
+                       [
+                         0,
+                         "Sheet1",
+                         "Sheet1 - A1\tSheet1 - B1\n" +
+                         "Sheet1 - A2\tSheet1 - B2\n" +
+                         "0.5\t0.5\n",
+                       ],
+                     ],
+                     decompose(fixture_path("xlsx", "not-shared-cell.xlsx")))
       end
 
-      def test_body
-        assert_equal([<<-BODY], decompose.collect(&:body))
-Sheet1 - A1\tSheet1 - B1
-Sheet1 - A2\tSheet1 - B2
-
-Sheet2 - A1\tSheet2 - B1
-Sheet2 - A2\tSheet2 - B2
-
-Sheet3 - A1\tSheet3 - B1
-Sheet3 - A2\tSheet3 - B2
-        BODY
+      def test_multi_sheets
+        assert_equal([
+                       [nil, nil, ""],
+                       [
+                         0,
+                         "Sheet1",
+                         "Sheet1 - A1\tSheet1 - B1\n" +
+                         "Sheet1 - A2\tSheet1 - B2\n",
+                       ],
+                       [
+                         1,
+                         "Sheet2",
+                         "Sheet2 - A1\tSheet2 - B1\n" +
+                         "Sheet2 - A2\tSheet2 - B2\n",
+                       ],
+                       [
+                         2,
+                         "Sheet3",
+                         "Sheet3 - A1\tSheet3 - B1\n" +
+                         "Sheet3 - A2\tSheet3 - B2\n",
+                       ],
+                     ],
+                     decompose(fixture_path("xlsx", "multi-sheets.xlsx")))
       end
     end
   end

data/test/decomposers/{test-open-document-spreadsheet.rb → test-opendocument-spreadsheet.rb}

@@ -87,9 +87,9 @@ class TestDecomposersOpenDocumentSpreadsheet < Test::Unit::TestCase
       end
     end
 
-    sub_test_case("one sheet") do
-      def decompose
-        super(fixture_path("ods", "one-sheet.ods")).collect do |data|
+    sub_test_case("sheets") do
+      def decompose(path)
+        super(path).collect do |data|
           [
             data["index"],
             data["name"],
@@ -98,7 +98,7 @@ class TestDecomposersOpenDocumentSpreadsheet < Test::Unit::TestCase
         end
       end
 
-      def test_body
+      def test_one_sheet
         assert_equal([
                        [nil, nil, ""],
                        [
@@ -108,22 +108,10 @@ class TestDecomposersOpenDocumentSpreadsheet < Test::Unit::TestCase
                          "Sheet1 - A2\tSheet1 - B2\n",
                        ],
                      ],
-                     decompose)
-      end
-    end
-
-    sub_test_case("multi sheets") do
-      def decompose
-        super(fixture_path("ods", "multi-sheets.ods")).collect do |data|
-          [
-            data["index"],
-            data["name"],
-            data.body,
-          ]
-        end
+                     decompose(fixture_path("ods", "one-sheet.ods")))
       end
 
-      def test_body
+      def test_multi_sheets
         assert_equal([
                        [nil, nil, ""],
                        [
@@ -145,7 +133,35 @@ class TestDecomposersOpenDocumentSpreadsheet < Test::Unit::TestCase
                          "Sheet3 - A2\tSheet3 - B2\n",
                        ],
                      ],
-                     decompose)
+                     decompose(fixture_path("ods", "multi-sheets.ods")))
+      end
+
+      def test_covered_table_cell
+        assert_equal([
+                       [nil, nil, ""],
+                       [
+                         0,
+                         "Sheet1",
+                         "Covered-table-cell\t\n",
+                       ],
+                     ],
+                     decompose(fixture_path("ods", "covered-table-cell.ods")))
+      end
+
+      def test_shapes
+        assert_equal([
+                       [nil, nil, ""],
+                       [
+                         0,
+                         "Sheet1",
+                         "Shape1\n" +
+                         "Shape1\n" +
+                         "Shape2\n" +
+                         "Shape2\n" +
+                         "Cell\n",
+                       ],
+                     ],
+                     decompose(fixture_path("ods", "shapes.ods")))
       end
     end
   end

data/test/decomposers/test-tar.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -70,5 +70,22 @@ class TestDecomposersTar < Test::Unit::TestCase
                      decompose(data))
       end
     end
+
+    sub_test_case("multibyte") do
+      test("UTF-8") do
+        data_path = Pathname.new(fixture_path("utf-8.tar"))
+        base_path = data_path.sub_ext("")
+        data = ChupaText::InputData.new(data_path)
+        path = CGI.escape("こんにちは.txt")
+        assert_equal([
+                       {
+                         :uri    => file_uri("#{base_path}/utf-8/#{path}").to_s,
+                         :body   => "こんにちは\n".b,
+                         :source => data.uri.to_s,
+                       },
+                     ],
+                     decompose(data))
+      end
+    end
   end
 end

data/test/decomposers/test-zip.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2017-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -63,6 +63,36 @@ class TestDecomposersZip < Test::Unit::TestCase
                    decompose(data_path))
     end
 
+    sub_test_case("multibyte") do
+      test("cp932") do
+        data_path = Pathname.new(fixture_path("cp932.zip"))
+        base_path = data_path.sub_ext("")
+        path = CGI.escape("こんにちは.txt")
+        assert_equal([
+                       {
+                         :uri    => file_uri("#{base_path}/cp932/#{path}").to_s,
+                         :body   => "こんにちは\n".encode("cp932").b,
+                         :source => file_uri(data_path).to_s,
+                       },
+                     ],
+                     decompose(data_path))
+      end
+
+      test("UTF-8") do
+        data_path = Pathname.new(fixture_path("utf-8.zip"))
+        base_path = data_path.sub_ext("")
+        path = CGI.escape("こんにちは.txt")
+        assert_equal([
+                       {
+                         :uri    => file_uri("#{base_path}/utf-8/#{path}").to_s,
+                         :body   => "こんにちは\n".b,
+                         :source => file_uri(data_path).to_s,
+                       },
+                     ],
+                     decompose(data_path))
+      end
+    end
+
     sub_test_case("encrypted") do
       test("without password") do
         data_path = Pathname.new(fixture_path("password.zip"))

data/test/helper.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -32,4 +32,34 @@ module Helper
   def file_uri(path)
     URI.parse("file://#{path}")
   end
+
+
+  class CaptureLogger
+    def initialize(output)
+      @output = output
+    end
+
+    def error(message=nil)
+      @output << [:error, message || yield]
+    end
+  end
+
+  def capture_log
+    original_logger = ChupaText.logger
+    begin
+      output = []
+      ChupaText.logger = CaptureLogger.new(output)
+      yield
+      normalize_log(output)
+    ensure
+      ChupaText.logger = original_logger
+    end
+  end
+
+  def normalize_log(log)
+    log.collect do |level, message|
+      message = message.split("\n", 2)[0]
+      [level, message]
+    end
+  end
 end

data/test/test-data.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -44,11 +44,15 @@ class TestData < Test::Unit::TestCase
 
       sub_test_case("body") do
         def test_txt
-          body = "Hello"
-          body.force_encoding("ASCII-8BIT")
+          body = "Hello".b
           assert_equal("text/plain", guess(body))
         end
 
+        def test_utf8_valid_binary
+          body = "GROONGA:IO:00001@\0\0\0\0\0\0\0\0\0\0".b
+          assert_nil(guess(body))
+        end
+
         private
         def guess(body)
           @data.body = body

data/test/test-extractor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -121,5 +121,112 @@ class TestExtractor < Test::Unit::TestCase
         assert_equal(["Hello", "Hello"], extract(data))
       end
     end
+
+    sub_test_case("body") do
+      def test_utf8
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは"
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf8_ascii_8bit
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".b
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf8_broken
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "\x82\x00こんにちは".b
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf16_le
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".encode("UTF-16LE")
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf16_le_ascii_8bit
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "\ufeffこんにちは".encode("UTF-16LE").b
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf16_be
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".encode("UTF-16BE")
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf16_be_ascii_8bit
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "\ufeffこんにちは".encode("UTF-16BE").b
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf32_le
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".encode("UTF-32LE")
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf32_le_ascii_8bit
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "\ufeffこんにちは".encode("UTF-32LE").b
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf32_be
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".encode("UTF-32BE")
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_utf32_be_ascii_8bit
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "\ufeffこんにちは".encode("UTF-32BE").b
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_cp932
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".encode("cp932")
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_cp932_ascii_8bit
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".encode("cp932").b
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_euc_jp
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".encode("euc-jp")
+        assert_equal(["こんにちは"], extract(data))
+      end
+
+      def test_euc_jp_ascii_8bit
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "こんにちは".encode("euc-jp").b
+        assert_equal(["こんにちは"], extract(data))
+      end
+    end
   end
 end