chupa-text 1.0.8 → 1.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 996c7628251b6467736c48363ff31cf3b2eedb8a
-  data.tar.gz: 53a0af15b4e03a8f59747bdbc30ff49209bdd796
+  metadata.gz: 683591985ea7d93bfde2dc2643964120d18c6a04
+  data.tar.gz: e0b1419f2d3a020d4036afbeb56c4b1366b8e4d8
 SHA512:
-  metadata.gz: e62ddb8fac42f82f1dea609b18fee3102ce4019a1e93d04dccc72505a14f5a2fb0b722a210cffb67a0cfa3857ae2af480137cd21ddb3624accf5ce9349e9a13f
-  data.tar.gz: 20496a45e2bee52e16d081a4fbdd3d41dd477183e82bd086a945ffc61ea955ece66d0a3e6094b324070315af3dd34375898bd9450e7bb801195b52a645ff9e7d
+  metadata.gz: c9b33ecfc733c19827d0ce2a8484f7cd4a128a1435616dae03bf42707b11e8c99bd3402b885fa2af073424a87e786220008f8bf5c02912aaad5874a1f9a5add4
+  data.tar.gz: 85f8df4be72276db69c8938120c9ccd99bbf424493cf69f8b2357cb5e81053da9fcc35c386c3d1816ad53ea39d3de478ee9606e49783e2ed9b74631296cc413a

data/chupa-text.gemspec

@@ -50,6 +50,8 @@ Gem::Specification.new do |spec|
     spec.executables = Dir.glob("*")
   end
 
+  spec.add_runtime_dependency("archive-zip")
+
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("rake")
   spec.add_development_dependency("test-unit")

data/data/mime-types.conf

@@ -7,6 +7,8 @@ mime_types["tgz"] = "application/x-gtar-compressed"
 
 mime_types["tar"] = "application/x-tar"
 
+mime_types["zip"] = "application/zip"
+
 mime_types["htm"]   = "text/html"
 mime_types["html"]  = "text/html"
 mime_types["xhtml"] = "application/xhtml+xml"

data/doc/text/news.md

@@ -1,5 +1,21 @@
 # News
 
+## 1.0.9: 2017-07-11
+
+### Improvements
+
+  * `ChupaText::TextData`: Changed extension to ".txt".
+
+  * `chupa-text`: Added `--uri` option.
+
+  * `chupa-text`: Added `--mime-type` option.
+
+  * `ChupaText::DownloadError`: Added.
+
+  * Supported zip.
+
+  * `ChupaText::ExternalCommand#path`: Added.
+
 ## 1.0.8: 2017-07-10
 
 ### Improvements

data/lib/chupa-text/command/chupa-text.rb

@@ -43,6 +43,8 @@ module ChupaText
         @input = nil
         @configuration = Configuration.load_default
         @enable_gems = true
+        @uri = nil
+        @mime_type = nil
         @format = :json
         @need_screenshot = true
         @expected_screenshot_size = [200, 200]
@@ -106,6 +108,17 @@ module ChupaText
           $LOAD_PATH << path
         end
 
+        parser.separator("")
+        parser.separator("Input related options")
+        parser.on("--uri=URI",
+                  "Input data URI.") do |uri|
+          @uri = URI.parse(uri)
+        end
+        parser.on("--mime-type=MIME_TYPE",
+                  "Input data MIME type.") do |mime_type|
+          @mime_type = mime_type
+        end
+
         parser.separator("")
         parser.separator("Output related options")
         parser.on("--format=FORMAT", AVAILABLE_FORMATS,
@@ -178,7 +191,7 @@ module ChupaText
 
       def create_data
         if @input.nil?
-          data = VirtualFileData.new(nil, $stdin)
+          data = VirtualFileData.new(@uri, $stdin)
         else
           case @input
           when /\A[a-z]+:\/\//i
@@ -186,8 +199,15 @@ module ChupaText
           else
             input = Pathname(@input)
           end
-          data = InputData.new(input)
+          if @uri
+            input.open("rb") do |io|
+              data = VirtualFileData.new(@uri, io)
+            end
+          else
+            data = InputData.new(input)
+          end
         end
+        data.mime_type = @mime_type if @mime_type
         data.need_screenshot = @need_screenshot
         data.expected_screenshot_size = @expected_screenshot_size
         data

data/lib/chupa-text/decomposers/tar.rb

@@ -31,8 +31,13 @@ module ChupaText
         Gem::Package::TarReader.new(StringIO.new(data.body)) do |reader|
           reader.each do |entry|
             next unless entry.file?
+
             entry.extend(CopyStreamable)
-            extracted = VirtualFileData.new(entry.full_name, entry,
+            entry_uri = data.uri.dup
+            base_path = entry_uri.path.gsub(/\.tar\z/i, "")
+            entry_uri.path = "#{base_path}/#{entry.full_name}"
+            extracted = VirtualFileData.new(entry_uri,
+                                            entry,
                                             :source_data => data)
             yield(extracted)
           end

data/lib/chupa-text/decomposers/zip.rb

@@ -0,0 +1,58 @@
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "stringio"
+require "tmpdir"
+
+require "archive/zip"
+
+module ChupaText
+  module Decomposers
+    class Zip < Decomposer
+      registry.register("zip", self)
+
+      def target?(data)
+        return true if data.extension == "zip"
+        return true if data.mime_type == "application/zip"
+
+        false
+      end
+
+      def decompose(data)
+        Archive::Zip.open(StringIO.new(data.body)) do |zip|
+          zip.each do |entry|
+            next unless entry.file?
+
+            case entry.encryption_codec
+            when Archive::Zip::Codec::NullEncryption
+            else
+              # TODO
+              # entry.password = ...
+              raise EncryptedError.new(data)
+            end
+            entry_uri = data.uri.dup
+            base_path = entry_uri.path.gsub(/\.zip\z/i, "")
+            entry_uri.path = "#{base_path}/#{entry.zip_path}"
+            entry_data = VirtualFileData.new(entry_uri,
+                                             entry.file_data,
+                                             source_data: data)
+            yield(entry_data)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/error.rb

@@ -18,6 +18,16 @@ module ChupaText
   class Error < StandardError
   end
 
+  class DownloadError < Error
+    attr_reader :uri
+    attr_reader :reason
+    def initialize(uri, reason)
+      @uri = uri
+      @reason = reason
+      super("Download error: <#{uri}>: #{reason}")
+    end
+  end
+
   class EncryptedError < Error
     attr_reader :data
     def initialize(data)

data/lib/chupa-text/external-command.rb

@@ -20,8 +20,9 @@ require "pathname"
 
 module ChupaText
   class ExternalCommand
-    def initialize(command)
-      @command = Pathname.new(command)
+    attr_reader :path
+    def initialize(path)
+      @path = Pathname.new(path)
     end
 
     def run(*arguments)
@@ -32,19 +33,19 @@ module ChupaText
       end
       spawn_options = options[:spawn_options] || {}
       pid = spawn(options[:env] || {},
-                  @command.to_s,
+                  @path.to_s,
                   *arguments,
-                  spawn_options.merge(default_spawn_options))
+                  default_spawn_options.merge(spawn_options))
       pid, status = Process.waitpid2(pid)
       status.success?
     end
 
     def exist?
-      if @command.absolute?
-        @command.file? and @command.executable?
+      if @path.absolute?
+        @path.file? and @path.executable?
       else
         (ENV['PATH'] || "").split(File::PATH_SEPARATOR).any? do |path|
-          (Pathname.new(path) + @command).expand_path.exist?
+          (Pathname.new(path) + @path).expand_path.exist?
         end
       end
     end

data/lib/chupa-text/input-data.rb

@@ -47,9 +47,13 @@ module ChupaText
     def download
       path = @uri.path
       path += "index.html" if path.end_with?("/")
-      @uri.open("rb") do |input|
-        self.mime_type = input.content_type.split(/;/).first
-        VirtualContent.new(input, path)
+      begin
+        @uri.open("rb") do |input|
+          self.mime_type = input.content_type.split(/;/).first
+          VirtualContent.new(input, path)
+        end
+      rescue OpenURI::HTTPError => error
+        raise DownloadError.new(@uri, error.message.strip)
       end
     end
   end

data/lib/chupa-text/text-data.rb

@@ -18,6 +18,8 @@ module ChupaText
   class TextData < Data
     def initialize(text, options={})
       super(options)
+      self.uri = uri.to_s.gsub(/\.[^.]+\z/, ".txt")
+      self.path = path.to_s.gsub(/\.[^.]+\z/, ".txt")
       self.mime_type = "text/plain"
       self.body = text
       self.size = text.bytesize

data/lib/chupa-text/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module ChupaText
-  VERSION = "1.0.8"
+  VERSION = "1.0.9"
 end

data/test/command/test-chupa-text.rb

@@ -116,7 +116,7 @@ class TestCommandChupaText < Test::Unit::TestCase
         @www_server_thread.kill
       end
 
-      def test_single
+      test("default") do
         @html = "<html><body>Hello</body></html>"
         assert_equal([
                        true,
@@ -136,10 +136,53 @@ class TestCommandChupaText < Test::Unit::TestCase
                      ],
                      run_command(@uri))
       end
+
+      test("--uri") do
+        virtual_uri = "http://127.0.0.1/hello.html"
+        @html = "<html><body>Hello</body></html>"
+        assert_equal([
+                       true,
+                       {
+                         "mime-type" => "text/html",
+                         "size"      => @html.bytesize,
+                         "uri"       => virtual_uri,
+                         "texts"     => [
+                           {
+                             "mime-type" => "text/html",
+                             "size"      => @html.bytesize,
+                             "uri"       => virtual_uri,
+                             "body"      => @html,
+                           },
+                         ],
+                       },
+                     ],
+                     run_command(@uri, "--uri", virtual_uri))
+      end
+
+      test("--mime-type") do
+        @html = "<html><body>Hello</body></html>"
+        assert_equal([
+                       true,
+                       {
+                         "mime-type" => "text/plain",
+                         "size"      => @html.bytesize,
+                         "uri"       => @uri,
+                         "texts"     => [
+                           {
+                             "mime-type" => "text/plain",
+                             "size"      => @html.bytesize,
+                             "uri"       => @uri,
+                             "body"      => @html,
+                           },
+                         ],
+                       },
+                     ],
+                     run_command(@uri, "--mime-type", "text/plain"))
+      end
     end
 
     sub_test_case("standard input") do
-      def test_single
+      test("default") do
         body = "Hello\n"
         @stdin << "Hello\n"
         @stdin.rewind
@@ -159,6 +202,51 @@ class TestCommandChupaText < Test::Unit::TestCase
                      ],
                      run_command)
       end
+
+      test("--uri") do
+        body = "Hello\n"
+        uri = "http://127.0.0.1/hello.txt"
+        @stdin << "Hello\n"
+        @stdin.rewind
+        assert_equal([
+                       true,
+                       {
+                         "mime-type" => "text/plain",
+                         "size"      => body.bytesize,
+                         "uri"       => uri,
+                         "texts"     => [
+                           {
+                             "mime-type" => "text/plain",
+                             "size"      => body.bytesize,
+                             "body"      => body,
+                             "uri"       => uri,
+                           },
+                         ],
+                       },
+                     ],
+                     run_command("--uri", "http://127.0.0.1/hello.txt"))
+      end
+
+      test("--mime-type") do
+        body = "Hello\n"
+        @stdin << "Hello\n"
+        @stdin.rewind
+        assert_equal([
+                       true,
+                       {
+                         "mime-type" => "text/html",
+                         "size"      => body.bytesize,
+                         "texts"     => [
+                           {
+                             "mime-type" => "text/html",
+                             "size"      => body.bytesize,
+                             "body"      => body,
+                           },
+                         ],
+                       },
+                     ],
+                     run_command("--mime-type", "text/html"))
+      end
     end
   end
 
@@ -197,8 +285,8 @@ class TestCommandChupaText < Test::Unit::TestCase
                        "size"      => path.stat.size,
                        "texts"     => [
                          {
-                           "uri"       => uri.to_s,
-                           "path"      => path.to_s,
+                           "uri"       => uri.to_s.gsub(/\.csv\z/, ".txt"),
+                           "path"      => path.sub_ext(".txt").to_s,
                            "mime-type" => "text/plain",
                            "source-mime-types" => ["text/csv"],
                            "body"      => "1 2 3\n4 5 6\n7 8 9\n",

data/test/decomposers/test-tar.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -40,38 +40,34 @@ class TestDecomposersTar < Test::Unit::TestCase
     end
 
     sub_test_case("top-level") do
-      def setup
-        super
-        @data = ChupaText::InputData.new(fixture_path("top-level.tar"))
-      end
-
       def test_decompose
+        data_path = Pathname.new(fixture_path("top-level.tar"))
+        base_path = data_path.sub_ext("")
+        data = ChupaText::InputData.new(data_path)
         assert_equal([
                        {
-                         :uri    => "top-level.txt",
+                         :uri    => "file:#{base_path}/top-level.txt",
                          :body   => "top level\n",
-                         :source => @data.uri.to_s,
+                         :source => data.uri.to_s,
                        },
                      ],
-                     decompose(@data))
+                     decompose(data))
       end
     end
 
     sub_test_case("directory") do
-      def setup
-        super
-        @data = ChupaText::InputData.new(fixture_path("directory.tar"))
-      end
-
       def test_decompose
+        data_path = Pathname.new(fixture_path("directory.tar"))
+        base_path = data_path.sub_ext("")
+        data = ChupaText::InputData.new(data_path)
         assert_equal([
                        {
-                         :uri    => "directory/hello.txt",
+                         :uri    => "file:#{base_path}/directory/hello.txt",
                          :body   => "Hello in directory\n",
-                         :source => @data.uri.to_s,
+                         :source => data.uri.to_s,
                        },
                      ],
-                     decompose(@data))
+                     decompose(data))
       end
     end
   end

data/test/decomposers/test-zip.rb

@@ -0,0 +1,100 @@
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestDecomposersZip < Test::Unit::TestCase
+  include Helper
+
+  def setup
+    @decomposer = ChupaText::Decomposers::Zip.new({})
+  end
+
+  private
+  def fixture_path(*components)
+    super("zip", *components)
+  end
+
+  sub_test_case("decompose") do
+    def decompose(data_path)
+      data = ChupaText::InputData.new(data_path)
+      decomposed = []
+      @decomposer.decompose(data) do |decomposed_data|
+        decomposed << {
+          :uri    => decomposed_data.uri.to_s,
+          :body   => decomposed_data.body,
+          :source => decomposed_data.source.uri.to_s,
+        }
+      end
+      decomposed
+    end
+
+    test("multiple") do
+      data_path = Pathname.new(fixture_path("hello.zip"))
+      base_path = data_path.sub_ext("")
+      assert_equal([
+                     {
+                       :uri    => "file:#{base_path}/hello.txt",
+                       :body   => "Hello!\n",
+                       :source => "file:#{data_path}",
+                     },
+                     {
+                       :uri    => "file:#{base_path}/hello.csv",
+                       :body   => "Hello,World\n",
+                       :source => "file:#{data_path}",
+                     },
+                     {
+                       :uri    => "file:#{base_path}/hello/world.txt",
+                       :body   => "World!\n",
+                       :source => "file:#{data_path}",
+                     },
+                   ],
+                   decompose(data_path))
+    end
+
+    sub_test_case("encrypted") do
+      test("without password") do
+        data_path = Pathname.new(fixture_path("password.zip"))
+        data = ChupaText::InputData.new(data_path)
+        assert_raise(ChupaText::EncryptedError.new(data)) do
+          @decomposer.decompose(data)
+        end
+      end
+
+      test("with password") do
+        omit("password is 'password'")
+        data_path = Pathname.new(fixture_path("password.zip"))
+        base_path = data_path.sub_ext("")
+        assert_equal([
+                       {
+                         :uri    => "file:#{base_path}/hello.txt",
+                         :body   => "Hello!\n",
+                         :source => "file:#{data_path}",
+                       },
+                       {
+                         :uri    => "file:#{base_path}/hello.csv",
+                         :body   => "Hello,World\n",
+                         :source => "file:#{data_path}",
+                       },
+                       {
+                         :uri    => "file:#{base_path}/hello/world.txt",
+                         :body   => "World!\n",
+                         :source => "file:#{data_path}",
+                       },
+                     ],
+                     decompose(data_path))
+      end
+    end
+  end
+end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.0.9
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-10 00:00:00.000000000 Z
+date: 2017-07-11 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: archive-zip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -118,6 +132,7 @@ files:
 - lib/chupa-text/decomposers/gzip.rb
 - lib/chupa-text/decomposers/tar.rb
 - lib/chupa-text/decomposers/xml.rb
+- lib/chupa-text/decomposers/zip.rb
 - lib/chupa-text/default-logger.rb
 - lib/chupa-text/error.rb
 - lib/chupa-text/external-command.rb
@@ -143,6 +158,7 @@ files:
 - test/decomposers/test-gzip.rb
 - test/decomposers/test-tar.rb
 - test/decomposers/test-xml.rb
+- test/decomposers/test-zip.rb
 - test/fixture/command/chupa-text/hello.txt
 - test/fixture/command/chupa-text/hello.txt.gz
 - test/fixture/command/chupa-text/no-decomposer.conf
@@ -153,6 +169,8 @@ files:
 - test/fixture/gzip/hello.txt.gz
 - test/fixture/tar/directory.tar
 - test/fixture/tar/top-level.tar
+- test/fixture/zip/hello.zip
+- test/fixture/zip/password.zip
 - test/helper.rb
 - test/run-test.rb
 - test/test-attributes.rb