chupa-text 1.2.1 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99a53085d7eca8e459b0944e44cdb415036d19b6dc5a366e10bc3921a8c8cb09
-  data.tar.gz: ce808a0a9e9352e2b4cf92c9e7348b57e522ee1717dc8070631451aa6a3b9a29
+  metadata.gz: f6d2f05926206d3da67e157161c9e4c35f036af80e80e5177fbc4edf9006e039
+  data.tar.gz: 5ed29a55f62d7a44cbbdbfc690b39005d35b354c2ba65629436b90d1ef32cef1
 SHA512:
-  metadata.gz: 334356c90df57f22ae1fe0871e93147470c7505782487f71613f3fa794ea7fe0ed03b16980892beb46d662f830b876b6b4ff5132b8677892be1246ea26e38f40
-  data.tar.gz: 86de4013d3f07d4d89c2113d01d78d4e6685b95af28ab1ad72267908d3c4b9ea30720eec515380989b6fb2a50d913041ee8223197cbcfa00c51b61b97506d1d7
+  metadata.gz: cd5e2b4b04d2572bb90ec832b618bc855e5c038d0a46f84a970f8e1ae1011609356387c30a71c6d476362fb27391094a25d0759d62bd56772a630914f543e644
+  data.tar.gz: c1c22bb010320fe5f48eb0c8b500e54b4557ca0f24461b832c3ae278c8dd9b0c6dc93f5682cf9b52d15b05673cd013c5546543e2f46e095b72fe46d64deb0aba

data/doc/text/news.md

@@ -1,5 +1,36 @@
 # News
 
+## 1.2.2: 2019-03-28
+
+### Improvements
+
+  * Added `http-server` decomposer.
+
+  * `ChupaText::Data#max_body_size`: Added.
+
+  * `ChupaText::Data#max_body_size=`: Added.
+
+  * `ChupaText::Data#timeout`: Added.
+
+  * `ChupaText::Data#timeout=`: Added.
+
+  * `ChupaText::Data#limit_cpu`: Added.
+
+  * `ChupaText::Data#limit_cpu=`: Added.
+
+  * `ChupaText::Data#limit_ax`: Added.
+
+  * `ChupaText::Data#limit_ax=`: Added.
+
+  * `ChupaText::ExternalCommand`: Added support for soft timeout and limits.
+
+  * `ChupaText::Extractor`: Stopped receiving the max body size as an
+    option. Use `ChupaText::Data#max_body_size=` instead.
+
+### Fixes
+
+  * Fixed decomposer choose logic.
+
 ## 1.2.1: 2019-03-04
 
 ### Improvements

data/lib/chupa-text/command/chupa-text.rb

@@ -196,7 +196,7 @@ module ChupaText
       end
 
       def create_extractor
-        extractor = Extractor.new(max_body_size: @max_body_size)
+        extractor = Extractor.new
         extractor.apply_configuration(@configuration)
         extractor
       end
@@ -222,6 +222,7 @@ module ChupaText
         data.mime_type = @mime_type if @mime_type
         data.need_screenshot = @need_screenshot
         data.expected_screenshot_size = @expected_screenshot_size
+        data.max_body_size = @max_body_size
         data
       end
 

data/lib/chupa-text/data.rb

@@ -65,6 +65,20 @@ module ChupaText
     # @return [Array<Integer, Integer>] the expected screenshot size.
     attr_accessor :expected_screenshot_size
 
+    # @return [Integer, nil] the max body size in bytes.
+    attr_accessor :max_body_size
+
+    # @return [Numeric, String, nil] the timeout on extraction.
+    attr_accessor :timeout
+
+    # @return [Numeric, String, nil] the max CPU time on extraction by
+    #   external command.
+    attr_accessor :limit_cpu
+
+    # @return [Numeric, String, nil] the max memory on extraction by
+    #   external command.
+    attr_accessor :limit_as
+
     def initialize(options={})
       @uri = nil
       @body = nil
@@ -76,6 +90,10 @@ module ChupaText
       @screenshot = nil
       @need_screenshot = true
       @expected_screenshot_size = [200, 200]
+      @max_body_size = nil
+      @timeout = nil
+      @limit_cpu = nil
+      @limit_as = nil
       @options = options || {}
       source_data = @options[:source_data]
       if source_data
@@ -107,6 +125,10 @@ module ChupaText
       end
       self.need_screenshot = data.need_screenshot?
       self.expected_screenshot_size = data.expected_screenshot_size
+      self.max_body_size = data.max_body_size
+      self.timeout = data.timeout
+      self.limit_cpu = data.limit_cpu
+      self.limit_as = data.limit_as
     end
 
     # @param [String, URI, nil] uri The URI for the data. If `uri` is
@@ -198,11 +220,11 @@ module ChupaText
       @need_screenshot
     end
 
-    def to_utf8_body_data(max_body_size: nil)
+    def to_utf8_body_data
       b = nil
-      if max_body_size
+      if @max_body_size
         open do |input|
-          b = input.read(max_body_size)
+          b = input.read(@max_body_size)
         end
       else
         b = body
@@ -211,7 +233,7 @@ module ChupaText
 
       converter = UTF8Converter.new(b)
       utf8_body = converter.convert
-      if max_body_size.nil? and b.equal?(utf8_body)
+      if @max_body_size.nil? and b.equal?(utf8_body)
         self
       else
         TextData.new(utf8_body, source_data: self)

data/lib/chupa-text/decomposers/http-server.rb

@@ -0,0 +1,160 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "net/http"
+require "pp"
+require "uri"
+
+module ChupaText
+  module Decomposers
+    class HTTPServer < Decomposer
+      include Loggable
+
+      registry.register("http-server", self)
+
+      @@default_url = nil
+      class << self
+        def default_url
+          @@default_url
+        end
+
+        def default_url=(url)
+          @@default_url = url
+        end
+      end
+
+      def initialize(options)
+        super
+        @url = @options[:url] ||
+               self.class.default_url ||
+               ENV["CHUPA_TEXT_HTTP_SERVER_URL"]
+        @url = URI(@url) if @url
+      end
+
+      def target?(data)
+        return false unless @url
+        return false if data.text_plain?
+        true
+      end
+
+      def target_score(data)
+        if target?(data)
+          100
+        else
+          nil
+        end
+      end
+
+      def decompose(data, &block)
+        http = Net::HTTP.new(@url.host, @url.port)
+        http.use_ssl = true if @url.is_a?(URI::HTTPS)
+        if data.timeout.is_a?(Numeric)
+          http.open_timeout = data.timeout * 1.5
+          http.read_timeout = data.timeout * 1.5
+          http.write_timeout = data.timeout * 1.5
+        end
+        begin
+          http.start do
+            process_request(http, data, &block)
+          end
+        rescue SystemCallError => error
+          error do
+            message = "#{log_tag}[connection] "
+            message << "Failed to process data in server: "
+            message << "#{@url}: "
+            message << "#{error.class}: #{error.message}\n"
+            message << error.backtrace.join("\n")
+            message
+          end
+        rescue Net::ReadTimeout => error
+          error do
+            message = "#{log_tag}[timeout] "
+            message << "Failed to process data in server: "
+            message << "#{@url}: "
+            message << "#{error.class}: #{error.message}\n"
+            message << error.backtrace.join("\n")
+            message
+          end
+        end
+      end
+
+      private
+      def process_request(http, data)
+        request = Net::HTTP::Post.new(@url)
+        request["transfer-encoding"] = "chunked"
+        data.open do |input|
+          request.set_form(build_parameters(data, input),
+                           "multipart/form-data")
+          response = http.request(request)
+          case response
+          when Net::HTTPOK
+            extracted = JSON.parse(response.body)
+            (extracted["texts"] || []).each do |text|
+              text_data = TextData.new(text["body"], source_data: data)
+              text.each do |key, value|
+                next if key == "body"
+                text_data[key] = value
+              end
+              yield(text_data)
+            end
+          else
+            error do
+              message = "#{log_tag} Failed to process data in server: "
+              message << "#{@url}: "
+              message << "#{response.code}: #{response.message.strip}\n"
+              case response.content_type
+              when "application/json"
+                PP.pp(JSON.parse(response.body), message)
+              else
+                message << response.body
+              end
+              message
+            end
+          end
+        end
+      end
+
+      def build_parameters(data, input)
+        parameters = []
+        [
+          ["timeout",
+           data.timeout || ChupaText::ExternalCommand.default_timeout],
+          ["limit_cpu",
+           data.limit_cpu || ChupaText::ExternalCommand.default_limit_cpu],
+          ["limit_as",
+           data.limit_as || ChupaText::ExternalCommand.default_limit_as],
+          ["max_body_size", data.max_body_size],
+        ].each do |key, value|
+          next if value.nil?
+          parameters << [key, StringIO.new(value.to_s)]
+        end
+        parameters << [
+          "data",
+          input,
+          {
+            filename: data.path.to_s,
+            content_type: data.mime_type,
+          },
+        ]
+        parameters
+      end
+
+      def log_tag
+        "[decomposer][http-server]"
+      end
+    end
+  end
+end

data/lib/chupa-text/external-command.rb

@@ -69,13 +69,19 @@ module ChupaText
       else
         options = {}
       end
+      data = options[:data]
       pid = spawn(options[:env] || {},
                   @path.to_s,
                   *arguments,
-                  spawn_options(options[:spawn_options]))
+                  spawn_options(options[:spawn_options], data))
+      if data
+        soft_timeout = data.timeout
+      else
+        soft_timeout = nil
+      end
       status = nil
       begin
-        status = wait_process(pid, options[:timeout])
+        status = wait_process(pid, options[:timeout], soft_timeout)
       ensure
         unless status
           begin
@@ -99,28 +105,44 @@ module ChupaText
     end
 
     private
-    def spawn_options(user_options)
+    def spawn_options(user_options, data)
       options = (user_options || {}).dup
-      apply_default_spawn_limit(options, :cpu, :int)
-      apply_default_spawn_limit(options, :as, :size)
+      if data
+        soft_limit_cpu = data.limit_cpu
+        soft_limit_as = data.limit_as
+      else
+        soft_limit_cpu = nil
+        soft_limit_as = nil
+      end
+      apply_default_spawn_limit(options, soft_limit_cpu, :cpu, :time)
+      apply_default_spawn_limit(options, soft_limit_as, :as, :size)
       options
     end
 
-    def apply_default_spawn_limit(options, key, type)
+    def apply_default_spawn_limit(options, soft_value, key, type)
       # TODO: Workaround for Ruby 2.3.3p222
       case key
       when :cpu
         option_key = :rlimit_cpu
+        unit = "s"
       when :as
         option_key = :rlimit_as
+        unit = ""
       else
         option_key = :"rlimit_#{key}"
+        unit = ""
       end
       return if options[option_key]
 
       tag = "[limit][#{key}]"
       value = self.class.__send__("default_limit_#{key}")
       value = __send__("parse_#{type}", tag, value)
+      soft_value = __send__("parse_#{type}", tag, soft_value)
+      if value
+        value = soft_value if soft_value and soft_value < value
+      else
+        value = soft_value
+      end
       return if value.nil?
       rlimit_number = Process.const_get("RLIMIT_#{key.to_s.upcase}")
       soft_limit, hard_limit = Process.getrlimit(rlimit_number)
@@ -129,7 +151,7 @@ module ChupaText
         return nil
       end
       limit_info = "soft-limit:#{soft_limit}, hard-limit:#{hard_limit}"
-      info("#{log_tag}#{tag}[set] <#{value}>(#{limit_info})")
+      info("#{log_tag}#{tag}[set] <#{value}#{unit}>(#{limit_info})")
 
       options[option_key] = value
     end
@@ -169,12 +191,21 @@ module ChupaText
         scale = 1
         case value
         when /GB?\z/i
+          scale = 1000 ** 3
+          number = $PREMATCH
+        when /GiB?\z/i
           scale = 1024 ** 3
           number = $PREMATCH
         when /MB?\z/i
+          scale = 1000 ** 2
+          number = $PREMATCH
+        when /MiB?\z/i
           scale = 1024 ** 2
           number = $PREMATCH
-        when /KB?\z/i
+        when /[kK]B?\z/i
+          scale = 1000 ** 1
+          number = $PREMATCH
+        when /KiB?\z/i
           scale = 1024 ** 1
           number = $PREMATCH
         when /B?\z/i
@@ -227,9 +258,15 @@ module ChupaText
       warn("#{log_tag}#{tag}[invalid] <#{value}>(#{type})")
     end
 
-    def wait_process(pid, timeout)
+    def wait_process(pid, timeout, soft_timeout)
       tag = "[timeout]"
       timeout = parse_time(tag, timeout || self.class.default_timeout)
+      soft_timeout = parse_time(tag, soft_timeout)
+      if timeout
+        timeout = soft_timeout if soft_timeout and soft_timeout < timeout
+      else
+        timeout = soft_timeout
+      end
       if timeout
         info("#{log_tag}#{tag}[use] <#{timeout}s>: <#{pid}>")
         status = wait_process_timeout(pid, timeout)

data/lib/chupa-text/extractor.rb

@@ -21,9 +21,8 @@ module ChupaText
   class Extractor
     include Loggable
 
-    def initialize(max_body_size: nil)
+    def initialize
       @decomposers = []
-      @max_body_size = max_body_size
     end
 
     # Sets the extractor up by the configuration. It adds decomposers
@@ -79,7 +78,7 @@ module ChupaText
         candidates << [score, decomposer]
       end
       return nil if candidates.empty?
-      candidate = candidates.sort_by {|score, _| score}.first
+      candidate = candidates.sort_by {|score, _| -score}.first
       candidate[1]
     end
 
@@ -91,11 +90,11 @@ module ChupaText
       if decomposer.nil?
         if target.text_plain?
           debug {"#{log_tag}[extract][text-plain]"}
-          yield(target.to_utf8_body_data(max_body_size: @max_body_size))
+          yield(target.to_utf8_body_data)
         else
           debug {"#{log_tag}[extract][decomposer] not found"}
           if target.text?
-            yield(target.to_utf8_body_data(max_body_size: @max_body_size))
+            yield(target.to_utf8_body_data)
           end
         end
       else

data/lib/chupa-text/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module ChupaText
-  VERSION = "1.2.1"
+  VERSION = "1.2.2"
 end

data/test/decomposers/test-csv.rb

@@ -14,7 +14,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-class TestDecomposersCSV< Test::Unit::TestCase
+class TestDecomposersCSV < Test::Unit::TestCase
   include Helper
 
   def setup

data/test/decomposers/test-http-server.rb

@@ -0,0 +1,175 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestDecomposersHTTPServer < Test::Unit::TestCase
+  include Helper
+
+  def setup
+    ChupaText::Decomposers::HTTPServer.default_url = nil
+    setup_server
+    setup_data
+    setup_decomposer
+  end
+
+  def setup_server
+    @port = 40080
+    @path = "/extraction.json"
+    @server_url = "http://127.0.0.1:#{@port}#{@path}"
+    logger = WEBrick::Log.new
+    logger.level = logger.class::ERROR
+    @server = WEBrick::HTTPServer.new(Port: @port,
+                                      Logger: logger,
+                                      AccessLog: [])
+    @response_status = 200
+    @server.mount_proc(@path) do |request, response|
+      sleep(@timeout * 2) if @timeout
+      response.status = @response_status
+      response.content_type = "application/json"
+      response.body = JSON.generate(@response)
+    end
+    @server_thread = Thread.new do
+      @server.start
+    end
+  end
+
+  def setup_data
+    @input_data = <<-CSV
+Hello,World
+Ruby,ChupaText
+    CSV
+    @input_mime_type = "text/csv"
+    @input_path = "/tmp/hello.csv"
+    @timeout = nil
+    @extracted_text = @input_data.gsub(/,/, "\t")
+    @extracted_path = @input_path.gsub(/\.csv\z/, ".txt")
+    @response = {
+      "mime-type" => @input_mime_type,
+      "uri" => "file://#{@input_path}",
+      "path" => @input_path,
+      "size" => @input_data.bytesize,
+      "texts" => [
+        {
+          "mime-type" => "text/plain",
+          "uri" => "file://#{@extracted_path}",
+          "path" => @extracted_path,
+          "size" => @extracted_text.bytesize,
+          "source-mime-types" => [
+            @input_mime_type,
+          ],
+          "body" => @extracted_text,
+        },
+      ],
+    }
+  end
+
+  def setup_decomposer
+    @decomposer = ChupaText::Decomposers::HTTPServer.new(:url => @server_url)
+  end
+
+  def teardown
+    teardown_server
+  end
+
+  def teardown_server
+    @server.shutdown
+    @server_thread.join
+  end
+
+  sub_test_case("decompose") do
+    def test_success
+      assert_equal([@extracted_text],
+                   decompose.collect(&:body))
+    end
+
+    def test_not_ok
+      @response_status = 404
+      messages = capture_log do
+        assert_equal([], decompose.collect(&:body))
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][http-server] " +
+                       "Failed to process data in server: " +
+                       "#{@server_url}: " +
+                       "#{@response_status}: Not Found",
+                     ],
+                   ],
+                   messages)
+    end
+
+    def test_no_server
+      no_server_url = "http://127.0.0.1:2929/extraction.json"
+      @decomposer = ChupaText::Decomposers::HTTPServer.new(:url => no_server_url)
+      messages = capture_log do
+        assert_equal([], decompose.collect(&:body))
+      end
+      messages = messages.collect do |level, message|
+        [level, message.gsub(/Errno::.*\z/, "")]
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][http-server][connection] " +
+                       "Failed to process data in server: " +
+                       "#{no_server_url}: ",
+                     ],
+                   ],
+                   messages)
+    end
+
+    def test_read_timeout
+      @timeout = 0.1
+      messages = capture_log do
+        assert_equal([], decompose.collect(&:body))
+      end
+      messages = messages.collect do |level, message|
+        [level, message.gsub(/Net::.*\z/, "")]
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][http-server][timeout] " +
+                       "Failed to process data in server: " +
+                       "#{@server_url}: ",
+                     ],
+                   ],
+                   messages)
+    end
+
+    def test_default_url
+      ChupaText::Decomposers::HTTPServer.default_url = @server_url
+      @decomposer = ChupaText::Decomposers::HTTPServer.new({})
+      assert_equal([@extracted_text],
+                   decompose.collect(&:body))
+    end
+
+    private
+    def decompose
+      data = ChupaText::Data.new
+      data.path = @input_path
+      data.mime_type = @input_mime_type
+      data.body = @input_data
+      data.timeout = @timeout
+
+      decomposed = []
+      @decomposer.decompose(data) do |decomposed_data|
+        decomposed << decomposed_data
+      end
+      decomposed
+    end
+  end
+end

data/test/helper.rb

@@ -15,8 +15,10 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "pathname"
+require "rbconfig"
 require "tempfile"
 require "uri"
+require "webrick"
 
 module Helper
   def fixture_path(*components)
@@ -39,4 +41,8 @@ module Helper
       [level, message]
     end
   end
+
+  def ruby
+    RbConfig.ruby
+  end
 end

data/test/test-external-command.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,12 +14,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "rbconfig"
-
 class TestExternalCommand < Test::Unit::TestCase
-  def ruby
-    RbConfig.ruby
-  end
+  include Helper
 
   def create_command(command)
     ChupaText::ExternalCommand.new(command)
@@ -76,4 +72,321 @@ class TestExternalCommand < Test::Unit::TestCase
       assert_false(exist?("nonexistent"))
     end
   end
+
+  class TestTimeout < self
+    def setup
+      @data = ChupaText::TextData.new("Hello")
+      timeout = ChupaText::ExternalCommand.default_timeout
+      begin
+        yield
+      ensure
+        ChupaText::ExternalCommand.default_timeout = timeout
+      end
+    end
+
+    def run_command(options={})
+      IO.pipe do |input, output|
+        command = create_command(ruby)
+        command.run("-e", "puts(Process.pid)",
+                    options.merge(data: @data,
+                                  spawn_options: {out: output}))
+        input.gets.chomp
+      end
+    end
+
+    def test_option
+      pid = nil
+      messages = capture_log do
+        pid = run_command(timeout: "60s")
+      end
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][timeout][use] <60.0s>: <#{pid}>",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_data_not_use
+      @data.timeout = "90s"
+      pid = nil
+      messages = capture_log do
+        pid = run_command(timeout: "60s")
+      end
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][timeout][use] <60.0s>: <#{pid}>",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_data_use
+      @data.timeout = "30s"
+      pid = nil
+      messages = capture_log do
+        pid = run_command(timeout: "60s")
+      end
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][timeout][use] <30.0s>: <#{pid}>",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_data_only
+      @data.timeout = "30s"
+      pid = nil
+      messages = capture_log do
+        pid = run_command
+      end
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][timeout][use] <30.0s>: <#{pid}>",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default
+      ChupaText::ExternalCommand.default_timeout = "60s"
+      pid = nil
+      messages = capture_log do
+        pid = run_command
+      end
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][timeout][use] <60.0s>: <#{pid}>",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_data_not_use
+      ChupaText::ExternalCommand.default_timeout = "60s"
+      @data.timeout = "90s"
+      pid = nil
+      messages = capture_log do
+        pid = run_command
+      end
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][timeout][use] <60.0s>: <#{pid}>",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_data_use
+      ChupaText::ExternalCommand.default_timeout = "60s"
+      @data.timeout = "30s"
+      pid = nil
+      messages = capture_log do
+        pid = run_command
+      end
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][timeout][use] <30.0s>: <#{pid}>",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_data_only
+      @data.timeout = "30s"
+      pid = nil
+      messages = capture_log do
+        pid = run_command
+      end
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][timeout][use] <30.0s>: <#{pid}>",
+                     ]
+                   ],
+                   messages)
+    end
+  end
+
+  class TestLimitCPU < self
+    def setup
+      @data = ChupaText::TextData.new("Hello")
+      limit_cpu = ChupaText::ExternalCommand.default_limit_cpu
+      begin
+        yield
+      ensure
+        ChupaText::ExternalCommand.default_limit_cpu = limit_cpu
+      end
+    end
+
+    def run_command(spawn_options={})
+      command = create_command(ruby)
+      command.run("-e", "true",
+                  data: @data,
+                  spawn_options: spawn_options)
+    end
+
+    def test_default
+      ChupaText::ExternalCommand.default_limit_cpu = "60s"
+      messages = capture_log do
+        run_command
+      end
+      soft_limit, hard_limit = Process.getrlimit(Process::RLIMIT_CPU)
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][limit][cpu][set] <60.0s>" +
+                       "(soft-limit:#{soft_limit}, hard-limit:#{hard_limit})",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_data_not_use
+      ChupaText::ExternalCommand.default_limit_cpu = "60s"
+      @data.limit_cpu = "90s"
+      messages = capture_log do
+        run_command
+      end
+      soft_limit, hard_limit = Process.getrlimit(Process::RLIMIT_CPU)
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][limit][cpu][set] <60.0s>" +
+                       "(soft-limit:#{soft_limit}, hard-limit:#{hard_limit})",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_data_use
+      ChupaText::ExternalCommand.default_limit_cpu = "60s"
+      @data.limit_cpu = "30s"
+      messages = capture_log do
+        run_command
+      end
+      soft_limit, hard_limit = Process.getrlimit(Process::RLIMIT_CPU)
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][limit][cpu][set] <30.0s>" +
+                       "(soft-limit:#{soft_limit}, hard-limit:#{hard_limit})",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_data_only
+      @data.limit_cpu = "30s"
+      messages = capture_log do
+        run_command
+      end
+      soft_limit, hard_limit = Process.getrlimit(Process::RLIMIT_CPU)
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][limit][cpu][set] <30.0s>" +
+                       "(soft-limit:#{soft_limit}, hard-limit:#{hard_limit})",
+                     ]
+                   ],
+                   messages)
+    end
+  end
+
+  class TestLimitAS < self
+    def setup
+      @data = ChupaText::TextData.new("Hello")
+      limit_as = ChupaText::ExternalCommand.default_limit_as
+      begin
+        yield
+      ensure
+        ChupaText::ExternalCommand.default_limit_as = limit_as
+      end
+    end
+
+    def run_command(spawn_options={})
+      command = create_command(ruby)
+      command.run("-e", "true",
+                  data: @data,
+                  spawn_options: spawn_options)
+    end
+
+    def test_default
+      ChupaText::ExternalCommand.default_limit_as = "100MiB"
+      messages = capture_log do
+        run_command
+      end
+      soft_limit, hard_limit = Process.getrlimit(Process::RLIMIT_AS)
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][limit][as][set] " +
+                       "<#{100 * 1024 * 1024}>" +
+                       "(soft-limit:#{soft_limit}, hard-limit:#{hard_limit})",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_data_not_use
+      ChupaText::ExternalCommand.default_limit_as = "100MiB"
+      @data.limit_as = "150MiB"
+      messages = capture_log do
+        run_command
+      end
+      soft_limit, hard_limit = Process.getrlimit(Process::RLIMIT_AS)
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][limit][as][set] " +
+                       "<#{100 * 1024 * 1024}>" +
+                       "(soft-limit:#{soft_limit}, hard-limit:#{hard_limit})",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_soft_use
+      ChupaText::ExternalCommand.default_limit_as = "100MiB"
+      @data.limit_as = "50MiB"
+      messages = capture_log do
+        run_command
+      end
+      soft_limit, hard_limit = Process.getrlimit(Process::RLIMIT_AS)
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][limit][as][set] " +
+                       "<#{50 * 1024 * 1024}>" +
+                       "(soft-limit:#{soft_limit}, hard-limit:#{hard_limit})",
+                     ]
+                   ],
+                   messages)
+    end
+
+    def test_default_soft_only
+      @data.limit_as = "50MiB"
+      messages = capture_log do
+        run_command
+      end
+      soft_limit, hard_limit = Process.getrlimit(Process::RLIMIT_AS)
+      assert_equal([
+                     [
+                       :info,
+                       "[external-command][limit][as][set] " +
+                       "<#{50 * 1024 * 1024}>" +
+                       "(soft-limit:#{soft_limit}, hard-limit:#{hard_limit})",
+                     ]
+                   ],
+                   messages)
+    end
+  end
 end

data/test/test-extractor.rb

@@ -231,10 +231,11 @@ class TestExtractor < Test::Unit::TestCase
 
     sub_test_case("max body size") do
       def test_last_invalid
-        @extractor = ChupaText::Extractor.new(max_body_size: 5)
+        @extractor = ChupaText::Extractor.new
         data = ChupaText::Data.new
         data.mime_type = "text/plain"
         data.body = "こん"
+        data.max_body_size = 5
         assert_equal(["こ"], extract(data))
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-03 00:00:00.000000000 Z
+date: 2019-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: archive-zip
@@ -159,6 +159,7 @@ files:
 - lib/chupa-text/decomposers.rb
 - lib/chupa-text/decomposers/csv.rb
 - lib/chupa-text/decomposers/gzip.rb
+- lib/chupa-text/decomposers/http-server.rb
 - lib/chupa-text/decomposers/office-open-xml-document.rb
 - lib/chupa-text/decomposers/office-open-xml-presentation.rb
 - lib/chupa-text/decomposers/office-open-xml-workbook.rb
@@ -198,6 +199,7 @@ files:
 - test/command/test-chupa-text.rb
 - test/decomposers/test-csv.rb
 - test/decomposers/test-gzip.rb
+- test/decomposers/test-http-server.rb
 - test/decomposers/test-office-open-xml-document.rb
 - test/decomposers/test-office-open-xml-presentation.rb
 - test/decomposers/test-office-open-xml-workbook.rb