dagger 1.7.1 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a94245616c7abe732448eb4b06a2a04182e217c1e9123a2591b7d8b770804b0d
-  data.tar.gz: '08b40cfc31554215b9f1cf38e8d4e3e820c5711a566941cc2a3ae31319cfe63a'
+  metadata.gz: a3be984806667b1144778e3695d47171000733ad0c2ecbc1323b36c8f97d666e
+  data.tar.gz: e27de03a5b62085bb6de4954e6be2a15f14bc97a7c6933dfa92467fdaec33ed5
 SHA512:
-  metadata.gz: f0cc209b26fbcae81e42f3e304459db4a7c17bcf344f461f3f4949bbef6dd99585b5c307dabb17f2e11c74ad0e945d3793204cfd566f04f3f270e10c917f6ffc
-  data.tar.gz: 0175b3cde3920b06273bce7041adc7628deac7dbbb120970cbcd4e86b61b9399c705f76694add8a07e9fec5c0cebfa11bd4fd21c4602d139a1b724cd4495537d
+  metadata.gz: 5237f119527a6a77f1075ab8b65048947e68c23d20fd523f1c220ebf5216997660f3b2acac75560082e956297087f7ba9f090ca118e973fbbb66c941d4ef393c
+  data.tar.gz: 9a9a509cb9ff96338fa73fd711532704c27267916c2dda6e2e05c35446eeba4ec449f6aeba5bea2d4387ecac73ea69f18cceed325e9dbac263bd4d5811b8bc84

data/dagger.gemspec

@@ -19,7 +19,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rspec-mocks"
   s.add_development_dependency "rspec-expectations"
 
-  s.add_runtime_dependency "net-http-persistent", "~> 3.0"
   s.add_runtime_dependency "oj", "~> 2.1"
   s.add_runtime_dependency "ox", "~> 2.4"
 

data/lib/dagger.rb

@@ -1,8 +1,7 @@
 require 'dagger/version'
 require 'dagger/response'
 require 'dagger/parsers'
-
-require 'net/http/persistent'
+require 'dagger/connection_manager'
 require 'net/https'
 require 'base64'
 
@@ -19,7 +18,13 @@ module Dagger
   DEFAULT_RETRY_WAIT = 5.freeze # seconds
   DEFAULT_HEADERS = {
     'Accept'     => '*/*',
-    'User-Agent' => "#{DAGGER_NAME} (Ruby Net::HTTP Wrapper, like curl)"
+    'User-Agent' => "#{DAGGER_NAME} (Ruby Net::HTTP wrapper, like curl)"
+  }
+
+  DEFAULTS = {
+    open_timeout: 10,
+    read_timeout: 10,
+    keep_alive_timeout: 10
   }
 
   module Utils
@@ -34,6 +39,7 @@ module Dagger
     end
 
     def self.resolve_uri(uri, host = nil, query = nil)
+      uri = host + uri if uri[0] == '/' && host
       uri = parse_uri(uri.to_s)
       uri.path.sub!(/\?.*|$/, '?' + Utils.encode(query)) if query and query.any?
       uri
@@ -61,25 +67,40 @@ module Dagger
 
   class Client
 
-    def self.init(uri, opts)
-      uri  = Utils.parse_uri(uri)
-      http = if opts.delete(:persistent)
-        pool_size = opts[:pool_size] || Net::HTTP::Persistent::DEFAULT_POOL_SIZE
-        Net::HTTP::Persistent.new(name: DAGGER_NAME, pool_size: pool_size)
-      else
-        Net::HTTP.new(opts[:ip] || uri.host, uri.port)
+    def self.init_persistent(opts = {})
+      # this line below forces one connection manager between multiple threads
+      # @persistent ||= Dagger::ConnectionManager.new(opts)
+
+      # here we initialize a connection manager for each thread
+      Thread.current[:dagger_persistent] ||= begin
+        Dagger::ConnectionManager.new(opts)
       end
+    end
+
+    def self.init_connection(uri, opts = {})
+      http = Net::HTTP.new(opts[:ip] || uri.host, uri.port)
 
       if uri.port == 443
         http.use_ssl = true if http.respond_to?(:use_ssl=) # persistent does it automatically
         http.verify_mode = opts[:verify_ssl] === false ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
       end
 
-      [:open_timeout, :read_timeout, :ssl_version, :ciphers].each do |key|
-        http.send("#{key}=", opts[key]) if opts.has_key?(key)
+      [:keep_alive_timeout, :open_timeout, :read_timeout, :ssl_version, :ciphers].each do |key|
+        http.send("#{key}=", opts[key] || DEFAULTS[key]) if (opts.has_key?(key) || DEFAULTS.has_key?(key))
+      end
+
+      http
+    end
+
+    def self.init(uri, opts)
+      uri  = Utils.parse_uri(uri)
+
+      http = if opts.delete(:persistent)
+        init_persistent(opts)
+      else
+        init_connection(uri, opts)
       end
 
-      # new(http, [uri.scheme, uri.host].join('://'))
       new(http, uri.scheme_and_host)
     end
 
@@ -89,8 +110,9 @@ module Dagger
 
     def get(uri, opts = {})
       uri = Utils.resolve_uri(uri, @host, opts[:query])
+
       if @host != uri.scheme_and_host
-        raise ArgumentError.new("#{uri.scheme_and_host} doesn't match #{@host}")
+        raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}")
       end
 
       opts[:follow] = 10 if opts[:follow] == true
@@ -109,7 +131,7 @@ module Dagger
         @http.start unless @http.started?
         resp, data = @http.request(request)
       else # persistent
-        resp, data = @http.request(uri, request)
+        resp, data = @http.send_request(uri, request)
       end
 
       if REDIRECT_CODES.include?(resp.code.to_i) && resp['Location'] && (opts[:follow] && opts[:follow] > 0)
@@ -121,7 +143,8 @@ module Dagger
       @response = build_response(resp, data || resp.body)
 
     rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EINVAL, Timeout::Error, \
-      SocketError, EOFError, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
+      Net::OpenTimeout, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, \
+      SocketError, EOFError, OpenSSL::SSL::SSLError => e
 
       if retries = opts[:retries] and retries.to_i > 0
         debug "Got #{e.class}! Retrying in a sec (#{retries} retries left)"
@@ -156,10 +179,11 @@ module Dagger
 
       uri = Utils.resolve_uri(uri, @host)
       if @host != uri.scheme_and_host
-        raise ArgumentError.new("#{uri.scheme_and_host} doesn't match #{@host}")
+        raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}")
       end
 
       headers = DEFAULT_HEADERS.merge(opts[:headers] || {})
+
       query = if data.is_a?(String)
         data
       elsif opts[:json]
@@ -185,7 +209,7 @@ module Dagger
         req = Kernel.const_get("Net::HTTP::#{method.capitalize}").new(uri.path, headers)
         # req.set_form_data(query)
         req.body = query
-        resp, data = @http.request(uri, req)
+        resp, data = @http.send_request(uri, req)
       end
 
       @response = build_response(resp, data || resp.body)
@@ -208,7 +232,7 @@ module Dagger
     end
 
     def open(&block)
-      if @http.is_a?(Net::HTTP::Persistent)
+      if @http.is_a?(Dagger::ConnectionManager)
         instance_eval(&block)
       else
         @http.start do
@@ -218,7 +242,7 @@ module Dagger
     end
 
     def close
-      if @http.is_a?(Net::HTTP::Persistent)
+      if @http.is_a?(Dagger::ConnectionManager)
         @http.shutdown # calls finish on pool connections
       else
         @http.finish if @http.started?
@@ -276,4 +300,4 @@ module Dagger
 
   end
 
-end
+end

data/lib/dagger/connection_manager.rb

@@ -0,0 +1,59 @@
+module Dagger
+
+  class ConnectionManager
+
+    def initialize(opts = {})
+      @opts = {}
+      @active_connections = {}
+      @mutex = Mutex.new
+    end
+
+    def shutdown
+      @mutex.synchronize do
+        # puts "Shutting down connections: #{@active_connections.count}"
+        @active_connections.each do |_, connection|
+          connection.finish
+        end
+        @active_connections = {}
+      end
+    end
+
+    # Gets a connection for a given URI. This is for internal use only as it's
+    # subject to change (we've moved between HTTP client schemes in the past
+    # and may do it again).
+    #
+    # `uri` is expected to be a string.
+    def connection_for(uri)
+      @mutex.synchronize do
+        connection = @active_connections[[uri.host, uri.port]]
+
+        if connection.nil?
+          connection = Dagger::Client.init_connection(uri, @opts)
+          connection.start
+
+          @active_connections[[uri.host, uri.port]] = connection
+          # puts "#{@active_connections.count} connections"
+        end
+
+        connection
+      end
+    end
+
+    # Executes an HTTP request to the given URI with the given method. Also
+    # allows a request body, headers, and query string to be specified.
+    def send_request(uri, request)
+      connection = connection_for(uri)
+      @mutex.synchronize do
+        begin
+          connection.request(request)
+        rescue StandardError => err
+          err
+        end
+      end.tap do |result|
+        raise(result) if result.is_a?(StandardError)
+      end
+    end
+
+  end
+
+end

data/lib/dagger/ox_extension.rb

@@ -5,12 +5,20 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
   alias_method :to_s, :text
   alias_method :value, :text
 
+  def to_node
+    self
+  end
+
   def count
     raise "Please call #children.count"
   end
 
-  def to_hash
-    self # for backwards compat
+  def keys
+    @keys ||= children.collect(&:name)
+  end
+
+  def is_array?
+    keys.count != keys.uniq.count
   end
 
   # this lets us traverse an parsed object like this:
@@ -20,30 +28,33 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
     found.empty? ? nil : found.size == 1 ? found.first : found
   end
 
+  # returns list of XMLNodes with matching names
   def slice(*arr)
     Array(arr).flatten.map { |key| self[key] }
   end
 
-  def values(arr = nil, include_empty: false)
-    if arr
-      Array(arr).flatten.each_with_object({}) do |key, memo| 
+  def values(keys_arr = nil, include_empty: false)
+    if keys_arr
+      Array(keys_arr).flatten.each_with_object({}) do |key, memo|
         if found = self[key] and (found.to_s || include_empty)
           memo[key] = found.to_s
         end
       end
+    elsif is_array?
+      children.map(&:values)
     else
-      children.each_with_object({}) do |child, memo| 
-        if child.to_s || include_empty
-          memo[child.name] = child.to_s
-        end
+      children.each_with_object({}) do |child, memo|
+        memo[child.name] = child.children.any? ? child.values : child.text
       end
     end
   end
 
+  alias_method :to_hash, :values
+  alias_method :to_h, :values
+
   def dig(*paths)
     list = Array(paths).flatten
     res = list.reduce([self]) do |parents, key|
-
       if parents
         found = parents.map do |parent|
           parent.children.select { |node| node.name.to_s == key.to_s }
@@ -79,14 +90,14 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
 end
 
 class Ox::Document
-  def to_hash
-    nodes.first.to_hash
+  def to_node
+    nodes.first.to_node
   end
 end
 
 class Ox::Element
-  def to_hash
-    children = nodes.map { |n| n.class == self.class ? n.to_hash : nil }.compact
+  def to_node
+    children = nodes.map { |n| n.class == self.class ? n.to_node : nil }.compact
     XMLNode.new(value, text, attributes, children)
   end
 end

data/lib/dagger/parsers.rb

@@ -26,7 +26,7 @@ class Parsers
 
   def text_xml(body)
     if res = Ox.parse(body)
-      res.to_hash
+      res.to_node
     end
   rescue Ox::ParseError
     nil

data/lib/dagger/version.rb

@@ -1,7 +1,7 @@
 module Dagger
   MAJOR = 1
-  MINOR = 7
-  PATCH = 1
+  MINOR = 8
+  PATCH = 0
 
   VERSION = [MAJOR, MINOR, PATCH].join('.')
 end

data/spec/parsers_spec.rb

@@ -14,6 +14,9 @@ describe 'Parsers' do
 
   before do
     allow(Net::HTTP).to receive(:new).and_return(fake_http)
+    allow(fake_http).to receive(:keep_alive_timeout=).and_return(true)
+    allow(fake_http).to receive(:read_timeout=).and_return(true)
+    allow(fake_http).to receive(:open_timeout=).and_return(true)
     allow(fake_http).to receive(:verify_mode=).and_return(true)
     allow(fake_http).to receive(:post).and_return(fake_resp)
   end
@@ -100,7 +103,7 @@ describe 'Parsers' do
       it 'returns XMLNode obj' do
         res = send_request.data
         expect(res).to be_a(XMLNode)
-        expect(res.to_hash).to eql(res)
+        expect(res.to_node).to eql(res)
         expect(res['foo']).to be_a(XMLNode)
         expect(res['foo'].text).to eql('123')
 
@@ -126,7 +129,7 @@ describe 'Parsers' do
 
       it 'works' do
         doc = Ox.parse(xml)
-        obj = doc.to_hash
+        obj = doc.to_node
 
         expect(obj[:nested][:item][:title].text).to eql('foobar')
       end

data/spec/persistent_spec.rb

@@ -17,4 +17,50 @@ describe 'Persistent mode' do
     end
   end
 
+end
+
+describe 'using threads' do
+
+  def get(url)
+    @http.get(url)
+  end
+
+  def connect(host)
+    raise if @http
+    @http = Dagger.open(host)
+  end
+
+  def disconnect
+    raise if @http.nil?
+    @http.close
+    @http = nil
+  end
+
+  it 'works' do
+    thread_count = 10
+    urls_count = 100
+    host = 'https://postman-echo.com'
+    urls = urls_count.times.map { |i| "/get?page/#{i}" }
+    result = []
+
+    mutex = Mutex.new
+    thread_count.times.map do
+      Thread.new(urls, result) do |urls, result|
+        # mutex.synchronize { Dagger.open(host) }
+        http = Dagger.open(host)
+        while url = mutex.synchronize { urls.pop }
+          puts "Fetching #{url}"
+          resp = http.get(url)
+          mutex.synchronize do
+            result.push(resp.code)
+          end
+        end
+        # mutex.synchronize { http.close }
+        http.close
+      end
+    end.each(&:join)
+
+    expect(result.count).to eq(urls_count)
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dagger
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.8.0
 platform: ruby
 authors:
 - Tomás Pollak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-13 00:00:00.000000000 Z
+date: 2020-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: net-http-persistent
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -124,6 +110,7 @@ files:
 - bin/dagger
 - dagger.gemspec
 - lib/dagger.rb
+- lib/dagger/connection_manager.rb
 - lib/dagger/ox_extension.rb
 - lib/dagger/parsers.rb
 - lib/dagger/response.rb