dagger 1.5.1 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a12a3d9db4308744812542f0e74b3d21ae3075fa91b2025114c2693ea20440f1
-  data.tar.gz: ab01f2d4accd0dfe778936fd2a6a1075cbd45e335f4e34f82f2d6507ebfcd7d0
+  metadata.gz: a3be984806667b1144778e3695d47171000733ad0c2ecbc1323b36c8f97d666e
+  data.tar.gz: e27de03a5b62085bb6de4954e6be2a15f14bc97a7c6933dfa92467fdaec33ed5
 SHA512:
-  metadata.gz: '0193ebca25917fb8d6be7194a533d6920128213b11a8bf05abb4c093439a5f7c0449b8637af0effcbb139d477a3d0c68bd966567277de4a3f0c670d103a34c75'
-  data.tar.gz: 473e6a93ba6ba26fb70ec0fdf931e7a1a046c3713c95f49a0b1675ccd4aac7311f00025e7856faed052a7cc20d10a3d0d5b0fbb8848fc24c23b0e08656b2e7ce
+  metadata.gz: 5237f119527a6a77f1075ab8b65048947e68c23d20fd523f1c220ebf5216997660f3b2acac75560082e956297087f7ba9f090ca118e973fbbb66c941d4ef393c
+  data.tar.gz: 9a9a509cb9ff96338fa73fd711532704c27267916c2dda6e2e05c35446eeba4ec449f6aeba5bea2d4387ecac73ea69f18cceed325e9dbac263bd4d5811b8bc84

data/dagger.gemspec

@@ -19,7 +19,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rspec-mocks"
   s.add_development_dependency "rspec-expectations"
 
-  s.add_runtime_dependency "net-http-persistent", "~> 3.0"
   s.add_runtime_dependency "oj", "~> 2.1"
   s.add_runtime_dependency "ox", "~> 2.4"
 

data/lib/dagger.rb

@@ -1,8 +1,7 @@
 require 'dagger/version'
 require 'dagger/response'
 require 'dagger/parsers'
-
-require 'net/http/persistent'
+require 'dagger/connection_manager'
 require 'net/https'
 require 'base64'
 
@@ -19,7 +18,13 @@ module Dagger
   DEFAULT_RETRY_WAIT = 5.freeze # seconds
   DEFAULT_HEADERS = {
     'Accept'     => '*/*',
-    'User-Agent' => "#{DAGGER_NAME} (Ruby Net::HTTP Wrapper, like curl)"
+    'User-Agent' => "#{DAGGER_NAME} (Ruby Net::HTTP wrapper, like curl)"
+  }
+
+  DEFAULTS = {
+    open_timeout: 10,
+    read_timeout: 10,
+    keep_alive_timeout: 10
   }
 
   module Utils
@@ -34,7 +39,7 @@ module Dagger
     end
 
     def self.resolve_uri(uri, host = nil, query = nil)
-      uri = host + uri if uri.to_s['//'].nil? && host
+      uri = host + uri if uri[0] == '/' && host
       uri = parse_uri(uri.to_s)
       uri.path.sub!(/\?.*|$/, '?' + Utils.encode(query)) if query and query.any?
       uri
@@ -50,7 +55,7 @@ module Dagger
       when Array then obj.map { |v| encode(v, "#{key}[]") }.join('&')
       when nil   then ''
       else
-        "#{key}=#{URI.escape(obj.to_s)}"
+        "#{key}=#{ERB::Util.url_encode(obj.to_s)}"
       end
     end
 
@@ -62,24 +67,40 @@ module Dagger
 
   class Client
 
-    def self.init(uri, opts)
-      uri  = Utils.parse_uri(uri)
-      http = if opts.delete(:persistent)
-        Net::HTTP::Persistent.new(name: DAGGER_NAME)
-      else
-        Net::HTTP.new(uri.host, uri.port)
+    def self.init_persistent(opts = {})
+      # this line below forces one connection manager between multiple threads
+      # @persistent ||= Dagger::ConnectionManager.new(opts)
+
+      # here we initialize a connection manager for each thread
+      Thread.current[:dagger_persistent] ||= begin
+        Dagger::ConnectionManager.new(opts)
       end
+    end
+
+    def self.init_connection(uri, opts = {})
+      http = Net::HTTP.new(opts[:ip] || uri.host, uri.port)
 
       if uri.port == 443
         http.use_ssl = true if http.respond_to?(:use_ssl=) # persistent does it automatically
         http.verify_mode = opts[:verify_ssl] === false ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
       end
 
-      [:open_timeout, :read_timeout, :ssl_version, :ciphers].each do |key|
-        http.send("#{key}=", opts[key]) if opts.has_key?(key)
+      [:keep_alive_timeout, :open_timeout, :read_timeout, :ssl_version, :ciphers].each do |key|
+        http.send("#{key}=", opts[key] || DEFAULTS[key]) if (opts.has_key?(key) || DEFAULTS.has_key?(key))
+      end
+
+      http
+    end
+
+    def self.init(uri, opts)
+      uri  = Utils.parse_uri(uri)
+
+      http = if opts.delete(:persistent)
+        init_persistent(opts)
+      else
+        init_connection(uri, opts)
       end
 
-      # new(http, [uri.scheme, uri.host].join('://'))
       new(http, uri.scheme_and_host)
     end
 
@@ -89,12 +110,20 @@ module Dagger
 
     def get(uri, opts = {})
       uri = Utils.resolve_uri(uri, @host, opts[:query])
-      raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}") if @host != uri.scheme_and_host
+
+      if @host != uri.scheme_and_host
+        raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}")
+      end
 
       opts[:follow] = 10 if opts[:follow] == true
       headers = opts[:headers] || {}
       headers['Accept'] = 'application/json' if opts[:json] && headers['Accept'].nil?
 
+      if opts[:ip]
+        headers['Host'] = uri.host
+        uri = opts[:ip]
+      end
+
       request = Net::HTTP::Get.new(uri, DEFAULT_HEADERS.merge(headers))
       request.basic_auth(opts.delete(:username), opts.delete(:password)) if opts[:username]
 
@@ -102,22 +131,23 @@ module Dagger
         @http.start unless @http.started?
         resp, data = @http.request(request)
       else # persistent
-        resp, data = @http.request(uri, request)
+        resp, data = @http.send_request(uri, request)
       end
 
       if REDIRECT_CODES.include?(resp.code.to_i) && resp['Location'] && (opts[:follow] && opts[:follow] > 0)
         opts[:follow] -= 1
-        puts "Following redirect to #{resp['Location']}"
+        debug "Following redirect to #{resp['Location']}"
         return get(resp['Location'], opts)
       end
 
       @response = build_response(resp, data || resp.body)
 
     rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EINVAL, Timeout::Error, \
-      SocketError, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
+      Net::OpenTimeout, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, \
+      SocketError, EOFError, OpenSSL::SSL::SSLError => e
 
       if retries = opts[:retries] and retries.to_i > 0
-        puts "Got #{e.class}! Retrying in a sec (#{retries} retries left)"
+        debug "Got #{e.class}! Retrying in a sec (#{retries} retries left)"
         sleep (opts[:retry_wait] || DEFAULT_RETRY_WAIT)
         get(uri, opts.merge(retries: retries - 1))
       else
@@ -148,7 +178,10 @@ module Dagger
       end
 
       uri = Utils.resolve_uri(uri, @host)
-      raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}") if @host != uri.scheme_and_host
+      if @host != uri.scheme_and_host
+        raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}")
+      end
+
       headers = DEFAULT_HEADERS.merge(opts[:headers] || {})
 
       query = if data.is_a?(String)
@@ -176,16 +209,17 @@ module Dagger
         req = Kernel.const_get("Net::HTTP::#{method.capitalize}").new(uri.path, headers)
         # req.set_form_data(query)
         req.body = query
-        resp, data = @http.request(uri, req)
+        resp, data = @http.send_request(uri, req)
       end
 
       @response = build_response(resp, data || resp.body)
 
     rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EINVAL, Timeout::Error, \
-      SocketError, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
+      Net::OpenTimeout, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, \
+      SocketError, EOFError, OpenSSL::SSL::SSLError => e
 
       if method.to_s.downcase != 'get' && retries = opts[:retries] and retries.to_i > 0
-        puts "[#{DAGGER_NAME}] Got #{e.class}! Retrying in a sec (#{retries} retries left)"
+        debug "[#{DAGGER_NAME}] Got #{e.class}! Retrying in a sec (#{retries} retries left)"
         sleep (opts[:retry_wait] || DEFAULT_RETRY_WAIT)
         request(method, uri, data, opts.merge(retries: retries - 1))
       else
@@ -198,7 +232,7 @@ module Dagger
     end
 
     def open(&block)
-      if @http.is_a?(Net::HTTP::Persistent)
+      if @http.is_a?(Dagger::ConnectionManager)
         instance_eval(&block)
       else
         @http.start do
@@ -208,7 +242,7 @@ module Dagger
     end
 
     def close
-      if @http.is_a?(Net::HTTP::Persistent)
+      if @http.is_a?(Dagger::ConnectionManager)
         @http.shutdown # calls finish on pool connections
       else
         @http.finish if @http.started?
@@ -217,6 +251,10 @@ module Dagger
 
     private
 
+    def debug(str)
+      puts str if ENV['DEBUGGING']
+    end
+
     def build_response(resp, body)
       resp.extend(Response)
       resp.set_body(body) unless resp.body
@@ -262,4 +300,4 @@ module Dagger
 
   end
 
-end
+end

data/lib/dagger/connection_manager.rb

@@ -0,0 +1,59 @@
+module Dagger
+
+  class ConnectionManager
+
+    def initialize(opts = {})
+      @opts = {}
+      @active_connections = {}
+      @mutex = Mutex.new
+    end
+
+    def shutdown
+      @mutex.synchronize do
+        # puts "Shutting down connections: #{@active_connections.count}"
+        @active_connections.each do |_, connection|
+          connection.finish
+        end
+        @active_connections = {}
+      end
+    end
+
+    # Gets a connection for a given URI. This is for internal use only as it's
+    # subject to change (we've moved between HTTP client schemes in the past
+    # and may do it again).
+    #
+    # `uri` is expected to be a string.
+    def connection_for(uri)
+      @mutex.synchronize do
+        connection = @active_connections[[uri.host, uri.port]]
+
+        if connection.nil?
+          connection = Dagger::Client.init_connection(uri, @opts)
+          connection.start
+
+          @active_connections[[uri.host, uri.port]] = connection
+          # puts "#{@active_connections.count} connections"
+        end
+
+        connection
+      end
+    end
+
+    # Executes an HTTP request to the given URI with the given method. Also
+    # allows a request body, headers, and query string to be specified.
+    def send_request(uri, request)
+      connection = connection_for(uri)
+      @mutex.synchronize do
+        begin
+          connection.request(request)
+        rescue StandardError => err
+          err
+        end
+      end.tap do |result|
+        raise(result) if result.is_a?(StandardError)
+      end
+    end
+
+  end
+
+end

data/lib/dagger/ox_extension.rb

@@ -5,12 +5,20 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
   alias_method :to_s, :text
   alias_method :value, :text
 
+  def to_node
+    self
+  end
+
   def count
     raise "Please call #children.count"
   end
 
-  def to_hash
-    self # for backwards compat
+  def keys
+    @keys ||= children.collect(&:name)
+  end
+
+  def is_array?
+    keys.count != keys.uniq.count
   end
 
   # this lets us traverse an parsed object like this:
@@ -20,30 +28,33 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
     found.empty? ? nil : found.size == 1 ? found.first : found
   end
 
+  # returns list of XMLNodes with matching names
   def slice(*arr)
     Array(arr).flatten.map { |key| self[key] }
   end
 
-  def values(arr = nil, include_empty: false)
-    if arr
-      Array(arr).flatten.each_with_object({}) do |key, memo| 
+  def values(keys_arr = nil, include_empty: false)
+    if keys_arr
+      Array(keys_arr).flatten.each_with_object({}) do |key, memo|
         if found = self[key] and (found.to_s || include_empty)
           memo[key] = found.to_s
         end
       end
+    elsif is_array?
+      children.map(&:values)
     else
-      children.each_with_object({}) do |child, memo| 
-        if child.to_s || include_empty
-          memo[child.name] = child.to_s
-        end
+      children.each_with_object({}) do |child, memo|
+        memo[child.name] = child.children.any? ? child.values : child.text
       end
     end
   end
 
+  alias_method :to_hash, :values
+  alias_method :to_h, :values
+
   def dig(*paths)
     list = Array(paths).flatten
     res = list.reduce([self]) do |parents, key|
-
       if parents
         found = parents.map do |parent|
           parent.children.select { |node| node.name.to_s == key.to_s }
@@ -79,14 +90,14 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
 end
 
 class Ox::Document
-  def to_hash
-    nodes.first.to_hash
+  def to_node
+    nodes.first.to_node
   end
 end
 
 class Ox::Element
-  def to_hash
-    children = nodes.map { |n| n.class == self.class ? n.to_hash : nil }.compact
+  def to_node
+    children = nodes.map { |n| n.class == self.class ? n.to_node : nil }.compact
     XMLNode.new(value, text, attributes, children)
   end
 end

data/lib/dagger/parsers.rb

@@ -26,7 +26,7 @@ class Parsers
 
   def text_xml(body)
     if res = Ox.parse(body)
-      res.to_hash
+      res.to_node
     end
   rescue Ox::ParseError
     nil

data/lib/dagger/version.rb

@@ -1,7 +1,7 @@
 module Dagger
   MAJOR = 1
-  MINOR = 5
-  PATCH = 1
+  MINOR = 8
+  PATCH = 0
 
   VERSION = [MAJOR, MINOR, PATCH].join('.')
 end

data/spec/arguments_spec.rb

@@ -23,7 +23,7 @@ describe 'arguments' do
     describe 'invalid URL' do
 
       it 'raises error' do
-        expect { send_request('asd123.rewqw') }.to raise_error(ArgumentError)
+        expect { send_request('asd123.rewqw') }.to raise_error(SocketError)
       end
 
     end
@@ -36,6 +36,15 @@ describe 'arguments' do
 
     end
 
+    describe 'host without protocol' do
+
+      it 'works' do
+        expect(send_request('www.google.com')).to be_a(Net::HTTPResponse)
+      end
+
+    end
+
+
     describe 'valid host' do
 
       it 'works' do

data/spec/ip_connect_spec.rb

@@ -0,0 +1,17 @@
+require './lib/dagger'
+
+require 'rspec/mocks'
+require 'rspec/expectations'
+
+describe 'IP Connection' do
+
+  it 'works' do
+    expect do
+      Dagger.get('http://www.awiefjoawijfaowef.com')
+    end.to raise_error(SocketError, /getaddrinfo/)
+
+    resp = Dagger.get('http://www.awiefjoawijfaowef.com', { ip: '1.1.1.1'} )
+    expect(resp.body).to match('<center>cloudflare</center>')
+  end
+
+end

data/spec/parsers_spec.rb

@@ -14,6 +14,9 @@ describe 'Parsers' do
 
   before do
     allow(Net::HTTP).to receive(:new).and_return(fake_http)
+    allow(fake_http).to receive(:keep_alive_timeout=).and_return(true)
+    allow(fake_http).to receive(:read_timeout=).and_return(true)
+    allow(fake_http).to receive(:open_timeout=).and_return(true)
     allow(fake_http).to receive(:verify_mode=).and_return(true)
     allow(fake_http).to receive(:post).and_return(fake_resp)
   end
@@ -100,7 +103,7 @@ describe 'Parsers' do
       it 'returns XMLNode obj' do
         res = send_request.data
         expect(res).to be_a(XMLNode)
-        expect(res.to_hash).to eql(res)
+        expect(res.to_node).to eql(res)
         expect(res['foo']).to be_a(XMLNode)
         expect(res['foo'].text).to eql('123')
 
@@ -126,7 +129,7 @@ describe 'Parsers' do
 
       it 'works' do
         doc = Ox.parse(xml)
-        obj = doc.to_hash
+        obj = doc.to_node
 
         expect(obj[:nested][:item][:title].text).to eql('foobar')
       end

data/spec/persistent_spec.rb

@@ -17,4 +17,50 @@ describe 'Persistent mode' do
     end
   end
 
+end
+
+describe 'using threads' do
+
+  def get(url)
+    @http.get(url)
+  end
+
+  def connect(host)
+    raise if @http
+    @http = Dagger.open(host)
+  end
+
+  def disconnect
+    raise if @http.nil?
+    @http.close
+    @http = nil
+  end
+
+  it 'works' do
+    thread_count = 10
+    urls_count = 100
+    host = 'https://postman-echo.com'
+    urls = urls_count.times.map { |i| "/get?page/#{i}" }
+    result = []
+
+    mutex = Mutex.new
+    thread_count.times.map do
+      Thread.new(urls, result) do |urls, result|
+        # mutex.synchronize { Dagger.open(host) }
+        http = Dagger.open(host)
+        while url = mutex.synchronize { urls.pop }
+          puts "Fetching #{url}"
+          resp = http.get(url)
+          mutex.synchronize do
+            result.push(resp.code)
+          end
+        end
+        # mutex.synchronize { http.close }
+        http.close
+      end
+    end.each(&:join)
+
+    expect(result.count).to eq(urls_count)
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dagger
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.8.0
 platform: ruby
 authors:
 - Tomás Pollak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-07 00:00:00.000000000 Z
+date: 2020-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: net-http-persistent
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -124,11 +110,13 @@ files:
 - bin/dagger
 - dagger.gemspec
 - lib/dagger.rb
+- lib/dagger/connection_manager.rb
 - lib/dagger/ox_extension.rb
 - lib/dagger/parsers.rb
 - lib/dagger/response.rb
 - lib/dagger/version.rb
 - spec/arguments_spec.rb
+- spec/ip_connect_spec.rb
 - spec/parsers_spec.rb
 - spec/persistent_spec.rb
 homepage: https://github.com/tomas/dagger