dagger 1.6.1 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d9231125c099e4e5e391e00719bede1da167f044242b6c2604d9690dee0d842
-  data.tar.gz: e997c56d4dc24be6cb747e5f1c350cffe2e804dd852a712ca3979381e4c6df03
+  metadata.gz: 692b25152c7fd3229ad3dc13bd25250b613e55a211e489b8835258b18405db19
+  data.tar.gz: 5560f64e580e4f72f9e243eaf1c5eed1b389e439074607dc24bdd8fbe9c532f0
 SHA512:
-  metadata.gz: 3e8c581c55ac7eb75af99f1ec50e2d85324d30f45faae37b45bf920f0bc6a63b5c38ba2ae99f5b98913698208f158dc8f0775e9b2833958a92a4a88dce392547
-  data.tar.gz: f870dc8b936d2e64e4d4dc0b06a94cd22527c0675f3317b5cb69b1c6c6e6263fd5a58a6fe35e37e8b87e2f74b535713127c92c525afbdcdb9e56ca9e64def6b8
+  metadata.gz: 85d160f0130c12fde8b2723cd54bcaf4a75c4f48319cb040c4725a93e6cc09d4f8a68a7977135f33c38922583cc8de481bc078ea6d3ff5636c42c3154a5dca74
+  data.tar.gz: 9bd29716a9e84b75e898f71b00e8fc145550fcccd15b2c0006172e56e3e5454b62668bc4536dacde371a33b429f437a942354368de4cce0394ff73296cefa8e8

data/dagger.gemspec

@@ -19,7 +19,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rspec-mocks"
   s.add_development_dependency "rspec-expectations"
 
-  s.add_runtime_dependency "net-http-persistent", "~> 3.0"
   s.add_runtime_dependency "oj", "~> 2.1"
   s.add_runtime_dependency "ox", "~> 2.4"
 

data/lib/dagger.rb

@@ -1,10 +1,10 @@
 require 'dagger/version'
 require 'dagger/response'
 require 'dagger/parsers'
-
-require 'net/http/persistent'
+require 'dagger/connection_manager'
 require 'net/https'
 require 'base64'
+require 'erb'
 
 class URI::HTTP
   def scheme_and_host
@@ -14,13 +14,19 @@ end
 
 module Dagger
 
-  DAGGER_NAME     = "Dagger/#{VERSION}"
+  DAGGER_NAME     = "Dagger/#{VERSION}".freeze
   REDIRECT_CODES  = [301, 302, 303].freeze
   DEFAULT_RETRY_WAIT = 5.freeze # seconds
   DEFAULT_HEADERS = {
     'Accept'     => '*/*',
-    'User-Agent' => "#{DAGGER_NAME} (Ruby Net::HTTP Wrapper, like curl)"
-  }
+    'User-Agent' => "#{DAGGER_NAME} (Ruby Net::HTTP wrapper, like curl)"
+  }.freeze
+
+  DEFAULTS = {
+    open_timeout: 10,
+    read_timeout: 10,
+    keep_alive_timeout: 10
+  }.freeze
 
   module Utils
 
@@ -34,7 +40,7 @@ module Dagger
     end
 
     def self.resolve_uri(uri, host = nil, query = nil)
-      uri = host + uri if uri.to_s['//'].nil? && host
+      uri = host + uri if uri[0] == '/' && host
       uri = parse_uri(uri.to_s)
       uri.path.sub!(/\?.*|$/, '?' + Utils.encode(query)) if query and query.any?
       uri
@@ -50,7 +56,7 @@ module Dagger
       when Array then obj.map { |v| encode(v, "#{key}[]") }.join('&')
       when nil   then ''
       else
-        "#{key}=#{URI.escape(obj.to_s)}"
+        "#{key}=#{ERB::Util.url_encode(obj.to_s)}"
       end
     end
 
@@ -62,25 +68,40 @@ module Dagger
 
   class Client
 
-    def self.init(uri, opts)
-      uri  = Utils.parse_uri(uri)
-      http = if opts.delete(:persistent)
-        pool_size = opts[:pool_size] || Net::HTTP::Persistent::DEFAULT_POOL_SIZE
-        Net::HTTP::Persistent.new(name: DAGGER_NAME, pool_size: pool_size)
-      else
-        Net::HTTP.new(opts[:ip] || uri.host, uri.port)
+    def self.init_persistent(opts = {})
+      # this line below forces one connection manager between multiple threads
+      # @persistent ||= Dagger::ConnectionManager.new(opts)
+
+      # here we initialize a connection manager for each thread
+      Thread.current[:dagger_persistent] ||= begin
+        Dagger::ConnectionManager.new(opts)
       end
+    end
+
+    def self.init_connection(uri, opts = {})
+      http = Net::HTTP.new(opts[:ip] || uri.host, uri.port)
 
       if uri.port == 443
         http.use_ssl = true if http.respond_to?(:use_ssl=) # persistent does it automatically
         http.verify_mode = opts[:verify_ssl] === false ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
       end
 
-      [:open_timeout, :read_timeout, :ssl_version, :ciphers].each do |key|
-        http.send("#{key}=", opts[key]) if opts.has_key?(key)
+      [:keep_alive_timeout, :open_timeout, :read_timeout, :ssl_version, :ciphers].each do |key|
+        http.send("#{key}=", opts[key] || DEFAULTS[key]) if (opts.has_key?(key) || DEFAULTS.has_key?(key))
+      end
+
+      http
+    end
+
+    def self.init(uri, opts)
+      uri  = Utils.parse_uri(uri)
+
+      http = if opts.delete(:persistent)
+        init_persistent(opts)
+      else
+        init_connection(uri, opts)
       end
 
-      # new(http, [uri.scheme, uri.host].join('://'))
       new(http, uri.scheme_and_host)
     end
 
@@ -91,7 +112,9 @@ module Dagger
     def get(uri, opts = {})
       uri = Utils.resolve_uri(uri, @host, opts[:query])
 
-      raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}") if @host != uri.scheme_and_host
+      if @host != uri.scheme_and_host
+        raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}")
+      end
 
       opts[:follow] = 10 if opts[:follow] == true
       headers = opts[:headers] || {}
@@ -109,23 +132,23 @@ module Dagger
         @http.start unless @http.started?
         resp, data = @http.request(request)
       else # persistent
-        puts @http.pool.inspect
-        resp, data = @http.request(uri, request)
+        resp, data = @http.send_request(uri, request)
       end
 
       if REDIRECT_CODES.include?(resp.code.to_i) && resp['Location'] && (opts[:follow] && opts[:follow] > 0)
         opts[:follow] -= 1
-        puts "Following redirect to #{resp['Location']}"
+        debug "Following redirect to #{resp['Location']}"
         return get(resp['Location'], opts)
       end
 
       @response = build_response(resp, data || resp.body)
 
     rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EINVAL, Timeout::Error, \
-      SocketError, EOFError, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
+      Net::OpenTimeout, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, \
+      SocketError, EOFError, OpenSSL::SSL::SSLError => e
 
       if retries = opts[:retries] and retries.to_i > 0
-        puts "Got #{e.class}! Retrying in a sec (#{retries} retries left)"
+        debug "Got #{e.class}! Retrying in a sec (#{retries} retries left)"
         sleep (opts[:retry_wait] || DEFAULT_RETRY_WAIT)
         get(uri, opts.merge(retries: retries - 1))
       else
@@ -156,7 +179,10 @@ module Dagger
       end
 
       uri = Utils.resolve_uri(uri, @host)
-      raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}") if @host != uri.scheme_and_host
+      if @host != uri.scheme_and_host
+        raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}")
+      end
+
       headers = DEFAULT_HEADERS.merge(opts[:headers] || {})
 
       query = if data.is_a?(String)
@@ -184,16 +210,17 @@ module Dagger
         req = Kernel.const_get("Net::HTTP::#{method.capitalize}").new(uri.path, headers)
         # req.set_form_data(query)
         req.body = query
-        resp, data = @http.request(uri, req)
+        resp, data = @http.send_request(uri, req)
       end
 
       @response = build_response(resp, data || resp.body)
 
     rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EINVAL, Timeout::Error, \
-      SocketError, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
+      Net::OpenTimeout, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, \
+      SocketError, EOFError, OpenSSL::SSL::SSLError => e
 
       if method.to_s.downcase != 'get' && retries = opts[:retries] and retries.to_i > 0
-        puts "[#{DAGGER_NAME}] Got #{e.class}! Retrying in a sec (#{retries} retries left)"
+        debug "[#{DAGGER_NAME}] Got #{e.class}! Retrying in a sec (#{retries} retries left)"
         sleep (opts[:retry_wait] || DEFAULT_RETRY_WAIT)
         request(method, uri, data, opts.merge(retries: retries - 1))
       else
@@ -206,7 +233,7 @@ module Dagger
     end
 
     def open(&block)
-      if @http.is_a?(Net::HTTP::Persistent)
+      if @http.is_a?(Dagger::ConnectionManager)
         instance_eval(&block)
       else
         @http.start do
@@ -216,7 +243,7 @@ module Dagger
     end
 
     def close
-      if @http.is_a?(Net::HTTP::Persistent)
+      if @http.is_a?(Dagger::ConnectionManager)
         @http.shutdown # calls finish on pool connections
       else
         @http.finish if @http.started?
@@ -225,6 +252,10 @@ module Dagger
 
     private
 
+    def debug(str)
+      puts str if ENV['DEBUGGING']
+    end
+
     def build_response(resp, body)
       resp.extend(Response)
       resp.set_body(body) unless resp.body
@@ -270,4 +301,4 @@ module Dagger
 
   end
 
-end
+end

data/lib/dagger/connection_manager.rb

@@ -0,0 +1,59 @@
+module Dagger
+
+  class ConnectionManager
+
+    def initialize(opts = {})
+      @opts = {}
+      @active_connections = {}
+      @mutex = Mutex.new
+    end
+
+    def shutdown
+      @mutex.synchronize do
+        # puts "Shutting down connections: #{@active_connections.count}"
+        @active_connections.each do |_, connection|
+          connection.finish
+        end
+        @active_connections = {}
+      end
+    end
+
+    # Gets a connection for a given URI. This is for internal use only as it's
+    # subject to change (we've moved between HTTP client schemes in the past
+    # and may do it again).
+    #
+    # `uri` is expected to be a string.
+    def connection_for(uri)
+      @mutex.synchronize do
+        connection = @active_connections[[uri.host, uri.port]]
+
+        if connection.nil?
+          connection = Dagger::Client.init_connection(uri, @opts)
+          connection.start
+
+          @active_connections[[uri.host, uri.port]] = connection
+          # puts "#{@active_connections.count} connections"
+        end
+
+        connection
+      end
+    end
+
+    # Executes an HTTP request to the given URI with the given method. Also
+    # allows a request body, headers, and query string to be specified.
+    def send_request(uri, request)
+      connection = connection_for(uri)
+      @mutex.synchronize do
+        begin
+          connection.request(request)
+        rescue StandardError => err
+          err
+        end
+      end.tap do |result|
+        raise(result) if result.is_a?(StandardError)
+      end
+    end
+
+  end
+
+end

data/lib/dagger/ox_extension.rb

@@ -5,12 +5,20 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
   alias_method :to_s, :text
   alias_method :value, :text
 
+  def to_node
+    self
+  end
+
   def count
     raise "Please call #children.count"
   end
 
-  def to_hash
-    self # for backwards compat
+  def keys
+    @keys ||= children.collect(&:name)
+  end
+
+  def is_array?
+    keys.count != keys.uniq.count
   end
 
   # this lets us traverse an parsed object like this:
@@ -20,30 +28,33 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
     found.empty? ? nil : found.size == 1 ? found.first : found
   end
 
+  # returns list of XMLNodes with matching names
   def slice(*arr)
     Array(arr).flatten.map { |key| self[key] }
   end
 
-  def values(arr = nil, include_empty: false)
-    if arr
-      Array(arr).flatten.each_with_object({}) do |key, memo| 
+  def values(keys_arr = nil, include_empty: false)
+    if keys_arr
+      Array(keys_arr).flatten.each_with_object({}) do |key, memo|
         if found = self[key] and (found.to_s || include_empty)
           memo[key] = found.to_s
         end
       end
+    elsif is_array?
+      children.map(&:values)
     else
-      children.each_with_object({}) do |child, memo| 
-        if child.to_s || include_empty
-          memo[child.name] = child.to_s
-        end
+      children.each_with_object({}) do |child, memo|
+        memo[child.name] = child.children.any? ? child.values : child.text
       end
     end
   end
 
+  alias_method :to_hash, :values
+  alias_method :to_h, :values
+
   def dig(*paths)
     list = Array(paths).flatten
     res = list.reduce([self]) do |parents, key|
-
       if parents
         found = parents.map do |parent|
           parent.children.select { |node| node.name.to_s == key.to_s }
@@ -79,14 +90,14 @@ XMLNode = Struct.new(:name, :text, :attributes, :children) do
 end
 
 class Ox::Document
-  def to_hash
-    nodes.first.to_hash
+  def to_node
+    nodes.first.to_node
   end
 end
 
 class Ox::Element
-  def to_hash
-    children = nodes.map { |n| n.class == self.class ? n.to_hash : nil }.compact
+  def to_node
+    children = nodes.map { |n| n.class == self.class ? n.to_node : nil }.compact
     XMLNode.new(value, text, attributes, children)
   end
 end

data/lib/dagger/parsers.rb

@@ -26,7 +26,7 @@ class Parsers
 
   def text_xml(body)
     if res = Ox.parse(body)
-      res.to_hash
+      res.to_node
     end
   rescue Ox::ParseError
     nil

data/lib/dagger/version.rb

@@ -1,7 +1,7 @@
 module Dagger
   MAJOR = 1
-  MINOR = 6
-  PATCH = 1
+  MINOR = 9
+  PATCH = 0
 
   VERSION = [MAJOR, MINOR, PATCH].join('.')
 end

data/lib/dagger/wrapper.rb

@@ -0,0 +1,151 @@
+require_relative '../dagger'
+
+module Dagger
+
+  module Wrapper
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def base_url(str = nil)
+        if str # set
+          @base_url = str
+        else
+          @base_url or raise "base_url unset!" # get
+        end
+      end
+
+      def base_options(opts = nil)
+        if opts # set
+          @base_options = opts
+        else
+          @base_options or raise "base_url unset!" # get
+        end
+      end
+    end
+
+    def initialize(opts = {})
+      @logger = opts.delete(:logger)
+      @options = opts
+    end
+
+    def get(path, params = {}, opts = {})
+      request(:get, path, params, opts)
+    end
+
+    def post(path, params = {}, opts = {})
+      request(:post, path, params, opts)
+    end
+
+    def put(path, params = {}, opts = {})
+      request(:put, path, params, opts)
+    end
+
+    def patch(path, params = {}, opts = {})
+      request(:patch, path, params, opts)
+    end
+
+    def delete(path, params = {}, opts = {})
+      request(:delete, path, params, opts)
+    end
+
+    def request(method, path, params = {}, opts = nil)
+      url = self.class.base_url + path
+      resp = benchmark("#{method} #{path}") do
+        http.request(method, url, params, base_options.merge(opts))
+      end
+
+      handle_response(resp, method, path, params)
+    end
+
+    def connect(&block)
+      open_http
+      if block_given?
+        yield
+        close_http
+      else
+        at_exit { close_http }
+      end
+    end
+
+    def disconnect
+      close_http
+    end
+
+    private
+    attr_reader :options
+
+    def handle_response(resp, method, path, params)
+      resp
+    end
+
+    def base_options
+      {}
+    end
+
+    def request_options
+      self.class.base_options.merge(base_options)
+    end
+
+    def benchmark(message, &block)
+      log(message)
+      start = Time.now
+      resp = yield
+      time = Time.now - start
+      log("Got response in #{time.round(2)} secs")
+      resp
+    end
+
+    def log(str)
+      logger.info(str)
+    end
+
+    def logger
+      @logger ||= begin
+        require 'logger'
+        Logger.new(@options[:logfile])
+      end
+    end
+
+    def http
+      @http || Dagger
+    end
+
+    def open_http
+      raise "Already open!" if @http
+      @http = Dagger.open(self.class.base_url)
+    end
+
+    def close_http
+      @http.close if @http
+      @http = nil
+    end
+
+    # def wrap(hash)
+    #   Entity.new(hash)
+    # end
+
+    # class Entity
+    #   def initialize(props)
+    #     @props = props
+    #   end
+
+    #   def get(prop)
+    #     val = @props[name.to_s]
+    #   end
+
+    #   def method_missing(name, args, &block)
+    #     if @props.key?(name.to_s)
+    #       get(name)
+    #     else
+    #       # raise NoMethodError, "undefined method #{name}"
+    #       super
+    #     end
+    #   end
+    # end
+
+  end
+
+end

data/spec/arguments_spec.rb

@@ -23,7 +23,7 @@ describe 'arguments' do
     describe 'invalid URL' do
 
       it 'raises error' do
-        expect { send_request('asd123.rewqw') }.to raise_error(ArgumentError)
+        expect { send_request('asd123.rewqw') }.to raise_error(SocketError)
       end
 
     end
@@ -36,6 +36,15 @@ describe 'arguments' do
 
     end
 
+    describe 'host without protocol' do
+
+      it 'works' do
+        expect(send_request('www.google.com')).to be_a(Net::HTTPResponse)
+      end
+
+    end
+
+
     describe 'valid host' do
 
       it 'works' do

data/spec/parsers_spec.rb

@@ -14,6 +14,9 @@ describe 'Parsers' do
 
   before do
     allow(Net::HTTP).to receive(:new).and_return(fake_http)
+    allow(fake_http).to receive(:keep_alive_timeout=).and_return(true)
+    allow(fake_http).to receive(:read_timeout=).and_return(true)
+    allow(fake_http).to receive(:open_timeout=).and_return(true)
     allow(fake_http).to receive(:verify_mode=).and_return(true)
     allow(fake_http).to receive(:post).and_return(fake_resp)
   end
@@ -100,7 +103,7 @@ describe 'Parsers' do
       it 'returns XMLNode obj' do
         res = send_request.data
         expect(res).to be_a(XMLNode)
-        expect(res.to_hash).to eql(res)
+        expect(res.to_node).to eql(res)
         expect(res['foo']).to be_a(XMLNode)
         expect(res['foo'].text).to eql('123')
 
@@ -119,16 +122,30 @@ describe 'Parsers' do
           <nested>
             <item>
               <title attr="downcased">foobar</title>
+              <category>cat</category>
+              <published>true</published>
             </item>
           </nested>
         </xml>
       )
 
-      it 'works' do
-        doc = Ox.parse(xml)
-        obj = doc.to_hash
+      describe '#to_node' do
+        it 'works as expected' do
+          doc = Ox.parse(xml)
+          obj = doc.to_node
+          expect(obj[:nested][:item][:title].text).to eql('foobar')
+        end
+      end
 
-        expect(obj[:nested][:item][:title].text).to eql('foobar')
+      describe '#values' do
+        it 'works as expected' do
+          doc = Ox.parse(xml)
+          obj = doc.to_node.values
+          expect(obj).to eql({
+            "foo" => "test",
+            "nested" => {"item"=>{"category"=>"cat", "published"=>"true", "title"=>"foobar"}},
+          })
+        end
       end
 
     end

data/spec/persistent_spec.rb

@@ -17,4 +17,50 @@ describe 'Persistent mode' do
     end
   end
 
+end
+
+describe 'using threads' do
+
+  def get(url)
+    @http.get(url)
+  end
+
+  def connect(host)
+    raise if @http
+    @http = Dagger.open(host)
+  end
+
+  def disconnect
+    raise if @http.nil?
+    @http.close
+    @http = nil
+  end
+
+  it 'works' do
+    thread_count = 10
+    urls_count = 100
+    host = 'https://postman-echo.com'
+    urls = urls_count.times.map { |i| "/get?page/#{i}" }
+    result = []
+
+    mutex = Mutex.new
+    thread_count.times.map do
+      Thread.new(urls, result) do |urls, result|
+        # mutex.synchronize { Dagger.open(host) }
+        http = Dagger.open(host)
+        while url = mutex.synchronize { urls.pop }
+          puts "Fetching #{url}"
+          resp = http.get(url)
+          mutex.synchronize do
+            result.push(resp.code)
+          end
+        end
+        # mutex.synchronize { http.close }
+        http.close
+      end
+    end.each(&:join)
+
+    expect(result.count).to eq(urls_count)
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dagger
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.9.0
 platform: ruby
 authors:
 - Tomás Pollak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-31 00:00:00.000000000 Z
+date: 2020-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: net-http-persistent
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -124,10 +110,12 @@ files:
 - bin/dagger
 - dagger.gemspec
 - lib/dagger.rb
+- lib/dagger/connection_manager.rb
 - lib/dagger/ox_extension.rb
 - lib/dagger/parsers.rb
 - lib/dagger/response.rb
 - lib/dagger/version.rb
+- lib/dagger/wrapper.rb
 - spec/arguments_spec.rb
 - spec/ip_connect_spec.rb
 - spec/parsers_spec.rb