requests_ruby 1.0.2 → 1.0.4

data/lib/requests/http2.rb

@@ -0,0 +1,469 @@
+# frozen_string_literal: true
+
+require 'socket'
+require 'openssl'
+require 'uri'
+require 'zlib'
+require 'timeout'
+require 'time'
+
+module Requests
+  class Http2Response
+    def initialize(header_pairs, body)
+      @status = nil
+      @fields = Hash.new { |h, k| h[k] = [] }
+      @headers = CIHash.new
+      header_pairs.each do |k, v|
+        if k == ':status'
+          @status = v
+        else
+          @fields[k.downcase] << v
+          @headers[k] = v
+        end
+      end
+      @body = body
+    end
+    def code
+      @status
+    end
+    def message
+      (Requests::STATUS_CODES[@status.to_i] || :unknown).to_s.split('_').map(&:capitalize).join(' ')
+    end
+    def [](k)
+      @headers[k]
+    end
+    def each_header
+      return enum_for(:each_header) unless block_given?
+      @headers.each { |k, v| yield k, v }
+    end
+    def get_fields(name)
+      f = @fields[name.to_s.downcase]
+      f.empty? ? nil : f
+    end
+    def body
+      @body
+    end
+  end
+
+  class Http2Connection
+    TYPE_DATA = 0x0
+    TYPE_HEADERS = 0x1
+    TYPE_PRIORITY = 0x2
+    TYPE_RST_STREAM = 0x3
+    TYPE_SETTINGS = 0x4
+    TYPE_PUSH_PROMISE = 0x5
+    TYPE_PING = 0x6
+    TYPE_GOAWAY = 0x7
+    TYPE_WINDOW_UPDATE = 0x8
+    TYPE_CONTINUATION = 0x9
+    FLAG_END_STREAM = 0x1
+    FLAG_ACK = 0x1
+    FLAG_END_HEADERS = 0x4
+    FLAG_PADDED = 0x8
+    FLAG_PRIORITY = 0x20
+    PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
+    DEFAULT_WINDOW = 65_535
+
+    def initialize(socket)
+      @socket = socket
+      @next_stream_id = 1
+      @encoder = HPACK::Encoder.new
+      @decoder = HPACK::Decoder.new
+      @send_window = DEFAULT_WINDOW
+      @conn_recv_window = DEFAULT_WINDOW
+      @peer_max_frame_size = 16_384
+      @peer_initial_window = DEFAULT_WINDOW
+      @streams = {}
+      @closed = false
+      handshake!
+    end
+    def alive?
+      !@closed && !@socket.closed?
+    end
+    def close
+      write_frame(TYPE_GOAWAY, 0, 0, [0, 0].pack('NN')) if alive?
+    rescue StandardError
+      nil
+    ensure
+      @closed = true
+      @socket.close unless @socket.closed?
+    end
+    def request(method, uri, header_list, body, io: nil, progress: nil)
+      raise Requests::ConnectionError, 'http/2 connection is closed' unless alive?
+      stream_id = @next_stream_id
+      @next_stream_id += 2
+      @streams[stream_id] = { send_window: @peer_initial_window, recv_window: DEFAULT_WINDOW }
+      end_stream = body.nil? || body.to_s.empty?
+      send_headers(stream_id, header_list, end_stream)
+      send_body(stream_id, body) unless end_stream
+      read_response(stream_id, io: io, progress: progress)
+    ensure
+      @streams.delete(stream_id) if stream_id
+    end
+    private
+    def handshake!
+      @socket.write(PREFACE)
+      write_frame(TYPE_SETTINGS, 0, 0, '')
+      loop do
+        type, flags, sid, payload = read_frame
+        case type
+        when TYPE_SETTINGS
+          if flags & FLAG_ACK == 0
+            apply_settings(payload)
+            write_frame(TYPE_SETTINGS, FLAG_ACK, 0, '')
+          end
+          break
+        when TYPE_WINDOW_UPDATE
+          apply_window_update(sid, payload)
+        end
+      end
+    end
+    def send_headers(stream_id, header_list, end_stream)
+      blob = @encoder.encode(header_list)
+      offset = 0
+      first = true
+      loop do
+        chunk = blob.byteslice(offset, @peer_max_frame_size) || ''
+        offset += chunk.bytesize
+        last = offset >= blob.bytesize
+        flags = 0
+        flags |= FLAG_END_HEADERS if last
+        flags |= FLAG_END_STREAM if last && end_stream
+        write_frame(first ? TYPE_HEADERS : TYPE_CONTINUATION, flags, stream_id, chunk)
+        first = false
+        break if last
+      end
+    end
+    def send_body(stream_id, body)
+      pos = 0
+      st = @streams[stream_id]
+      body = body.dup.force_encoding(Encoding::BINARY)
+      loop do
+        avail = [st[:send_window], @send_window, @peer_max_frame_size].min
+        if avail <= 0
+          type, flags, sid, payload = read_frame
+          dispatch_control_frame(type, flags, sid, payload)
+          next
+        end
+        chunk = body.byteslice(pos, [avail, body.bytesize - pos].min)
+        pos += chunk.bytesize
+        st[:send_window] -= chunk.bytesize
+        @send_window -= chunk.bytesize
+        last = pos >= body.bytesize
+        write_frame(TYPE_DATA, last ? FLAG_END_STREAM : 0, stream_id, chunk)
+        break if last
+      end
+    end
+    def read_response(stream_id, io: nil, progress: nil)
+      header_blob = String.new(encoding: Encoding::BINARY)
+      headers = nil
+      body = io ? nil : String.new(encoding: Encoding::BINARY)
+      decoder = nil
+      total = 0
+      content_length = nil
+      done = false
+      until done
+        type, flags, sid, payload = read_frame
+        case type
+        when TYPE_HEADERS, TYPE_CONTINUATION
+          next unless sid == stream_id || headers.nil?
+          block = type == TYPE_HEADERS ? strip_header_padding(payload, flags) : payload
+          header_blob << block
+          if flags & FLAG_END_HEADERS != 0
+            headers = @decoder.decode(header_blob)
+            if io
+              hdrs = CIHash.new
+              headers.each { |k, v| hdrs[k] = v unless k == ':status' }
+              decoder = stream_decoder(hdrs)
+              content_length = hdrs['content-length']&.to_i
+            end
+          end
+          done = true if sid == stream_id && flags & FLAG_END_STREAM != 0
+        when TYPE_DATA
+          if sid == stream_id
+            chunk = strip_data_padding(payload, flags)
+            update_recv_window(stream_id, payload.bytesize)
+            if io
+              piece = decoder == :raw ? chunk : decoder.inflate(chunk)
+              io.write(piece)
+              total += piece.bytesize
+              progress.call(total, content_length) if progress
+            else
+              body << chunk
+            end
+          end
+          done = true if sid == stream_id && flags & FLAG_END_STREAM != 0
+        else
+          dispatch_control_frame(type, flags, sid, payload)
+          done = true if type == TYPE_RST_STREAM && sid == stream_id
+        end
+      end
+      Http2Response.new(headers || [], body)
+    end
+    def dispatch_control_frame(type, flags, sid, payload)
+      case type
+      when TYPE_WINDOW_UPDATE
+        apply_window_update(sid, payload)
+      when TYPE_SETTINGS
+        if flags & FLAG_ACK == 0
+          apply_settings(payload)
+          write_frame(TYPE_SETTINGS, FLAG_ACK, 0, '')
+        end
+      when TYPE_PING
+        write_frame(TYPE_PING, FLAG_ACK, 0, payload) if flags & FLAG_ACK == 0
+      when TYPE_GOAWAY
+        @closed = true
+      when TYPE_RST_STREAM
+        raise Requests::ConnectionError, 'stream reset by server'
+      when TYPE_PUSH_PROMISE
+        psid = payload.byteslice(0, 4).unpack1('N') & 0x7fffffff
+        write_frame(TYPE_RST_STREAM, 0, psid, [0x8].pack('N'))
+      end
+    end
+    def apply_settings(payload)
+      i = 0
+      while i + 6 <= payload.bytesize
+        id = payload.byteslice(i, 2).unpack1('n')
+        val = payload.byteslice(i + 2, 4).unpack1('N')
+        @peer_initial_window = val if id == 4
+        @peer_max_frame_size = val if id == 5
+        i += 6
+      end
+    end
+    def apply_window_update(sid, payload)
+      inc = payload.unpack1('N') & 0x7fffffff
+      if sid == 0
+        @send_window += inc
+      else
+        (@streams[sid] ||= { send_window: @peer_initial_window, recv_window: DEFAULT_WINDOW })[:send_window] += inc
+      end
+    end
+    def update_recv_window(stream_id, n)
+      st = @streams[stream_id]
+      return unless st
+      st[:recv_window] -= n
+      @conn_recv_window -= n
+      if st[:recv_window] < 32_768
+        inc = DEFAULT_WINDOW - st[:recv_window]
+        write_frame(TYPE_WINDOW_UPDATE, 0, stream_id, [inc].pack('N'))
+        st[:recv_window] += inc
+      end
+      return unless @conn_recv_window < 32_768
+      inc = DEFAULT_WINDOW - @conn_recv_window
+      write_frame(TYPE_WINDOW_UPDATE, 0, 0, [inc].pack('N'))
+      @conn_recv_window += inc
+    end
+    def stream_decoder(hdrs)
+      case hdrs['content-encoding'].to_s.downcase
+      when 'gzip', 'x-gzip' then Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
+      when 'deflate' then Zlib::Inflate.new
+      else :raw
+      end
+    end
+    def strip_header_padding(payload, flags)
+      pos = 0
+      padlen = 0
+      if flags & FLAG_PADDED != 0
+        padlen = payload.getbyte(0)
+        pos += 1
+      end
+      pos += 5 if flags & FLAG_PRIORITY != 0
+      payload.byteslice(pos, payload.bytesize - pos - padlen)
+    end
+    def strip_data_padding(payload, flags)
+      return payload if flags & FLAG_PADDED == 0
+      padlen = payload.getbyte(0)
+      payload.byteslice(1, payload.bytesize - 1 - padlen)
+    end
+    def read_frame
+      hdr = read_exactly(9)
+      length = (hdr.getbyte(0) << 16) | (hdr.getbyte(1) << 8) | hdr.getbyte(2)
+      type = hdr.getbyte(3)
+      flags = hdr.getbyte(4)
+      sid = hdr.byteslice(5, 4).unpack1('N') & 0x7fffffff
+      payload = length.positive? ? read_exactly(length) : ''
+      [type, flags, sid, payload]
+    end
+    def write_frame(type, flags, stream_id, payload)
+      len = payload.bytesize
+      hdr = [(len >> 16) & 0xff, (len >> 8) & 0xff, len & 0xff, type, flags].pack('C5') + [stream_id & 0x7fffffff].pack('N')
+      @socket.write(hdr + payload)
+    end
+    def read_exactly(n)
+      buf = String.new(encoding: Encoding::BINARY)
+      while buf.bytesize < n
+        chunk = @socket.read(n - buf.bytesize)
+        raise Requests::ConnectionError, 'connection closed by peer' if chunk.nil?
+        buf << chunk
+      end
+      buf
+    end
+  end
+
+  class Http2Adapter
+    attr_accessor :max_retries, :backoff_factor, :status_forcelist
+    def initialize(max_retries: 0, backoff_factor: 0, status_forcelist: [])
+      @max_retries = max_retries
+      @backoff_factor = backoff_factor
+      @status_forcelist = status_forcelist
+      @pool = {}
+      @alpn = {}
+      @fallback = HTTPAdapter.new(max_retries: max_retries, backoff_factor: backoff_factor, status_forcelist: status_forcelist)
+    end
+    def send_once(meth, url, hdrs, body, timeout, proxies, verify, cert, auth)
+      u = URI.parse(url)
+      sync_fallback!
+      return @fallback.send_once(meth, url, hdrs, body, timeout, proxies, verify, cert, auth) unless usable?(u, proxies)
+      conn = conn_for(u, verify, cert, timeout)
+      return @fallback.send_once(meth, url, hdrs, body, timeout, proxies, verify, cert, auth) if conn == :http1
+      attempt = 0
+      loop do
+        begin
+          auth.call(hdrs) if auth.respond_to?(:call) && !auth.is_a?(Requests::DigestAuth)
+          resp = perform(conn, meth, u, hdrs, body, timeout)
+          if @status_forcelist.include?(resp.code.to_i) && attempt < @max_retries
+            attempt += 1
+            sleep(retry_delay(resp, attempt))
+            conn = conn_for(u, verify, cert, timeout)
+            next
+          end
+          return resp
+        rescue Requests::ConnectionError, Requests::ConnectTimeout => e
+          drop_conn(u)
+          attempt += 1
+          raise e unless attempt <= @max_retries && idempotent?(meth)
+          sleep(@backoff_factor * attempt) if @backoff_factor.to_f > 0
+          conn = conn_for(u, verify, cert, timeout)
+          return @fallback.send_once(meth, url, hdrs, body, timeout, proxies, verify, cert, auth) if conn == :http1
+        end
+      end
+    end
+    def stream_to(meth, url, hdrs, body, timeout, proxies, verify, cert, auth, io, chunk_size: 65_536, progress: nil)
+      u = URI.parse(url)
+      sync_fallback!
+      unless usable?(u, proxies)
+        return @fallback.stream_to(meth, url, hdrs, body, timeout, proxies, verify, cert, auth, io, chunk_size: chunk_size, progress: progress)
+      end
+      conn = conn_for(u, verify, cert, timeout)
+      if conn == :http1
+        return @fallback.stream_to(meth, url, hdrs, body, timeout, proxies, verify, cert, auth, io, chunk_size: chunk_size, progress: progress)
+      end
+      auth.call(hdrs) if auth.respond_to?(:call) && !auth.is_a?(Requests::DigestAuth)
+      begin
+        header_list = build_headers(meth, u, hdrs)
+        _open_t, read_t = split_timeout(timeout)
+        with_timeout(read_t) { conn.request(meth, u, header_list, body, io: io, progress: progress) }
+      rescue Requests::ConnectionError, Requests::ConnectTimeout => e
+        drop_conn(u)
+        raise e
+      end
+    end
+    def decode_body(net_resp)
+      @fallback.decode_body(net_resp)
+    end
+    def close_pool
+      @pool.each_value { |c| c.close }
+      @pool.clear
+      @fallback.close_pool
+    end
+    private
+    def perform(conn, meth, u, hdrs, body, timeout)
+      header_list = build_headers(meth, u, hdrs)
+      _open_t, read_t = split_timeout(timeout)
+      with_timeout(read_t) { conn.request(meth, u, header_list, body) }
+    end
+    def with_timeout(read_t, &blk)
+      return blk.call unless read_t
+      ::Timeout.timeout(read_t, Requests::ReadTimeout, &blk)
+    end
+    def sync_fallback!
+      @fallback.max_retries = @max_retries
+      @fallback.backoff_factor = @backoff_factor
+      @fallback.status_forcelist = @status_forcelist
+    end
+    def idempotent?(meth)
+      %w[GET HEAD OPTIONS PUT DELETE].include?(meth.to_s.upcase)
+    end
+    def retry_delay(resp, attempt)
+      ra = resp['retry-after']
+      if ra
+        return ra.to_i if ra =~ /\A\d+\z/
+        begin
+          d = Time.httpdate(ra) - Time.now
+          return d.positive? ? d : 0
+        rescue ArgumentError
+          nil
+        end
+      end
+      @backoff_factor.to_f > 0 ? @backoff_factor * attempt : 0
+    end
+    def usable?(u, proxies)
+      return false unless u.scheme == 'https'
+      return false if proxies && !proxies.empty? && (proxies[u.scheme] || proxies[u.scheme.to_sym])
+      true
+    end
+    def build_headers(meth, u, hdrs)
+      port = u.port && u.port != 443 ? ":#{u.port}" : ''
+      list = [[':method', meth.to_s.upcase], [':scheme', 'https'], [':path', u.request_uri], [':authority', "#{u.host}#{port}"]]
+      hdrs.each { |k, v| list << [k.to_s, v.to_s] unless k.to_s.downcase == 'host' }
+      list
+    end
+    def key_for(u)
+      "#{u.host}:#{u.port}"
+    end
+    def conn_for(u, verify, cert, timeout)
+      k = key_for(u)
+      cached = @pool[k]
+      return cached if cached && cached.alive?
+      @pool.delete(k)
+      return :http1 if @alpn[k] == 'http1'
+      sock = open_tls(u, verify, cert, timeout)
+      if sock.alpn_protocol == 'h2'
+        @alpn[k] = 'h2'
+        @pool[k] = Http2Connection.new(sock)
+      else
+        @alpn[k] = 'http1'
+        sock.close rescue nil
+        :http1
+      end
+    end
+    def drop_conn(u)
+      c = @pool.delete(key_for(u))
+      c.close if c
+    end
+    def open_tls(u, verify, cert, timeout)
+      open_t, _read_t = split_timeout(timeout)
+      tcp = open_t ? Socket.tcp(u.host, u.port, connect_timeout: open_t) : Socket.tcp(u.host, u.port)
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.alpn_protocols = ['h2', 'http/1.1']
+      if verify == false
+        ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      else
+        ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        ctx.ca_file = verify.is_a?(String) ? verify : Requests::CA_FILE
+      end
+      if cert
+        ctx.cert = OpenSSL::X509::Certificate.new(File.read(cert[0]))
+        ctx.key = OpenSSL::PKey::RSA.new(File.read(cert[1]))
+      end
+      ssl = OpenSSL::SSL::SSLSocket.new(tcp, ctx)
+      ssl.hostname = u.host
+      ssl.sync_close = true
+      ssl.connect
+      ssl
+    rescue Errno::ETIMEDOUT, IO::TimeoutError
+      raise Requests::ConnectTimeout, "connect timeout: #{u}"
+    rescue OpenSSL::SSL::SSLError => e
+      raise Requests::SSLError, e.message
+    rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ECONNRESET => e
+      raise Requests::ConnectionError, e.message
+    end
+    def split_timeout(t)
+      case t
+      when nil then [nil, nil]
+      when Array then [t[0], t[1]]
+      else [t, t]
+      end
+    end
+  end
+end

data/lib/requests/sessions.rb

@@ -6,8 +6,8 @@ require 'time'
 module Requests
   class Session
     attr_accessor :headers, :cookies, :auth, :params, :proxies, :verify,
-                  :cert, :max_redirects, :timeout, :retries, :backoff_factor
-    def initialize
+                  :cert, :max_redirects, :timeout, :retries, :backoff_factor, :status_forcelist, :trust_env
+    def initialize(http2: false)
       @headers = Requests::Utils.default_headers
       @cookies = Jar.new
       @auth = nil
@@ -19,8 +19,12 @@ module Requests
       @timeout = nil
       @retries = 0
       @backoff_factor = 0
+      @status_forcelist = []
+      @trust_env = true
       @hooks = { response: [] }
-      @adapters = { 'https://' => HTTPAdapter.new, 'http://' => HTTPAdapter.new }
+      @adapters = { 'https://' => http2 ? Http2Adapter.new : HTTPAdapter.new, 'http://' => HTTPAdapter.new }
+      @default_adapters = @adapters.values.dup
+      @http2_adapter = http2 ? @adapters['https://'] : nil
     end
     def hooks
       @hooks
@@ -29,6 +33,12 @@ module Requests
       @adapters[prefix] = adapter
       self
     end
+    def http2!
+      @http2_adapter ||= Http2Adapter.new
+      @adapters['https://'] = @http2_adapter
+      @default_adapters = @adapters.values.dup
+      self
+    end
     def get(url, **kw); request('GET', url, **kw); end
     def post(url, **kw); request('POST', url, **kw); end
     def put(url, **kw); request('PUT', url, **kw); end
@@ -41,12 +51,44 @@ module Requests
     end
     def get!(url, **kw); get(url, **kw).raise_for_status; end
     def post!(url, **kw); post(url, **kw).raise_for_status; end
+    def put!(url, **kw); put(url, **kw).raise_for_status; end
+    def patch!(url, **kw); patch(url, **kw).raise_for_status; end
+    def delete!(url, **kw); delete(url, **kw).raise_for_status; end
+    def head!(url, **kw); head(url, **kw).raise_for_status; end
+    def options!(url, **kw); options(url, **kw).raise_for_status; end
     def close
+      @adapters.values.uniq.each { |a| a.close_pool if a.respond_to?(:close_pool) }
       true
     end
+    def download(url, to:, chunk_size: 65536, resume: false, progress: nil, headers: nil, params: nil,
+                 timeout: nil, proxies: nil, verify: nil, cert: nil, auth: nil, http2: nil)
+      validate_url!(url)
+      full_url = build_url(url, merge_hash(@params, params))
+      hdrs = CIHash.new(@headers.to_h)
+      hdrs.merge!(headers)
+      mode = 'wb'
+      if resume && File.exist?(to) && File.size(to) > 0
+        hdrs['Range'] = "bytes=#{File.size(to)}-"
+        mode = 'ab'
+      end
+      adapter = pick_adapter(full_url, http2)
+      adapter.max_retries = @retries
+      adapter.backoff_factor = @backoff_factor
+      use_auth = auth || @auth
+      use_auth.call(hdrs) if use_auth.respond_to?(:call) && !use_auth.is_a?(DigestAuth)
+      File.open(to, mode) do |f|
+        net_resp = adapter.stream_to('GET', full_url, hdrs, nil, timeout || @timeout, effective_proxies(full_url, proxies),
+                                      verify.nil? ? @verify : verify, cert || @cert, nil, f,
+                                      chunk_size: chunk_size, progress: progress)
+        unless [200, 206].include?(net_resp.code.to_i)
+          raise Requests::HTTPError, "download failed: #{net_resp.code} for #{full_url}"
+        end
+      end
+      to
+    end
     def request(method, url, params: nil, data: nil, json: nil, headers: nil, cookies: nil, files: nil,
                 auth: nil, timeout: nil, allow_redirects: true, proxies: nil, verify: nil, stream: false,
-                cert: nil, hooks: nil, retries: nil)
+                cert: nil, hooks: nil, retries: nil, http2: nil)
       validate_url!(url)
       use_auth = auth || @auth
       merged_params = merge_hash(@params, params)
@@ -62,23 +104,27 @@ module Requests
         hdrs.merge!(headers)
         hdrs['Content-Type'] = content_type if content_type && !hdrs.key?('content-type')
         hdrs.delete('Authorization') if redirs > 0 && URI.parse(cur_url).host != original_host
-        jar = merge_jar(@cookies, cookies)
-        hdrs['Cookie'] = jar.to_header unless jar.empty?
-        adapter = adapter_for(cur_url)
-        adapter.max_retries = retries.nil? ? @retries : retries
-        adapter.backoff_factor = @backoff_factor
+        ck = @cookies.to_header(cur_url)
+        ck = [ck, cookies.map { |k, v| "#{k}=#{v}" }.join('; ')].reject { |x| x.nil? || x.empty? }.join('; ') if cookies
+        hdrs['Cookie'] = ck unless ck.empty?
+        adapter = pick_adapter(cur_url, http2)
+        adapter.max_retries = retries unless retries.nil?
+        adapter.max_retries = @retries if @retries.to_i > 0 && retries.nil? && default_adapter?(adapter)
+        adapter.backoff_factor = @backoff_factor if @backoff_factor.to_f > 0 && default_adapter?(adapter)
+        adapter.status_forcelist = @status_forcelist if !@status_forcelist.empty? && adapter.respond_to?(:status_forcelist=) && default_adapter?(adapter)
+        eff_proxies = effective_proxies(cur_url, proxies)
         t0 = Time.now
-        net_resp = adapter.send_once(cur_method, cur_url, hdrs, body, timeout || @timeout,proxies || @proxies, verify.nil? ? @verify : verify,cert || @cert, use_auth)
+        net_resp = adapter.send_once(cur_method, cur_url, hdrs, body, timeout || @timeout, eff_proxies, verify.nil? ? @verify : verify,cert || @cert, use_auth)
         elapsed = Time.now - t0
         resp = to_response(adapter, net_resp, cur_url, elapsed, cur_method, hdrs, body)
-        @cookies.update(net_resp)
+        @cookies.update(net_resp, cur_url)
         resp.cookies = @cookies
         if use_auth.is_a?(DigestAuth) && resp.status_code == 401 && redirs.zero? && hist.empty?
           use_auth.call(hdrs, meth: cur_method, url: cur_url, prev_resp: resp)
           t1 = Time.now
-          net_resp = adapter.send_once(cur_method, cur_url, hdrs, body, timeout || @timeout,proxies || @proxies, verify.nil? ? @verify : verify,cert || @cert, nil)
+          net_resp = adapter.send_once(cur_method, cur_url, hdrs, body, timeout || @timeout, eff_proxies, verify.nil? ? @verify : verify,cert || @cert, nil)
           resp = to_response(adapter, net_resp, cur_url, Time.now - t1, cur_method, hdrs, body)
-          @cookies.update(net_resp)
+          @cookies.update(net_resp, cur_url)
           resp.cookies = @cookies
         end
         hist << resp
@@ -107,21 +153,31 @@ module Requests
       raise Requests::MissingSchema, "invalid url, no scheme: #{url}" unless s.include?('://')
       raise Requests::InvalidSchema, "unsupported scheme: #{url}" unless s =~ %r{\Ahttps?://}
     end
+    def default_adapter?(a)
+      @default_adapters.include?(a)
+    end
     def adapter_for(url)
       match = @adapters.keys.select { |prefix| url.start_with?(prefix) }.max_by(&:length)
       match ? @adapters[match] : @adapters['https://']
     end
+    def pick_adapter(url, http2)
+      return adapter_for(url) unless http2 == true
+      unless @http2_adapter
+        @http2_adapter = Http2Adapter.new
+        @default_adapters << @http2_adapter
+      end
+      @http2_adapter
+    end
+    def effective_proxies(url, proxies)
+      p = proxies || @proxies
+      return p if p && !p.empty?
+      @trust_env ? Requests::Utils.env_proxies(url) : (p || {})
+    end
     def merge_hash(a, b)
       return b unless a
       return a unless b
       a.merge(b)
     end
-    def merge_jar(session_jar, extra)
-      j = Jar.new
-      session_jar.each { |k, v| j[k] = v }
-      extra&.each { |k, v| j[k] = v }
-      j
-    end
     def build_url(base, params)
       return base if params.nil? || (params.respond_to?(:empty?) && params.empty?)
       qs = build_qs(params)

data/lib/requests/utils.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'uri'
+
 module Requests
   module Utils
     module_function
@@ -51,5 +53,21 @@ module Requests
     def looks_like_html?(str)
       !!(str.to_s.lstrip =~ /\A(<!DOCTYPE html|<html)/i)
     end
+    def env_proxies(url)
+      host = URI.parse(url).host.to_s
+      no_proxy = ENV['NO_PROXY'] || ENV['no_proxy']
+      if no_proxy
+        list = no_proxy.split(',').map { |s| s.strip.downcase }.reject(&:empty?)
+        return {} if list.include?('*') || list.any? { |p| host.downcase == p || host.downcase.end_with?(".#{p.sub(/\A\./, '')}") }
+      end
+      out = {}
+      http = ENV['HTTP_PROXY'] || ENV['http_proxy']
+      https = ENV['HTTPS_PROXY'] || ENV['https_proxy']
+      out['http'] = http if http && !http.empty?
+      out['https'] = https if https && !https.empty?
+      out
+    rescue URI::InvalidURIError
+      {}
+    end
   end
 end

data/lib/requests/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Requests
-  VERSION = '1.0.2'
+  VERSION = '1.0.4'
 end

data/lib/requests.rb

@@ -15,5 +15,7 @@ require_relative 'requests/auth'
 require_relative 'requests/utils'
 require_relative 'requests/models'
 require_relative 'requests/adapters'
+require_relative 'requests/hpack'
+require_relative 'requests/http2'
 require_relative 'requests/sessions'
 require_relative 'requests/api'