httparty 0.16.2 → 0.22.0

data/Gemfile

@@ -3,6 +3,7 @@ gemspec
 
 gem 'rake'
 gem 'mongrel',  '1.2.0.pre2'
+gem 'json'
 
 group :development do
   gem 'guard'
@@ -11,9 +12,15 @@ group :development do
 end
 
 group :test do
+  gem 'rexml'
   gem 'rspec',    '~> 3.4'
   gem 'simplecov', require: false
   gem 'aruba'
   gem 'cucumber', '~> 2.3'
   gem 'webmock'
+  gem 'addressable'
+end
+
+group :development, :test do
+  gem 'pry'
 end

data/Guardfile

@@ -1,7 +1,8 @@
 rspec_options = {
-  version:        1,
   all_after_pass: false,
-  all_on_start:   false
+  all_on_start: false,
+  failed_mode: :keep,
+  cmd: 'bundle exec rspec',
 }
 
 guard 'rspec', rspec_options do

data/README.md

@@ -1,5 +1,7 @@
 # httparty
 
+[![CI](https://github.com/jnunemaker/httparty/actions/workflows/ci.yml/badge.svg)](https://github.com/jnunemaker/httparty/actions/workflows/ci.yml)
+
 Makes http fun again!  Ain't no party like a httparty, because a httparty don't stop.
 
 ## Install
@@ -10,7 +12,7 @@ gem install httparty
 
 ## Requirements
 
-* Ruby 2.0.0 or higher
+* Ruby 2.3.0 or higher
 * multi_xml
 * You like to party!
 
@@ -46,7 +48,6 @@ puts stack_exchange.users
 ```
 
 See the [examples directory](http://github.com/jnunemaker/httparty/tree/master/examples) for even more goodies.
-
 ## Command Line Interface
 
 httparty also includes the executable `httparty` which can be
@@ -63,9 +64,8 @@ httparty "https://api.stackexchange.com/2.2/questions?site=stackoverflow"
 ## Help and Docs
 
 * [Docs](https://github.com/jnunemaker/httparty/tree/master/docs)
-* https://groups.google.com/forum/#!forum/httparty-gem
-* http://rdoc.info/projects/jnunemaker/httparty
-* http://stackoverflow.com/questions/tagged/httparty
+* https://github.com/jnunemaker/httparty/discussions
+* https://www.rubydoc.info/github/jnunemaker/httparty
 
 ## Contributing
 

data/docs/README.md

@@ -9,11 +9,25 @@ Makes http fun again!
 ## Parsing JSON
 If the response Content Type is `application/json`, HTTParty will parse the response and return Ruby objects such as a hash or array. The default behavior for parsing JSON will return keys as strings. This can be supressed with the `format` option. To get hash keys as symbols:
 
-```
+```ruby
 response = HTTParty.get('http://example.com', format: :plain)
 JSON.parse response, symbolize_names: true
 ```
 
+## Posting JSON
+When using Content Type `application/json` with `POST`, `PUT` or `PATCH` requests, the body should be a string of valid JSON:
+
+```ruby
+# With written JSON
+HTTParty.post('http://example.com', body: "{\"foo\":\"bar\"}", headers: { 'Content-Type' => 'application/json' })
+
+# Using JSON.generate
+HTTParty.post('http://example.com', body: JSON.generate({ foo: 'bar' }), headers: { 'Content-Type' => 'application/json' })
+
+# Using object.to_json
+HTTParty.post('http://example.com', body: { foo: 'bar' }.to_json, headers: { 'Content-Type' => 'application/json' })
+```
+
 ## Working with SSL
 
 You can use this guide to work with SSL certificates.
@@ -70,7 +84,7 @@ class Client
 end
 ```
 
-You can also include this options with the call:
+You can also include all of these options with the call:
 
 ```ruby
 class Client
@@ -79,14 +93,14 @@ class Client
   base_uri "https://example.com"
 
   def self.fetch
-    get("/resources", pem: (File.read("#{File.expand_path('.')}/path/to/certs/cert.pem"), "123456")
+    get("/resources", pem: File.read("#{File.expand_path('.')}/path/to/certs/cert.pem"), pem_password: "123456")
   end
 end
 ```
 
 ### Avoid SSL verification
 
-In some cases you may want to skip SSL verification, because the entity that issue the certificate is not a valid one, but you still want to work with it. You can achieve this through:
+In some cases you may want to skip SSL verification, because the entity that issued the certificate is not a valid one, but you still want to work with it. You can achieve this through:
 
 ```ruby
 # Skips SSL certificate verification
@@ -103,4 +117,75 @@ class Client
     # get("resources", verify_peer: false)
   end
 end
-```
+```
+
+### HTTP Compression
+
+The `Accept-Encoding` request header and `Content-Encoding` response header
+are used to control compression (gzip, etc.) over the wire. Refer to
+[RFC-2616](https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html) for details.
+(For clarity: these headers are **not** used for character encoding i.e. `utf-8`
+which is specified in the `Accept` and `Content-Type` headers.)
+
+Unless you have specific requirements otherwise, we recommend to **not** set
+set the `Accept-Encoding` header on HTTParty requests. In this case, `Net::HTTP`
+will set a sensible default compression scheme and automatically decompress the response.
+
+If you explicitly set `Accept-Encoding`, there be dragons:
+
+* If the HTTP response `Content-Encoding` received on the wire is `gzip` or `deflate`,
+  `Net::HTTP` will automatically decompress it, and will omit `Content-Encoding`
+  from your `HTTParty::Response` headers.
+
+* For the following encodings, HTTParty will automatically decompress them if you include
+  the required gem into your project. Similar to above, if decompression succeeds,
+  `Content-Encoding` will be omitted from your `HTTParty::Response` headers.
+  **Warning:** Support for these encodings is experimental and not fully battle-tested.
+
+  | Content-Encoding | Required Gem |
+  | --- | --- |
+  | `br` (Brotli)      | [brotli](https://rubygems.org/gems/brotli) |
+  | `compress` (LZW)   | [ruby-lzws](https://rubygems.org/gems/ruby-lzws) |
+  | `zstd` (Zstandard) | [zstd-ruby](https://rubygems.org/gems/zstd-ruby) |
+
+* For other encodings, `HTTParty::Response#body` will return the raw uncompressed byte string,
+  and you'll need to inspect the `Content-Encoding` response header and decompress it yourself.
+  In this case, `HTTParty::Response#parsed_response` will be `nil`.
+
+* Lastly, you may use the `skip_decompression` option to disable all automatic decompression
+  and always get `HTTParty::Response#body` in its raw form along with the `Content-Encoding` header.
+
+```ruby
+# Accept-Encoding=gzip,deflate can be safely assumed to be auto-decompressed
+
+res = HTTParty.get('https://example.com/test.json', headers: { 'Accept-Encoding' => 'gzip,deflate,identity' })
+JSON.parse(res.body) # safe
+
+
+# Accept-Encoding=br,compress requires third-party gems
+
+require 'brotli'
+require 'lzws'
+require 'zstd-ruby'
+res = HTTParty.get('https://example.com/test.json', headers: { 'Accept-Encoding' => 'br,compress,zstd' })
+JSON.parse(res.body)
+
+
+# Accept-Encoding=* may return unhandled Content-Encoding
+
+res = HTTParty.get('https://example.com/test.json', headers: { 'Accept-Encoding' => '*' })
+encoding = res.headers['Content-Encoding']
+if encoding
+JSON.parse(your_decompression_handling(res.body, encoding))
+else
+# Content-Encoding not present implies decompressed
+JSON.parse(res.body)
+end
+
+
+# Gimme the raw data!
+
+res = HTTParty.get('https://example.com/test.json', skip_decompression: true)
+encoding = res.headers['Content-Encoding']
+JSON.parse(your_decompression_handling(res.body, encoding))
+```

data/examples/README.md

@@ -13,22 +13,22 @@
     * Creates a custom parser for XML using crack gem
     * Uses `get` request
 
-* [Create HTML Nokogiri parser](nokogiri_html_parser.rb)   
+* [Create HTML Nokogiri parser](nokogiri_html_parser.rb)
     * Adds Html as a format
     * passed the body of request to Nokogiri
-    
+
 * [More Custom Parsers](custom_parsers.rb)
   * Create an additional parser for atom or make it the ONLY parser
-  
+
 * [Basic Auth, Delicious](delicious.rb)
   * Basic Auth, shows how to merge those into options
   * Uses `get` requests
-  
+
 * [Passing Headers, User Agent](headers_and_user_agents.rb)
   * Use the class method of Httparty
   * Pass the User-Agent in the headers
   * Uses `get` requests
-  
+
 * [Basic Post Request](basic.rb)
     * Httparty included into poro class
     * Uses `post` requests
@@ -36,7 +36,7 @@
 * [Access Rubyurl Shortener](rubyurl.rb)
   * Httparty included into poro class
   * Uses `post` requests
-  
+
 * [Add a custom log file](logging.rb)
   * create a log file and have httparty log requests
 
@@ -44,23 +44,23 @@
   * Httparty included into poro class
   * Creates methods for different endpoints
   * Uses `get` requests
-  
+
 * [Accessing Tripit](tripit_sign_in.rb)
   * Httparty included into poro class
   * Example of using `debug_output` to see headers/urls passed
   * Getting and using Cookies
   * Uses `get` requests
-  
+
 * [Accessing Twitter](twitter.rb)
   * Httparty included into poro class
   * Basic Auth
-  * Loads settings from a config file 
+  * Loads settings from a config file
   * Uses `get` requests
   * Uses `post` requests
-  
+
 * [Accessing WhoIsMyRep](whoismyrep.rb)
   * Httparty included into poro class
-  * Uses `get` requests     
+  * Uses `get` requests
   * Two ways to pass params to get, inline on the url or in query hash
 
 * [Rescue Json Error](rescue_json.rb)
@@ -70,3 +70,20 @@
   * Uses `get` requests
   * Uses `stream_body` mode
   * Download file without using the memory
+
+* [Microsoft graph](microsoft_graph.rb)
+  * Basic Auth
+  * Uses `post` requests
+  * Uses multipart
+
+* [Multipart](multipart.rb)
+  * Multipart data upload _(with and without file)_
+
+* [Uploading File](body_stream.rb)
+  * Uses `body_stream` to upload file
+
+* [Accessing x509 Peer Certificate](peer_cert.rb)
+  * Provides access to the server's TLS certificate
+
+* [Accessing IDNs](idn.rb)
+  * Uses a `get` request with an International domain names, which are Urls with emojis and non-ASCII characters such as accented letters.

data/examples/aaws.rb

@@ -1,5 +1,7 @@
 require 'rubygems'
 require 'active_support'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/string'
 
 dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
 require File.join(dir, 'httparty')
@@ -13,14 +15,16 @@ module AAWS
     default_params Service: 'AWSECommerceService', Operation: 'ItemSearch', SearchIndex: 'Books'
 
     def initialize(key)
-      self.class.default_params AWSAccessKeyId: key
+      @auth = { AWSAccessKeyId: key }
     end
 
     def search(options = {})
       raise ArgumentError, 'You must search for something' if options[:query].blank?
 
       # amazon uses nasty camelized query params
-      options[:query] = options[:query].inject({}) { |h, q| h[q[0].to_s.camelize] = q[1]; h }
+      options[:query] = options[:query]
+        .reverse_merge(@auth)
+        .transform_keys { |k| k.to_s.camelize }
 
       # make a request and return the items (NOTE: this doesn't handle errors at this point)
       self.class.get('/onca/xml', options)['ItemSearchResponse']['Items']

data/examples/body_stream.rb

@@ -0,0 +1,14 @@
+# To upload file to a server use :body_stream
+
+HTTParty.put(
+  'http://localhost:3000/train',
+  body_stream: File.open('sample_configs/config_train_server_md.yml', 'r')
+)
+
+
+# Actually, it works with any IO object
+
+HTTParty.put(
+  'http://localhost:3000/train',
+  body_stream: StringIO.new('foo')
+)

data/examples/idn.rb

@@ -0,0 +1,10 @@
+dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+require File.join(dir, 'httparty')
+require 'pp'
+
+class Idn
+    include HTTParty
+    uri_adapter Addressable::URI
+end
+
+pp Idn.get("https://i❤️.ws/emojidomain/💎?format=json")

data/examples/microsoft_graph.rb

@@ -0,0 +1,52 @@
+require 'httparty'
+
+class MicrosoftGraph
+  MS_BASE_URL = "https://login.microsoftonline.com".freeze
+  TOKEN_REQUEST_PATH = "oauth2/v2.0/token".freeze
+
+  def initialize(tenant_id)
+    @tenant_id = tenant_id
+  end
+
+  # Make a request to the Microsoft Graph API, for instance https://graph.microsoft.com/v1.0/users
+  def request(url)
+    return false unless (token = bearer_token)
+
+    response = HTTParty.get(
+      url,
+      headers: {
+        Authorization: "Bearer #{token}"
+      }
+    )
+
+    return false unless response.code == 200
+
+    return JSON.parse(response.body)
+  end
+
+  private
+
+  # A post to the Microsoft Graph to get a bearer token for the specified tenant.  In this example
+  # our Rails application has already been given permission to request these tokens by the admin of
+  # the specified tenant_id.
+  #
+  # See here for more information https://developer.microsoft.com/en-us/graph/docs/concepts/auth_v2_service
+  #
+  # This request also makes use of the multipart/form-data post body.
+  def bearer_token
+    response = HTTParty.post(
+      "#{MS_BASE_URL}/#{@tenant_id}/#{TOKEN_REQUEST_PATH}",
+      multipart: true,
+      body: {
+        client_id: Rails.application.credentials[Rails.env.to_sym][:microsoft_client_id],
+        client_secret: Rails.application.credentials[Rails.env.to_sym][:microsoft_client_secret],
+        scope: 'https://graph.microsoft.com/.default',
+        grant_type: 'client_credentials'
+      }
+    )
+
+    return false unless response.code == 200
+
+    JSON.parse(response.body)['access_token']
+  end
+end

data/examples/multipart.rb

@@ -0,0 +1,22 @@
+# If you are uploading file in params, multipart will used as content-type automatically
+
+HTTParty.post(
+  'http://localhost:3000/user',
+  body: {
+    name: 'Foo Bar',
+    email: 'example@email.com',
+    avatar: File.open('/full/path/to/avatar.jpg')
+  }
+)
+
+
+# However, you can force it yourself
+
+HTTParty.post(
+  'http://localhost:3000/user',
+  multipart: true,
+  body: {
+    name: 'Foo Bar',
+    email: 'example@email.com'
+  }
+)

data/examples/peer_cert.rb

@@ -0,0 +1,9 @@
+dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+require File.join(dir, 'httparty')
+
+peer_cert = nil
+HTTParty.get("https://www.example.com") do |fragment|
+  peer_cert ||= fragment.connection.peer_cert
+end
+
+puts "The server's certificate expires #{peer_cert.not_after}"

data/examples/stream_download.rb

@@ -9,8 +9,14 @@ url = "https://cdn.kernel.org/pub/linux/kernel/v4.x/#{filename}"
 
 File.open(filename, "w") do |file|
   response = HTTParty.get(url, stream_body: true) do |fragment|
-    print "."
-    file.write(fragment)
+    if [301, 302].include?(fragment.code)
+      print "skip writing for redirect"
+    elsif fragment.code == 200
+      print "."
+      file.write(fragment)
+    else
+      raise StandardError, "Non-success status code while streaming #{fragment.code}"
+    end
   end
 end
 puts

data/httparty.gemspec

@@ -9,13 +9,15 @@ Gem::Specification.new do |s|
   s.licenses    = ['MIT']
   s.authors     = ["John Nunemaker", "Sandro Turriate"]
   s.email       = ["nunemaker@gmail.com"]
-  s.homepage    = "http://jnunemaker.github.com/httparty"
+  s.homepage    = "https://github.com/jnunemaker/httparty"
   s.summary     = 'Makes http fun! Also, makes consuming restful web services dead easy.'
   s.description = 'Makes http fun! Also, makes consuming restful web services dead easy.'
 
-  s.required_ruby_version     = '>= 2.0.0'
+  s.required_ruby_version     = '>= 2.7.0'
 
+  s.add_dependency 'csv'
   s.add_dependency 'multi_xml', ">= 0.5.2"
+  s.add_dependency 'mini_mime', ">= 1.0.0"
 
   # If this line is removed, all hard partying will cease.
   s.post_install_message = "When you HTTParty, you must party hard!"
@@ -24,7 +26,6 @@ Gem::Specification.new do |s|
   test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
 
   s.files         = all_files - test_files
-  s.test_files    = test_files
   s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_paths = ["lib"]
 end

data/lib/httparty/connection_adapter.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 module HTTParty
   # Default connection adapter that returns a new Net::HTTP each time
   #
   # == Custom Connection Factories
   #
   # If you like to implement your own connection adapter, subclassing
-  # HTTPParty::ConnectionAdapter will make it easier. Just override
+  # HTTParty::ConnectionAdapter will make it easier. Just override
   # the #connection method. The uri and options attributes will have
   # all the info you need to construct your http connection. Whatever
   # you return from your connection method needs to adhere to the
@@ -38,12 +40,13 @@ module HTTParty
   # in the #options attribute. It is up to you to interpret them within your
   # connection adapter. Take a look at the implementation of
   # HTTParty::ConnectionAdapter#connection for examples of how they are used.
-  # The keys used in options are 
+  # The keys used in options are
   # * :+timeout+: timeout in seconds
   # * :+open_timeout+: http connection open_timeout in seconds, overrides timeout if set
   # * :+read_timeout+: http connection read_timeout in seconds, overrides timeout if set
+  # * :+write_timeout+: http connection write_timeout in seconds, overrides timeout if set (Ruby >= 2.6.0 required)
   # * :+debug_output+: see HTTParty::ClassMethods.debug_output.
-  # * :+cert_store+: contains certificate data. see method 'attach_ssl_certificates' 
+  # * :+cert_store+: contains certificate data. see method 'attach_ssl_certificates'
   # * :+pem+: contains pem client certificate data. see method 'attach_ssl_certificates'
   # * :+p12+: contains PKCS12 client client certificate data.  see method 'attach_ssl_certificates'
   # * :+verify+: verify the server’s certificate against the ca certificate.
@@ -77,6 +80,12 @@ module HTTParty
       new(uri, options).connection
     end
 
+    def self.default_cert_store
+      @default_cert_store ||= OpenSSL::X509::Store.new.tap do |cert_store|
+        cert_store.set_default_paths
+      end
+    end
+
     attr_reader :uri, :options
 
     def initialize(uri, options = {})
@@ -91,7 +100,14 @@ module HTTParty
       host = clean_host(uri.host)
       port = uri.port || (uri.scheme == 'https' ? 443 : 80)
       if options.key?(:http_proxyaddr)
-        http = Net::HTTP.new(host, port, options[:http_proxyaddr], options[:http_proxyport], options[:http_proxyuser], options[:http_proxypass])
+        http = Net::HTTP.new(
+          host,
+          port,
+          options[:http_proxyaddr],
+          options[:http_proxyport],
+          options[:http_proxyuser],
+          options[:http_proxypass]
+        )
       else
         http = Net::HTTP.new(host, port)
       end
@@ -100,19 +116,28 @@ module HTTParty
 
       attach_ssl_certificates(http, options)
 
-      if options[:timeout] && (options[:timeout].is_a?(Integer) || options[:timeout].is_a?(Float))
+      if add_timeout?(options[:timeout])
         http.open_timeout = options[:timeout]
         http.read_timeout = options[:timeout]
+        http.write_timeout = options[:timeout]
       end
 
-      if options[:read_timeout] && (options[:read_timeout].is_a?(Integer) || options[:read_timeout].is_a?(Float))
+      if add_timeout?(options[:read_timeout])
         http.read_timeout = options[:read_timeout]
       end
 
-      if options[:open_timeout] && (options[:open_timeout].is_a?(Integer) || options[:open_timeout].is_a?(Float))
+      if add_timeout?(options[:open_timeout])
         http.open_timeout = options[:open_timeout]
       end
 
+      if add_timeout?(options[:write_timeout])
+        http.write_timeout = options[:write_timeout]
+      end
+
+      if add_max_retries?(options[:max_retries])
+        http.max_retries = options[:max_retries]
+      end
+
       if options[:debug_output]
         http.set_debug_output(options[:debug_output])
       end
@@ -125,19 +150,11 @@ module HTTParty
       #
       # @see https://bugs.ruby-lang.org/issues/6617
       if options[:local_host]
-        if RUBY_VERSION >= "2.0.0"
-          http.local_host = options[:local_host]
-        else
-          Kernel.warn("Warning: option :local_host requires Ruby version 2.0 or later")
-        end
+        http.local_host = options[:local_host]
       end
 
       if options[:local_port]
-        if RUBY_VERSION >= "2.0.0"
-          http.local_port = options[:local_port]
-        else
-          Kernel.warn("Warning: option :local_port requires Ruby version 2.0 or later")
-        end
+        http.local_port = options[:local_port]
       end
 
       http
@@ -145,6 +162,14 @@ module HTTParty
 
     private
 
+    def add_timeout?(timeout)
+      timeout && (timeout.is_a?(Integer) || timeout.is_a?(Float))
+    end
+
+    def add_max_retries?(max_retries)
+      max_retries && max_retries.is_a?(Integer) && max_retries >= 0
+    end
+
     def clean_host(host)
       strip_ipv6_brackets(host)
     end
@@ -169,8 +194,7 @@ module HTTParty
             http.cert_store = options[:cert_store]
           else
             # Use the default cert store by default, i.e. system ca certs
-            http.cert_store = OpenSSL::X509::Store.new
-            http.cert_store.set_default_paths
+            http.cert_store = self.class.default_cert_store
           end
         else
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -180,7 +204,7 @@ module HTTParty
         # Note: options[:pem] must contain the content of a PEM file having the private key appended
         if options[:pem]
           http.cert = OpenSSL::X509::Certificate.new(options[:pem])
-          http.key = OpenSSL::PKey::RSA.new(options[:pem], options[:pem_password])
+          http.key = OpenSSL::PKey.read(options[:pem], options[:pem_password])
           http.verify_mode = verify_ssl_certificate? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         end
 

data/lib/httparty/cookie_hash.rb

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 class HTTParty::CookieHash < Hash #:nodoc:
-  CLIENT_COOKIES = %w(path expires domain path secure httponly)
+  CLIENT_COOKIES = %w(path expires domain path secure httponly samesite)
 
-  def add_cookies(value)
-    case value
+  def add_cookies(data)
+    case data
     when Hash
-      merge!(value)
+      merge!(data)
     when String
-      value.split('; ').each do |cookie|
-        array = cookie.split('=', 2)
-        self[array[0].to_sym] = array[1]
+      data.split('; ').each do |cookie|
+        key, value = cookie.split('=', 2)
+        self[key.to_sym] = value if key
       end
     else
       raise "add_cookies only takes a Hash or a String"
@@ -16,6 +18,6 @@ class HTTParty::CookieHash < Hash #:nodoc:
   end
 
   def to_cookie_string
-    select { |k, v| !CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join("; ")
+    select { |k, v| !CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join('; ')
   end
 end