httparty-responsibly 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +7 -0
  2. data/.editorconfig +18 -0
  3. data/.gitignore +13 -0
  4. data/.rubocop.yml +92 -0
  5. data/.rubocop_todo.yml +124 -0
  6. data/.simplecov +1 -0
  7. data/.travis.yml +11 -0
  8. data/CONTRIBUTING.md +23 -0
  9. data/Changelog.md +509 -0
  10. data/Gemfile +24 -0
  11. data/Guardfile +16 -0
  12. data/MIT-LICENSE +20 -0
  13. data/README.md +78 -0
  14. data/Rakefile +10 -0
  15. data/bin/httparty +123 -0
  16. data/cucumber.yml +1 -0
  17. data/docs/README.md +106 -0
  18. data/examples/README.md +86 -0
  19. data/examples/aaws.rb +32 -0
  20. data/examples/basic.rb +28 -0
  21. data/examples/body_stream.rb +14 -0
  22. data/examples/crack.rb +19 -0
  23. data/examples/custom_parsers.rb +68 -0
  24. data/examples/delicious.rb +37 -0
  25. data/examples/google.rb +16 -0
  26. data/examples/headers_and_user_agents.rb +10 -0
  27. data/examples/logging.rb +36 -0
  28. data/examples/microsoft_graph.rb +52 -0
  29. data/examples/multipart.rb +22 -0
  30. data/examples/nokogiri_html_parser.rb +19 -0
  31. data/examples/peer_cert.rb +9 -0
  32. data/examples/rescue_json.rb +17 -0
  33. data/examples/rubyurl.rb +14 -0
  34. data/examples/stackexchange.rb +24 -0
  35. data/examples/stream_download.rb +26 -0
  36. data/examples/tripit_sign_in.rb +44 -0
  37. data/examples/twitter.rb +31 -0
  38. data/examples/whoismyrep.rb +10 -0
  39. data/httparty-responsibly.gemspec +27 -0
  40. data/lib/httparty.rb +668 -0
  41. data/lib/httparty/connection_adapter.rb +254 -0
  42. data/lib/httparty/cookie_hash.rb +21 -0
  43. data/lib/httparty/exceptions.rb +33 -0
  44. data/lib/httparty/hash_conversions.rb +69 -0
  45. data/lib/httparty/headers_processor.rb +30 -0
  46. data/lib/httparty/logger/apache_formatter.rb +45 -0
  47. data/lib/httparty/logger/curl_formatter.rb +91 -0
  48. data/lib/httparty/logger/logger.rb +28 -0
  49. data/lib/httparty/logger/logstash_formatter.rb +59 -0
  50. data/lib/httparty/module_inheritable_attributes.rb +56 -0
  51. data/lib/httparty/net_digest_auth.rb +136 -0
  52. data/lib/httparty/parser.rb +150 -0
  53. data/lib/httparty/request.rb +386 -0
  54. data/lib/httparty/request/body.rb +84 -0
  55. data/lib/httparty/request/multipart_boundary.rb +11 -0
  56. data/lib/httparty/response.rb +140 -0
  57. data/lib/httparty/response/headers.rb +33 -0
  58. data/lib/httparty/response_fragment.rb +19 -0
  59. data/lib/httparty/text_encoder.rb +70 -0
  60. data/lib/httparty/utils.rb +11 -0
  61. data/lib/httparty/version.rb +3 -0
  62. data/script/release +42 -0
  63. data/website/css/common.css +47 -0
  64. data/website/index.html +73 -0
  65. metadata +138 -0
@@ -0,0 +1,91 @@
1
+ module HTTParty
2
+ module Logger
3
+ class CurlFormatter #:nodoc:
4
+ TAG_NAME = HTTParty.name
5
+ OUT = '>'.freeze
6
+ IN = '<'.freeze
7
+
8
+ attr_accessor :level, :logger
9
+
10
+ def initialize(logger, level)
11
+ @logger = logger
12
+ @level = level.to_sym
13
+ @messages = []
14
+ end
15
+
16
+ def format(request, response)
17
+ @request = request
18
+ @response = response
19
+
20
+ log_request
21
+ log_response
22
+
23
+ logger.public_send level, messages.join("\n")
24
+ end
25
+
26
+ private
27
+
28
+ attr_reader :request, :response
29
+ attr_accessor :messages
30
+
31
+ def log_request
32
+ log_url
33
+ log_headers
34
+ log_query
35
+ log OUT, request.raw_body if request.raw_body
36
+ log OUT
37
+ end
38
+
39
+ def log_response
40
+ log IN, "HTTP/#{response.http_version} #{response.code}"
41
+ log_response_headers
42
+ log IN, "\n#{response.body}"
43
+ log IN
44
+ end
45
+
46
+ def log_url
47
+ http_method = request.http_method.name.split("::").last.upcase
48
+ uri = if request.options[:base_uri]
49
+ request.options[:base_uri] + request.path.path
50
+ else
51
+ request.path.to_s
52
+ end
53
+
54
+ log OUT, "#{http_method} #{uri}"
55
+ end
56
+
57
+ def log_headers
58
+ return unless request.options[:headers] && request.options[:headers].size > 0
59
+
60
+ log OUT, 'Headers: '
61
+ log_hash request.options[:headers]
62
+ end
63
+
64
+ def log_query
65
+ return unless request.options[:query]
66
+
67
+ log OUT, 'Query: '
68
+ log_hash request.options[:query]
69
+ end
70
+
71
+ def log_response_headers
72
+ headers = response.respond_to?(:headers) ? response.headers : response
73
+ response.each_header do |response_header|
74
+ log IN, "#{response_header.capitalize}: #{headers[response_header]}"
75
+ end
76
+ end
77
+
78
+ def log_hash(hash)
79
+ hash.each { |k, v| log(OUT, "#{k}: #{v}") }
80
+ end
81
+
82
+ def log(direction, line = '')
83
+ messages << "[#{TAG_NAME}] [#{current_time}] #{direction} #{line}"
84
+ end
85
+
86
+ def current_time
87
+ Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
88
+ end
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,28 @@
1
+ require 'httparty/logger/apache_formatter'
2
+ require 'httparty/logger/curl_formatter'
3
+ require 'httparty/logger/logstash_formatter'
4
+
5
+ module HTTParty
6
+ module Logger
7
+ def self.formatters
8
+ @formatters ||= {
9
+ :curl => Logger::CurlFormatter,
10
+ :apache => Logger::ApacheFormatter,
11
+ :logstash => Logger::LogstashFormatter,
12
+ }
13
+ end
14
+
15
+ def self.add_formatter(name, formatter)
16
+ raise HTTParty::Error.new("Log Formatter with name #{name} already exists") if formatters.include?(name)
17
+ formatters.merge!(name.to_sym => formatter)
18
+ end
19
+
20
+ def self.build(logger, level, formatter)
21
+ level ||= :info
22
+ formatter ||= :apache
23
+
24
+ logger_klass = formatters[formatter] || Logger::ApacheFormatter
25
+ logger_klass.new(logger, level)
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,59 @@
1
+ module HTTParty
2
+ module Logger
3
+ class LogstashFormatter #:nodoc:
4
+ TAG_NAME = HTTParty.name
5
+
6
+ attr_accessor :level, :logger
7
+
8
+ def initialize(logger, level)
9
+ @logger = logger
10
+ @level = level.to_sym
11
+ end
12
+
13
+ def format(request, response)
14
+ @request = request
15
+ @response = response
16
+
17
+ logger.public_send level, logstash_message
18
+ end
19
+
20
+ private
21
+
22
+ attr_reader :request, :response
23
+
24
+ def logstash_message
25
+ {
26
+ '@timestamp' => current_time,
27
+ '@version' => 1,
28
+ 'content_length' => content_length || '-',
29
+ 'http_method' => http_method,
30
+ 'message' => message,
31
+ 'path' => path,
32
+ 'response_code' => response.code,
33
+ 'severity' => level,
34
+ 'tags' => [TAG_NAME],
35
+ }.to_json
36
+ end
37
+
38
+ def message
39
+ "[#{TAG_NAME}] #{response.code} \"#{http_method} #{path}\" #{content_length || '-'} "
40
+ end
41
+
42
+ def current_time
43
+ Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
44
+ end
45
+
46
+ def http_method
47
+ @http_method ||= request.http_method.name.split("::").last.upcase
48
+ end
49
+
50
+ def path
51
+ @path ||= request.path.to_s
52
+ end
53
+
54
+ def content_length
55
+ @content_length ||= response.respond_to?(:headers) ? response.headers['Content-Length'] : response['Content-Length']
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,56 @@
1
+ module HTTParty
2
+ module ModuleInheritableAttributes #:nodoc:
3
+ def self.included(base)
4
+ base.extend(ClassMethods)
5
+ end
6
+
7
+ # borrowed from Rails 3.2 ActiveSupport
8
+ def self.hash_deep_dup(hash)
9
+ duplicate = hash.dup
10
+
11
+ duplicate.each_pair do |key, value|
12
+ if value.is_a?(Hash)
13
+ duplicate[key] = hash_deep_dup(value)
14
+ elsif value.is_a?(Proc)
15
+ duplicate[key] = value.dup
16
+ else
17
+ duplicate[key] = value
18
+ end
19
+ end
20
+
21
+ duplicate
22
+ end
23
+
24
+ module ClassMethods #:nodoc:
25
+ def mattr_inheritable(*args)
26
+ @mattr_inheritable_attrs ||= [:mattr_inheritable_attrs]
27
+ @mattr_inheritable_attrs += args
28
+
29
+ args.each do |arg|
30
+ module_eval %(class << self; attr_accessor :#{arg} end)
31
+ end
32
+
33
+ @mattr_inheritable_attrs
34
+ end
35
+
36
+ def inherited(subclass)
37
+ super
38
+ @mattr_inheritable_attrs.each do |inheritable_attribute|
39
+ ivar = "@#{inheritable_attribute}"
40
+ subclass.instance_variable_set(ivar, instance_variable_get(ivar).clone)
41
+
42
+ if instance_variable_get(ivar).respond_to?(:merge)
43
+ method = <<-EOM
44
+ def self.#{inheritable_attribute}
45
+ duplicate = ModuleInheritableAttributes.hash_deep_dup(#{ivar})
46
+ #{ivar} = superclass.#{inheritable_attribute}.merge(duplicate)
47
+ end
48
+ EOM
49
+
50
+ subclass.class_eval method
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,136 @@
1
+ require 'digest/md5'
2
+ require 'net/http'
3
+
4
+ module Net
5
+ module HTTPHeader
6
+ def digest_auth(username, password, response)
7
+ authenticator = DigestAuthenticator.new(
8
+ username,
9
+ password,
10
+ @method,
11
+ @path,
12
+ response
13
+ )
14
+
15
+ authenticator.authorization_header.each do |v|
16
+ add_field('Authorization', v)
17
+ end
18
+
19
+ authenticator.cookie_header.each do |v|
20
+ add_field('Cookie', v)
21
+ end
22
+ end
23
+
24
+ class DigestAuthenticator
25
+ def initialize(username, password, method, path, response_header)
26
+ @username = username
27
+ @password = password
28
+ @method = method
29
+ @path = path
30
+ @response = parse(response_header)
31
+ @cookies = parse_cookies(response_header)
32
+ end
33
+
34
+ def authorization_header
35
+ @cnonce = md5(random)
36
+ header = [
37
+ %(Digest username="#{@username}"),
38
+ %(realm="#{@response['realm']}"),
39
+ %(nonce="#{@response['nonce']}"),
40
+ %(uri="#{@path}"),
41
+ %(response="#{request_digest}")
42
+ ]
43
+
44
+ header << %(algorithm="#{@response['algorithm']}") if algorithm_present?
45
+
46
+ if qop_present?
47
+ fields = [
48
+ %(cnonce="#{@cnonce}"),
49
+ %(qop="#{@response['qop']}"),
50
+ "nc=00000001"
51
+ ]
52
+ fields.each { |field| header << field }
53
+ end
54
+
55
+ header << %(opaque="#{@response['opaque']}") if opaque_present?
56
+ header
57
+ end
58
+
59
+ def cookie_header
60
+ @cookies
61
+ end
62
+
63
+ private
64
+
65
+ def parse(response_header)
66
+ header = response_header['www-authenticate']
67
+
68
+ header = header.gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
69
+
70
+ header =~ /Digest (.*)/
71
+ params = {}
72
+ if $1
73
+ non_quoted = $1.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
74
+ non_quoted.gsub(/(\w+)=([^,]*)/) { params[$1] = $2 }
75
+ end
76
+ params
77
+ end
78
+
79
+ def parse_cookies(response_header)
80
+ return [] unless response_header['Set-Cookie']
81
+
82
+ cookies = response_header['Set-Cookie'].split('; ')
83
+
84
+ cookies.reduce([]) do |ret, cookie|
85
+ ret << cookie
86
+ ret
87
+ end
88
+
89
+ cookies
90
+ end
91
+
92
+ def opaque_present?
93
+ @response.key?('opaque') && !@response['opaque'].empty?
94
+ end
95
+
96
+ def qop_present?
97
+ @response.key?('qop') && !@response['qop'].empty?
98
+ end
99
+
100
+ def random
101
+ format "%x", (Time.now.to_i + rand(65535))
102
+ end
103
+
104
+ def request_digest
105
+ a = [md5(a1), @response['nonce'], md5(a2)]
106
+ a.insert(2, "00000001", @cnonce, @response['qop']) if qop_present?
107
+ md5(a.join(":"))
108
+ end
109
+
110
+ def md5(str)
111
+ Digest::MD5.hexdigest(str)
112
+ end
113
+
114
+ def algorithm_present?
115
+ @response.key?('algorithm') && !@response['algorithm'].empty?
116
+ end
117
+
118
+ def use_md5_sess?
119
+ algorithm_present? && @response['algorithm'] == 'MD5-sess'
120
+ end
121
+
122
+ def a1
123
+ a1_user_realm_pwd = [@username, @response['realm'], @password].join(':')
124
+ if use_md5_sess?
125
+ [ md5(a1_user_realm_pwd), @response['nonce'], @cnonce ].join(':')
126
+ else
127
+ a1_user_realm_pwd
128
+ end
129
+ end
130
+
131
+ def a2
132
+ [@method, @path].join(":")
133
+ end
134
+ end
135
+ end
136
+ end
@@ -0,0 +1,150 @@
1
+ module HTTParty
2
+ # The default parser used by HTTParty, supports xml, json, html, csv and
3
+ # plain text.
4
+ #
5
+ # == Custom Parsers
6
+ #
7
+ # If you'd like to do your own custom parsing, subclassing HTTParty::Parser
8
+ # will make that process much easier. There are a few different ways you can
9
+ # utilize HTTParty::Parser as a superclass.
10
+ #
11
+ # @example Intercept the parsing for all formats
12
+ # class SimpleParser < HTTParty::Parser
13
+ # def parse
14
+ # perform_parsing
15
+ # end
16
+ # end
17
+ #
18
+ # @example Add the atom format and parsing method to the default parser
19
+ # class AtomParsingIncluded < HTTParty::Parser
20
+ # SupportedFormats.merge!(
21
+ # {"application/atom+xml" => :atom}
22
+ # )
23
+ #
24
+ # def atom
25
+ # perform_atom_parsing
26
+ # end
27
+ # end
28
+ #
29
+ # @example Only support the atom format
30
+ # class ParseOnlyAtom < HTTParty::Parser
31
+ # SupportedFormats = {"application/atom+xml" => :atom}
32
+ #
33
+ # def atom
34
+ # perform_atom_parsing
35
+ # end
36
+ # end
37
+ #
38
+ # @abstract Read the Custom Parsers section for more information.
39
+ class Parser
40
+ SupportedFormats = {
41
+ 'text/xml' => :xml,
42
+ 'application/xml' => :xml,
43
+ 'application/json' => :json,
44
+ 'application/vnd.api+json' => :json,
45
+ 'application/hal+json' => :json,
46
+ 'text/json' => :json,
47
+ 'application/javascript' => :plain,
48
+ 'text/javascript' => :plain,
49
+ 'text/html' => :html,
50
+ 'text/plain' => :plain,
51
+ 'text/csv' => :csv,
52
+ 'application/csv' => :csv,
53
+ 'text/comma-separated-values' => :csv
54
+ }
55
+
56
+ # The response body of the request
57
+ # @return [String]
58
+ attr_reader :body
59
+
60
+ # The intended parsing format for the request
61
+ # @return [Symbol] e.g. :json
62
+ attr_reader :format
63
+
64
+ # Instantiate the parser and call {#parse}.
65
+ # @param [String] body the response body
66
+ # @param [Symbol] format the response format
67
+ # @return parsed response
68
+ def self.call(body, format)
69
+ new(body, format).parse
70
+ end
71
+
72
+ # @return [Hash] the SupportedFormats hash
73
+ def self.formats
74
+ const_get(:SupportedFormats)
75
+ end
76
+
77
+ # @param [String] mimetype response MIME type
78
+ # @return [Symbol]
79
+ # @return [nil] mime type not supported
80
+ def self.format_from_mimetype(mimetype)
81
+ formats[formats.keys.detect {|k| mimetype.include?(k)}]
82
+ end
83
+
84
+ # @return [Array<Symbol>] list of supported formats
85
+ def self.supported_formats
86
+ formats.values.uniq
87
+ end
88
+
89
+ # @param [Symbol] format e.g. :json, :xml
90
+ # @return [Boolean]
91
+ def self.supports_format?(format)
92
+ supported_formats.include?(format)
93
+ end
94
+
95
+ def initialize(body, format)
96
+ @body = body
97
+ @format = format
98
+ end
99
+
100
+ # @return [Object] the parsed body
101
+ # @return [nil] when the response body is nil, an empty string, spaces only or "null"
102
+ def parse
103
+ return nil if body.nil?
104
+ return nil if body == "null"
105
+ return nil if body.valid_encoding? && body.strip.empty?
106
+ if body.valid_encoding? && body.encoding == Encoding::UTF_8
107
+ @body = body.gsub(/\A#{UTF8_BOM}/, '')
108
+ end
109
+ if supports_format?
110
+ parse_supported_format
111
+ else
112
+ body
113
+ end
114
+ end
115
+
116
+ protected
117
+
118
+ def xml
119
+ MultiXml.parse(body)
120
+ end
121
+
122
+ UTF8_BOM = "\xEF\xBB\xBF".freeze
123
+
124
+ def json
125
+ JSON.parse(body, :quirks_mode => true, :allow_nan => true)
126
+ end
127
+
128
+ def csv
129
+ CSV.parse(body)
130
+ end
131
+
132
+ def html
133
+ body
134
+ end
135
+
136
+ def plain
137
+ body
138
+ end
139
+
140
+ def supports_format?
141
+ self.class.supports_format?(format)
142
+ end
143
+
144
+ def parse_supported_format
145
+ send(format)
146
+ rescue NoMethodError => e
147
+ raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
148
+ end
149
+ end
150
+ end