httparty-responsibly 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.editorconfig +18 -0
- data/.gitignore +13 -0
- data/.rubocop.yml +92 -0
- data/.rubocop_todo.yml +124 -0
- data/.simplecov +1 -0
- data/.travis.yml +11 -0
- data/CONTRIBUTING.md +23 -0
- data/Changelog.md +509 -0
- data/Gemfile +24 -0
- data/Guardfile +16 -0
- data/MIT-LICENSE +20 -0
- data/README.md +78 -0
- data/Rakefile +10 -0
- data/bin/httparty +123 -0
- data/cucumber.yml +1 -0
- data/docs/README.md +106 -0
- data/examples/README.md +86 -0
- data/examples/aaws.rb +32 -0
- data/examples/basic.rb +28 -0
- data/examples/body_stream.rb +14 -0
- data/examples/crack.rb +19 -0
- data/examples/custom_parsers.rb +68 -0
- data/examples/delicious.rb +37 -0
- data/examples/google.rb +16 -0
- data/examples/headers_and_user_agents.rb +10 -0
- data/examples/logging.rb +36 -0
- data/examples/microsoft_graph.rb +52 -0
- data/examples/multipart.rb +22 -0
- data/examples/nokogiri_html_parser.rb +19 -0
- data/examples/peer_cert.rb +9 -0
- data/examples/rescue_json.rb +17 -0
- data/examples/rubyurl.rb +14 -0
- data/examples/stackexchange.rb +24 -0
- data/examples/stream_download.rb +26 -0
- data/examples/tripit_sign_in.rb +44 -0
- data/examples/twitter.rb +31 -0
- data/examples/whoismyrep.rb +10 -0
- data/httparty-responsibly.gemspec +27 -0
- data/lib/httparty.rb +668 -0
- data/lib/httparty/connection_adapter.rb +254 -0
- data/lib/httparty/cookie_hash.rb +21 -0
- data/lib/httparty/exceptions.rb +33 -0
- data/lib/httparty/hash_conversions.rb +69 -0
- data/lib/httparty/headers_processor.rb +30 -0
- data/lib/httparty/logger/apache_formatter.rb +45 -0
- data/lib/httparty/logger/curl_formatter.rb +91 -0
- data/lib/httparty/logger/logger.rb +28 -0
- data/lib/httparty/logger/logstash_formatter.rb +59 -0
- data/lib/httparty/module_inheritable_attributes.rb +56 -0
- data/lib/httparty/net_digest_auth.rb +136 -0
- data/lib/httparty/parser.rb +150 -0
- data/lib/httparty/request.rb +386 -0
- data/lib/httparty/request/body.rb +84 -0
- data/lib/httparty/request/multipart_boundary.rb +11 -0
- data/lib/httparty/response.rb +140 -0
- data/lib/httparty/response/headers.rb +33 -0
- data/lib/httparty/response_fragment.rb +19 -0
- data/lib/httparty/text_encoder.rb +70 -0
- data/lib/httparty/utils.rb +11 -0
- data/lib/httparty/version.rb +3 -0
- data/script/release +42 -0
- data/website/css/common.css +47 -0
- data/website/index.html +73 -0
- metadata +138 -0
@@ -0,0 +1,91 @@
|
|
1
|
+
module HTTParty
|
2
|
+
module Logger
|
3
|
+
class CurlFormatter #:nodoc:
|
4
|
+
TAG_NAME = HTTParty.name
|
5
|
+
OUT = '>'.freeze
|
6
|
+
IN = '<'.freeze
|
7
|
+
|
8
|
+
attr_accessor :level, :logger
|
9
|
+
|
10
|
+
def initialize(logger, level)
|
11
|
+
@logger = logger
|
12
|
+
@level = level.to_sym
|
13
|
+
@messages = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def format(request, response)
|
17
|
+
@request = request
|
18
|
+
@response = response
|
19
|
+
|
20
|
+
log_request
|
21
|
+
log_response
|
22
|
+
|
23
|
+
logger.public_send level, messages.join("\n")
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
attr_reader :request, :response
|
29
|
+
attr_accessor :messages
|
30
|
+
|
31
|
+
def log_request
|
32
|
+
log_url
|
33
|
+
log_headers
|
34
|
+
log_query
|
35
|
+
log OUT, request.raw_body if request.raw_body
|
36
|
+
log OUT
|
37
|
+
end
|
38
|
+
|
39
|
+
def log_response
|
40
|
+
log IN, "HTTP/#{response.http_version} #{response.code}"
|
41
|
+
log_response_headers
|
42
|
+
log IN, "\n#{response.body}"
|
43
|
+
log IN
|
44
|
+
end
|
45
|
+
|
46
|
+
def log_url
|
47
|
+
http_method = request.http_method.name.split("::").last.upcase
|
48
|
+
uri = if request.options[:base_uri]
|
49
|
+
request.options[:base_uri] + request.path.path
|
50
|
+
else
|
51
|
+
request.path.to_s
|
52
|
+
end
|
53
|
+
|
54
|
+
log OUT, "#{http_method} #{uri}"
|
55
|
+
end
|
56
|
+
|
57
|
+
def log_headers
|
58
|
+
return unless request.options[:headers] && request.options[:headers].size > 0
|
59
|
+
|
60
|
+
log OUT, 'Headers: '
|
61
|
+
log_hash request.options[:headers]
|
62
|
+
end
|
63
|
+
|
64
|
+
def log_query
|
65
|
+
return unless request.options[:query]
|
66
|
+
|
67
|
+
log OUT, 'Query: '
|
68
|
+
log_hash request.options[:query]
|
69
|
+
end
|
70
|
+
|
71
|
+
def log_response_headers
|
72
|
+
headers = response.respond_to?(:headers) ? response.headers : response
|
73
|
+
response.each_header do |response_header|
|
74
|
+
log IN, "#{response_header.capitalize}: #{headers[response_header]}"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def log_hash(hash)
|
79
|
+
hash.each { |k, v| log(OUT, "#{k}: #{v}") }
|
80
|
+
end
|
81
|
+
|
82
|
+
def log(direction, line = '')
|
83
|
+
messages << "[#{TAG_NAME}] [#{current_time}] #{direction} #{line}"
|
84
|
+
end
|
85
|
+
|
86
|
+
def current_time
|
87
|
+
Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'httparty/logger/apache_formatter'
|
2
|
+
require 'httparty/logger/curl_formatter'
|
3
|
+
require 'httparty/logger/logstash_formatter'
|
4
|
+
|
5
|
+
module HTTParty
|
6
|
+
module Logger
|
7
|
+
def self.formatters
|
8
|
+
@formatters ||= {
|
9
|
+
:curl => Logger::CurlFormatter,
|
10
|
+
:apache => Logger::ApacheFormatter,
|
11
|
+
:logstash => Logger::LogstashFormatter,
|
12
|
+
}
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.add_formatter(name, formatter)
|
16
|
+
raise HTTParty::Error.new("Log Formatter with name #{name} already exists") if formatters.include?(name)
|
17
|
+
formatters.merge!(name.to_sym => formatter)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.build(logger, level, formatter)
|
21
|
+
level ||= :info
|
22
|
+
formatter ||= :apache
|
23
|
+
|
24
|
+
logger_klass = formatters[formatter] || Logger::ApacheFormatter
|
25
|
+
logger_klass.new(logger, level)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module HTTParty
|
2
|
+
module Logger
|
3
|
+
class LogstashFormatter #:nodoc:
|
4
|
+
TAG_NAME = HTTParty.name
|
5
|
+
|
6
|
+
attr_accessor :level, :logger
|
7
|
+
|
8
|
+
def initialize(logger, level)
|
9
|
+
@logger = logger
|
10
|
+
@level = level.to_sym
|
11
|
+
end
|
12
|
+
|
13
|
+
def format(request, response)
|
14
|
+
@request = request
|
15
|
+
@response = response
|
16
|
+
|
17
|
+
logger.public_send level, logstash_message
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
attr_reader :request, :response
|
23
|
+
|
24
|
+
def logstash_message
|
25
|
+
{
|
26
|
+
'@timestamp' => current_time,
|
27
|
+
'@version' => 1,
|
28
|
+
'content_length' => content_length || '-',
|
29
|
+
'http_method' => http_method,
|
30
|
+
'message' => message,
|
31
|
+
'path' => path,
|
32
|
+
'response_code' => response.code,
|
33
|
+
'severity' => level,
|
34
|
+
'tags' => [TAG_NAME],
|
35
|
+
}.to_json
|
36
|
+
end
|
37
|
+
|
38
|
+
def message
|
39
|
+
"[#{TAG_NAME}] #{response.code} \"#{http_method} #{path}\" #{content_length || '-'} "
|
40
|
+
end
|
41
|
+
|
42
|
+
def current_time
|
43
|
+
Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
|
44
|
+
end
|
45
|
+
|
46
|
+
def http_method
|
47
|
+
@http_method ||= request.http_method.name.split("::").last.upcase
|
48
|
+
end
|
49
|
+
|
50
|
+
def path
|
51
|
+
@path ||= request.path.to_s
|
52
|
+
end
|
53
|
+
|
54
|
+
def content_length
|
55
|
+
@content_length ||= response.respond_to?(:headers) ? response.headers['Content-Length'] : response['Content-Length']
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module HTTParty
|
2
|
+
module ModuleInheritableAttributes #:nodoc:
|
3
|
+
def self.included(base)
|
4
|
+
base.extend(ClassMethods)
|
5
|
+
end
|
6
|
+
|
7
|
+
# borrowed from Rails 3.2 ActiveSupport
|
8
|
+
def self.hash_deep_dup(hash)
|
9
|
+
duplicate = hash.dup
|
10
|
+
|
11
|
+
duplicate.each_pair do |key, value|
|
12
|
+
if value.is_a?(Hash)
|
13
|
+
duplicate[key] = hash_deep_dup(value)
|
14
|
+
elsif value.is_a?(Proc)
|
15
|
+
duplicate[key] = value.dup
|
16
|
+
else
|
17
|
+
duplicate[key] = value
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
duplicate
|
22
|
+
end
|
23
|
+
|
24
|
+
module ClassMethods #:nodoc:
|
25
|
+
def mattr_inheritable(*args)
|
26
|
+
@mattr_inheritable_attrs ||= [:mattr_inheritable_attrs]
|
27
|
+
@mattr_inheritable_attrs += args
|
28
|
+
|
29
|
+
args.each do |arg|
|
30
|
+
module_eval %(class << self; attr_accessor :#{arg} end)
|
31
|
+
end
|
32
|
+
|
33
|
+
@mattr_inheritable_attrs
|
34
|
+
end
|
35
|
+
|
36
|
+
def inherited(subclass)
|
37
|
+
super
|
38
|
+
@mattr_inheritable_attrs.each do |inheritable_attribute|
|
39
|
+
ivar = "@#{inheritable_attribute}"
|
40
|
+
subclass.instance_variable_set(ivar, instance_variable_get(ivar).clone)
|
41
|
+
|
42
|
+
if instance_variable_get(ivar).respond_to?(:merge)
|
43
|
+
method = <<-EOM
|
44
|
+
def self.#{inheritable_attribute}
|
45
|
+
duplicate = ModuleInheritableAttributes.hash_deep_dup(#{ivar})
|
46
|
+
#{ivar} = superclass.#{inheritable_attribute}.merge(duplicate)
|
47
|
+
end
|
48
|
+
EOM
|
49
|
+
|
50
|
+
subclass.class_eval method
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
module Net
|
5
|
+
module HTTPHeader
|
6
|
+
def digest_auth(username, password, response)
|
7
|
+
authenticator = DigestAuthenticator.new(
|
8
|
+
username,
|
9
|
+
password,
|
10
|
+
@method,
|
11
|
+
@path,
|
12
|
+
response
|
13
|
+
)
|
14
|
+
|
15
|
+
authenticator.authorization_header.each do |v|
|
16
|
+
add_field('Authorization', v)
|
17
|
+
end
|
18
|
+
|
19
|
+
authenticator.cookie_header.each do |v|
|
20
|
+
add_field('Cookie', v)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class DigestAuthenticator
|
25
|
+
def initialize(username, password, method, path, response_header)
|
26
|
+
@username = username
|
27
|
+
@password = password
|
28
|
+
@method = method
|
29
|
+
@path = path
|
30
|
+
@response = parse(response_header)
|
31
|
+
@cookies = parse_cookies(response_header)
|
32
|
+
end
|
33
|
+
|
34
|
+
def authorization_header
|
35
|
+
@cnonce = md5(random)
|
36
|
+
header = [
|
37
|
+
%(Digest username="#{@username}"),
|
38
|
+
%(realm="#{@response['realm']}"),
|
39
|
+
%(nonce="#{@response['nonce']}"),
|
40
|
+
%(uri="#{@path}"),
|
41
|
+
%(response="#{request_digest}")
|
42
|
+
]
|
43
|
+
|
44
|
+
header << %(algorithm="#{@response['algorithm']}") if algorithm_present?
|
45
|
+
|
46
|
+
if qop_present?
|
47
|
+
fields = [
|
48
|
+
%(cnonce="#{@cnonce}"),
|
49
|
+
%(qop="#{@response['qop']}"),
|
50
|
+
"nc=00000001"
|
51
|
+
]
|
52
|
+
fields.each { |field| header << field }
|
53
|
+
end
|
54
|
+
|
55
|
+
header << %(opaque="#{@response['opaque']}") if opaque_present?
|
56
|
+
header
|
57
|
+
end
|
58
|
+
|
59
|
+
def cookie_header
|
60
|
+
@cookies
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def parse(response_header)
|
66
|
+
header = response_header['www-authenticate']
|
67
|
+
|
68
|
+
header = header.gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
|
69
|
+
|
70
|
+
header =~ /Digest (.*)/
|
71
|
+
params = {}
|
72
|
+
if $1
|
73
|
+
non_quoted = $1.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
|
74
|
+
non_quoted.gsub(/(\w+)=([^,]*)/) { params[$1] = $2 }
|
75
|
+
end
|
76
|
+
params
|
77
|
+
end
|
78
|
+
|
79
|
+
def parse_cookies(response_header)
|
80
|
+
return [] unless response_header['Set-Cookie']
|
81
|
+
|
82
|
+
cookies = response_header['Set-Cookie'].split('; ')
|
83
|
+
|
84
|
+
cookies.reduce([]) do |ret, cookie|
|
85
|
+
ret << cookie
|
86
|
+
ret
|
87
|
+
end
|
88
|
+
|
89
|
+
cookies
|
90
|
+
end
|
91
|
+
|
92
|
+
def opaque_present?
|
93
|
+
@response.key?('opaque') && !@response['opaque'].empty?
|
94
|
+
end
|
95
|
+
|
96
|
+
def qop_present?
|
97
|
+
@response.key?('qop') && !@response['qop'].empty?
|
98
|
+
end
|
99
|
+
|
100
|
+
def random
|
101
|
+
format "%x", (Time.now.to_i + rand(65535))
|
102
|
+
end
|
103
|
+
|
104
|
+
def request_digest
|
105
|
+
a = [md5(a1), @response['nonce'], md5(a2)]
|
106
|
+
a.insert(2, "00000001", @cnonce, @response['qop']) if qop_present?
|
107
|
+
md5(a.join(":"))
|
108
|
+
end
|
109
|
+
|
110
|
+
def md5(str)
|
111
|
+
Digest::MD5.hexdigest(str)
|
112
|
+
end
|
113
|
+
|
114
|
+
def algorithm_present?
|
115
|
+
@response.key?('algorithm') && !@response['algorithm'].empty?
|
116
|
+
end
|
117
|
+
|
118
|
+
def use_md5_sess?
|
119
|
+
algorithm_present? && @response['algorithm'] == 'MD5-sess'
|
120
|
+
end
|
121
|
+
|
122
|
+
def a1
|
123
|
+
a1_user_realm_pwd = [@username, @response['realm'], @password].join(':')
|
124
|
+
if use_md5_sess?
|
125
|
+
[ md5(a1_user_realm_pwd), @response['nonce'], @cnonce ].join(':')
|
126
|
+
else
|
127
|
+
a1_user_realm_pwd
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def a2
|
132
|
+
[@method, @path].join(":")
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,150 @@
|
|
1
|
+
module HTTParty
|
2
|
+
# The default parser used by HTTParty, supports xml, json, html, csv and
|
3
|
+
# plain text.
|
4
|
+
#
|
5
|
+
# == Custom Parsers
|
6
|
+
#
|
7
|
+
# If you'd like to do your own custom parsing, subclassing HTTParty::Parser
|
8
|
+
# will make that process much easier. There are a few different ways you can
|
9
|
+
# utilize HTTParty::Parser as a superclass.
|
10
|
+
#
|
11
|
+
# @example Intercept the parsing for all formats
|
12
|
+
# class SimpleParser < HTTParty::Parser
|
13
|
+
# def parse
|
14
|
+
# perform_parsing
|
15
|
+
# end
|
16
|
+
# end
|
17
|
+
#
|
18
|
+
# @example Add the atom format and parsing method to the default parser
|
19
|
+
# class AtomParsingIncluded < HTTParty::Parser
|
20
|
+
# SupportedFormats.merge!(
|
21
|
+
# {"application/atom+xml" => :atom}
|
22
|
+
# )
|
23
|
+
#
|
24
|
+
# def atom
|
25
|
+
# perform_atom_parsing
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
#
|
29
|
+
# @example Only support the atom format
|
30
|
+
# class ParseOnlyAtom < HTTParty::Parser
|
31
|
+
# SupportedFormats = {"application/atom+xml" => :atom}
|
32
|
+
#
|
33
|
+
# def atom
|
34
|
+
# perform_atom_parsing
|
35
|
+
# end
|
36
|
+
# end
|
37
|
+
#
|
38
|
+
# @abstract Read the Custom Parsers section for more information.
|
39
|
+
class Parser
|
40
|
+
SupportedFormats = {
|
41
|
+
'text/xml' => :xml,
|
42
|
+
'application/xml' => :xml,
|
43
|
+
'application/json' => :json,
|
44
|
+
'application/vnd.api+json' => :json,
|
45
|
+
'application/hal+json' => :json,
|
46
|
+
'text/json' => :json,
|
47
|
+
'application/javascript' => :plain,
|
48
|
+
'text/javascript' => :plain,
|
49
|
+
'text/html' => :html,
|
50
|
+
'text/plain' => :plain,
|
51
|
+
'text/csv' => :csv,
|
52
|
+
'application/csv' => :csv,
|
53
|
+
'text/comma-separated-values' => :csv
|
54
|
+
}
|
55
|
+
|
56
|
+
# The response body of the request
|
57
|
+
# @return [String]
|
58
|
+
attr_reader :body
|
59
|
+
|
60
|
+
# The intended parsing format for the request
|
61
|
+
# @return [Symbol] e.g. :json
|
62
|
+
attr_reader :format
|
63
|
+
|
64
|
+
# Instantiate the parser and call {#parse}.
|
65
|
+
# @param [String] body the response body
|
66
|
+
# @param [Symbol] format the response format
|
67
|
+
# @return parsed response
|
68
|
+
def self.call(body, format)
|
69
|
+
new(body, format).parse
|
70
|
+
end
|
71
|
+
|
72
|
+
# @return [Hash] the SupportedFormats hash
|
73
|
+
def self.formats
|
74
|
+
const_get(:SupportedFormats)
|
75
|
+
end
|
76
|
+
|
77
|
+
# @param [String] mimetype response MIME type
|
78
|
+
# @return [Symbol]
|
79
|
+
# @return [nil] mime type not supported
|
80
|
+
def self.format_from_mimetype(mimetype)
|
81
|
+
formats[formats.keys.detect {|k| mimetype.include?(k)}]
|
82
|
+
end
|
83
|
+
|
84
|
+
# @return [Array<Symbol>] list of supported formats
|
85
|
+
def self.supported_formats
|
86
|
+
formats.values.uniq
|
87
|
+
end
|
88
|
+
|
89
|
+
# @param [Symbol] format e.g. :json, :xml
|
90
|
+
# @return [Boolean]
|
91
|
+
def self.supports_format?(format)
|
92
|
+
supported_formats.include?(format)
|
93
|
+
end
|
94
|
+
|
95
|
+
def initialize(body, format)
|
96
|
+
@body = body
|
97
|
+
@format = format
|
98
|
+
end
|
99
|
+
|
100
|
+
# @return [Object] the parsed body
|
101
|
+
# @return [nil] when the response body is nil, an empty string, spaces only or "null"
|
102
|
+
def parse
|
103
|
+
return nil if body.nil?
|
104
|
+
return nil if body == "null"
|
105
|
+
return nil if body.valid_encoding? && body.strip.empty?
|
106
|
+
if body.valid_encoding? && body.encoding == Encoding::UTF_8
|
107
|
+
@body = body.gsub(/\A#{UTF8_BOM}/, '')
|
108
|
+
end
|
109
|
+
if supports_format?
|
110
|
+
parse_supported_format
|
111
|
+
else
|
112
|
+
body
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
protected
|
117
|
+
|
118
|
+
def xml
|
119
|
+
MultiXml.parse(body)
|
120
|
+
end
|
121
|
+
|
122
|
+
UTF8_BOM = "\xEF\xBB\xBF".freeze
|
123
|
+
|
124
|
+
def json
|
125
|
+
JSON.parse(body, :quirks_mode => true, :allow_nan => true)
|
126
|
+
end
|
127
|
+
|
128
|
+
def csv
|
129
|
+
CSV.parse(body)
|
130
|
+
end
|
131
|
+
|
132
|
+
def html
|
133
|
+
body
|
134
|
+
end
|
135
|
+
|
136
|
+
def plain
|
137
|
+
body
|
138
|
+
end
|
139
|
+
|
140
|
+
def supports_format?
|
141
|
+
self.class.supports_format?(format)
|
142
|
+
end
|
143
|
+
|
144
|
+
def parse_supported_format
|
145
|
+
send(format)
|
146
|
+
rescue NoMethodError => e
|
147
|
+
raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|