httparty-responsibly 0.17.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.editorconfig +18 -0
- data/.gitignore +13 -0
- data/.rubocop.yml +92 -0
- data/.rubocop_todo.yml +124 -0
- data/.simplecov +1 -0
- data/.travis.yml +11 -0
- data/CONTRIBUTING.md +23 -0
- data/Changelog.md +509 -0
- data/Gemfile +24 -0
- data/Guardfile +16 -0
- data/MIT-LICENSE +20 -0
- data/README.md +78 -0
- data/Rakefile +10 -0
- data/bin/httparty +123 -0
- data/cucumber.yml +1 -0
- data/docs/README.md +106 -0
- data/examples/README.md +86 -0
- data/examples/aaws.rb +32 -0
- data/examples/basic.rb +28 -0
- data/examples/body_stream.rb +14 -0
- data/examples/crack.rb +19 -0
- data/examples/custom_parsers.rb +68 -0
- data/examples/delicious.rb +37 -0
- data/examples/google.rb +16 -0
- data/examples/headers_and_user_agents.rb +10 -0
- data/examples/logging.rb +36 -0
- data/examples/microsoft_graph.rb +52 -0
- data/examples/multipart.rb +22 -0
- data/examples/nokogiri_html_parser.rb +19 -0
- data/examples/peer_cert.rb +9 -0
- data/examples/rescue_json.rb +17 -0
- data/examples/rubyurl.rb +14 -0
- data/examples/stackexchange.rb +24 -0
- data/examples/stream_download.rb +26 -0
- data/examples/tripit_sign_in.rb +44 -0
- data/examples/twitter.rb +31 -0
- data/examples/whoismyrep.rb +10 -0
- data/httparty-responsibly.gemspec +27 -0
- data/lib/httparty.rb +668 -0
- data/lib/httparty/connection_adapter.rb +254 -0
- data/lib/httparty/cookie_hash.rb +21 -0
- data/lib/httparty/exceptions.rb +33 -0
- data/lib/httparty/hash_conversions.rb +69 -0
- data/lib/httparty/headers_processor.rb +30 -0
- data/lib/httparty/logger/apache_formatter.rb +45 -0
- data/lib/httparty/logger/curl_formatter.rb +91 -0
- data/lib/httparty/logger/logger.rb +28 -0
- data/lib/httparty/logger/logstash_formatter.rb +59 -0
- data/lib/httparty/module_inheritable_attributes.rb +56 -0
- data/lib/httparty/net_digest_auth.rb +136 -0
- data/lib/httparty/parser.rb +150 -0
- data/lib/httparty/request.rb +386 -0
- data/lib/httparty/request/body.rb +84 -0
- data/lib/httparty/request/multipart_boundary.rb +11 -0
- data/lib/httparty/response.rb +140 -0
- data/lib/httparty/response/headers.rb +33 -0
- data/lib/httparty/response_fragment.rb +19 -0
- data/lib/httparty/text_encoder.rb +70 -0
- data/lib/httparty/utils.rb +11 -0
- data/lib/httparty/version.rb +3 -0
- data/script/release +42 -0
- data/website/css/common.css +47 -0
- data/website/index.html +73 -0
- metadata +138 -0
@@ -0,0 +1,91 @@
|
|
1
|
+
module HTTParty
|
2
|
+
module Logger
|
3
|
+
class CurlFormatter #:nodoc:
|
4
|
+
TAG_NAME = HTTParty.name
|
5
|
+
OUT = '>'.freeze
|
6
|
+
IN = '<'.freeze
|
7
|
+
|
8
|
+
attr_accessor :level, :logger
|
9
|
+
|
10
|
+
def initialize(logger, level)
|
11
|
+
@logger = logger
|
12
|
+
@level = level.to_sym
|
13
|
+
@messages = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def format(request, response)
|
17
|
+
@request = request
|
18
|
+
@response = response
|
19
|
+
|
20
|
+
log_request
|
21
|
+
log_response
|
22
|
+
|
23
|
+
logger.public_send level, messages.join("\n")
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
attr_reader :request, :response
|
29
|
+
attr_accessor :messages
|
30
|
+
|
31
|
+
def log_request
|
32
|
+
log_url
|
33
|
+
log_headers
|
34
|
+
log_query
|
35
|
+
log OUT, request.raw_body if request.raw_body
|
36
|
+
log OUT
|
37
|
+
end
|
38
|
+
|
39
|
+
def log_response
|
40
|
+
log IN, "HTTP/#{response.http_version} #{response.code}"
|
41
|
+
log_response_headers
|
42
|
+
log IN, "\n#{response.body}"
|
43
|
+
log IN
|
44
|
+
end
|
45
|
+
|
46
|
+
def log_url
|
47
|
+
http_method = request.http_method.name.split("::").last.upcase
|
48
|
+
uri = if request.options[:base_uri]
|
49
|
+
request.options[:base_uri] + request.path.path
|
50
|
+
else
|
51
|
+
request.path.to_s
|
52
|
+
end
|
53
|
+
|
54
|
+
log OUT, "#{http_method} #{uri}"
|
55
|
+
end
|
56
|
+
|
57
|
+
def log_headers
|
58
|
+
return unless request.options[:headers] && request.options[:headers].size > 0
|
59
|
+
|
60
|
+
log OUT, 'Headers: '
|
61
|
+
log_hash request.options[:headers]
|
62
|
+
end
|
63
|
+
|
64
|
+
def log_query
|
65
|
+
return unless request.options[:query]
|
66
|
+
|
67
|
+
log OUT, 'Query: '
|
68
|
+
log_hash request.options[:query]
|
69
|
+
end
|
70
|
+
|
71
|
+
def log_response_headers
|
72
|
+
headers = response.respond_to?(:headers) ? response.headers : response
|
73
|
+
response.each_header do |response_header|
|
74
|
+
log IN, "#{response_header.capitalize}: #{headers[response_header]}"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def log_hash(hash)
|
79
|
+
hash.each { |k, v| log(OUT, "#{k}: #{v}") }
|
80
|
+
end
|
81
|
+
|
82
|
+
def log(direction, line = '')
|
83
|
+
messages << "[#{TAG_NAME}] [#{current_time}] #{direction} #{line}"
|
84
|
+
end
|
85
|
+
|
86
|
+
def current_time
|
87
|
+
Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'httparty/logger/apache_formatter'
|
2
|
+
require 'httparty/logger/curl_formatter'
|
3
|
+
require 'httparty/logger/logstash_formatter'
|
4
|
+
|
5
|
+
module HTTParty
|
6
|
+
module Logger
|
7
|
+
def self.formatters
|
8
|
+
@formatters ||= {
|
9
|
+
:curl => Logger::CurlFormatter,
|
10
|
+
:apache => Logger::ApacheFormatter,
|
11
|
+
:logstash => Logger::LogstashFormatter,
|
12
|
+
}
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.add_formatter(name, formatter)
|
16
|
+
raise HTTParty::Error.new("Log Formatter with name #{name} already exists") if formatters.include?(name)
|
17
|
+
formatters.merge!(name.to_sym => formatter)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.build(logger, level, formatter)
|
21
|
+
level ||= :info
|
22
|
+
formatter ||= :apache
|
23
|
+
|
24
|
+
logger_klass = formatters[formatter] || Logger::ApacheFormatter
|
25
|
+
logger_klass.new(logger, level)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module HTTParty
|
2
|
+
module Logger
|
3
|
+
class LogstashFormatter #:nodoc:
|
4
|
+
TAG_NAME = HTTParty.name
|
5
|
+
|
6
|
+
attr_accessor :level, :logger
|
7
|
+
|
8
|
+
def initialize(logger, level)
|
9
|
+
@logger = logger
|
10
|
+
@level = level.to_sym
|
11
|
+
end
|
12
|
+
|
13
|
+
def format(request, response)
|
14
|
+
@request = request
|
15
|
+
@response = response
|
16
|
+
|
17
|
+
logger.public_send level, logstash_message
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
attr_reader :request, :response
|
23
|
+
|
24
|
+
def logstash_message
|
25
|
+
{
|
26
|
+
'@timestamp' => current_time,
|
27
|
+
'@version' => 1,
|
28
|
+
'content_length' => content_length || '-',
|
29
|
+
'http_method' => http_method,
|
30
|
+
'message' => message,
|
31
|
+
'path' => path,
|
32
|
+
'response_code' => response.code,
|
33
|
+
'severity' => level,
|
34
|
+
'tags' => [TAG_NAME],
|
35
|
+
}.to_json
|
36
|
+
end
|
37
|
+
|
38
|
+
def message
|
39
|
+
"[#{TAG_NAME}] #{response.code} \"#{http_method} #{path}\" #{content_length || '-'} "
|
40
|
+
end
|
41
|
+
|
42
|
+
def current_time
|
43
|
+
Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
|
44
|
+
end
|
45
|
+
|
46
|
+
def http_method
|
47
|
+
@http_method ||= request.http_method.name.split("::").last.upcase
|
48
|
+
end
|
49
|
+
|
50
|
+
def path
|
51
|
+
@path ||= request.path.to_s
|
52
|
+
end
|
53
|
+
|
54
|
+
def content_length
|
55
|
+
@content_length ||= response.respond_to?(:headers) ? response.headers['Content-Length'] : response['Content-Length']
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module HTTParty
|
2
|
+
module ModuleInheritableAttributes #:nodoc:
|
3
|
+
def self.included(base)
|
4
|
+
base.extend(ClassMethods)
|
5
|
+
end
|
6
|
+
|
7
|
+
# borrowed from Rails 3.2 ActiveSupport
|
8
|
+
def self.hash_deep_dup(hash)
|
9
|
+
duplicate = hash.dup
|
10
|
+
|
11
|
+
duplicate.each_pair do |key, value|
|
12
|
+
if value.is_a?(Hash)
|
13
|
+
duplicate[key] = hash_deep_dup(value)
|
14
|
+
elsif value.is_a?(Proc)
|
15
|
+
duplicate[key] = value.dup
|
16
|
+
else
|
17
|
+
duplicate[key] = value
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
duplicate
|
22
|
+
end
|
23
|
+
|
24
|
+
module ClassMethods #:nodoc:
|
25
|
+
def mattr_inheritable(*args)
|
26
|
+
@mattr_inheritable_attrs ||= [:mattr_inheritable_attrs]
|
27
|
+
@mattr_inheritable_attrs += args
|
28
|
+
|
29
|
+
args.each do |arg|
|
30
|
+
module_eval %(class << self; attr_accessor :#{arg} end)
|
31
|
+
end
|
32
|
+
|
33
|
+
@mattr_inheritable_attrs
|
34
|
+
end
|
35
|
+
|
36
|
+
def inherited(subclass)
|
37
|
+
super
|
38
|
+
@mattr_inheritable_attrs.each do |inheritable_attribute|
|
39
|
+
ivar = "@#{inheritable_attribute}"
|
40
|
+
subclass.instance_variable_set(ivar, instance_variable_get(ivar).clone)
|
41
|
+
|
42
|
+
if instance_variable_get(ivar).respond_to?(:merge)
|
43
|
+
method = <<-EOM
|
44
|
+
def self.#{inheritable_attribute}
|
45
|
+
duplicate = ModuleInheritableAttributes.hash_deep_dup(#{ivar})
|
46
|
+
#{ivar} = superclass.#{inheritable_attribute}.merge(duplicate)
|
47
|
+
end
|
48
|
+
EOM
|
49
|
+
|
50
|
+
subclass.class_eval method
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
module Net
|
5
|
+
module HTTPHeader
|
6
|
+
def digest_auth(username, password, response)
|
7
|
+
authenticator = DigestAuthenticator.new(
|
8
|
+
username,
|
9
|
+
password,
|
10
|
+
@method,
|
11
|
+
@path,
|
12
|
+
response
|
13
|
+
)
|
14
|
+
|
15
|
+
authenticator.authorization_header.each do |v|
|
16
|
+
add_field('Authorization', v)
|
17
|
+
end
|
18
|
+
|
19
|
+
authenticator.cookie_header.each do |v|
|
20
|
+
add_field('Cookie', v)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class DigestAuthenticator
|
25
|
+
def initialize(username, password, method, path, response_header)
|
26
|
+
@username = username
|
27
|
+
@password = password
|
28
|
+
@method = method
|
29
|
+
@path = path
|
30
|
+
@response = parse(response_header)
|
31
|
+
@cookies = parse_cookies(response_header)
|
32
|
+
end
|
33
|
+
|
34
|
+
def authorization_header
|
35
|
+
@cnonce = md5(random)
|
36
|
+
header = [
|
37
|
+
%(Digest username="#{@username}"),
|
38
|
+
%(realm="#{@response['realm']}"),
|
39
|
+
%(nonce="#{@response['nonce']}"),
|
40
|
+
%(uri="#{@path}"),
|
41
|
+
%(response="#{request_digest}")
|
42
|
+
]
|
43
|
+
|
44
|
+
header << %(algorithm="#{@response['algorithm']}") if algorithm_present?
|
45
|
+
|
46
|
+
if qop_present?
|
47
|
+
fields = [
|
48
|
+
%(cnonce="#{@cnonce}"),
|
49
|
+
%(qop="#{@response['qop']}"),
|
50
|
+
"nc=00000001"
|
51
|
+
]
|
52
|
+
fields.each { |field| header << field }
|
53
|
+
end
|
54
|
+
|
55
|
+
header << %(opaque="#{@response['opaque']}") if opaque_present?
|
56
|
+
header
|
57
|
+
end
|
58
|
+
|
59
|
+
def cookie_header
|
60
|
+
@cookies
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def parse(response_header)
|
66
|
+
header = response_header['www-authenticate']
|
67
|
+
|
68
|
+
header = header.gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
|
69
|
+
|
70
|
+
header =~ /Digest (.*)/
|
71
|
+
params = {}
|
72
|
+
if $1
|
73
|
+
non_quoted = $1.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
|
74
|
+
non_quoted.gsub(/(\w+)=([^,]*)/) { params[$1] = $2 }
|
75
|
+
end
|
76
|
+
params
|
77
|
+
end
|
78
|
+
|
79
|
+
def parse_cookies(response_header)
|
80
|
+
return [] unless response_header['Set-Cookie']
|
81
|
+
|
82
|
+
cookies = response_header['Set-Cookie'].split('; ')
|
83
|
+
|
84
|
+
cookies.reduce([]) do |ret, cookie|
|
85
|
+
ret << cookie
|
86
|
+
ret
|
87
|
+
end
|
88
|
+
|
89
|
+
cookies
|
90
|
+
end
|
91
|
+
|
92
|
+
def opaque_present?
|
93
|
+
@response.key?('opaque') && !@response['opaque'].empty?
|
94
|
+
end
|
95
|
+
|
96
|
+
def qop_present?
|
97
|
+
@response.key?('qop') && !@response['qop'].empty?
|
98
|
+
end
|
99
|
+
|
100
|
+
def random
|
101
|
+
format "%x", (Time.now.to_i + rand(65535))
|
102
|
+
end
|
103
|
+
|
104
|
+
def request_digest
|
105
|
+
a = [md5(a1), @response['nonce'], md5(a2)]
|
106
|
+
a.insert(2, "00000001", @cnonce, @response['qop']) if qop_present?
|
107
|
+
md5(a.join(":"))
|
108
|
+
end
|
109
|
+
|
110
|
+
def md5(str)
|
111
|
+
Digest::MD5.hexdigest(str)
|
112
|
+
end
|
113
|
+
|
114
|
+
def algorithm_present?
|
115
|
+
@response.key?('algorithm') && !@response['algorithm'].empty?
|
116
|
+
end
|
117
|
+
|
118
|
+
def use_md5_sess?
|
119
|
+
algorithm_present? && @response['algorithm'] == 'MD5-sess'
|
120
|
+
end
|
121
|
+
|
122
|
+
def a1
|
123
|
+
a1_user_realm_pwd = [@username, @response['realm'], @password].join(':')
|
124
|
+
if use_md5_sess?
|
125
|
+
[ md5(a1_user_realm_pwd), @response['nonce'], @cnonce ].join(':')
|
126
|
+
else
|
127
|
+
a1_user_realm_pwd
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def a2
|
132
|
+
[@method, @path].join(":")
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,150 @@
|
|
1
|
+
module HTTParty
|
2
|
+
# The default parser used by HTTParty, supports xml, json, html, csv and
|
3
|
+
# plain text.
|
4
|
+
#
|
5
|
+
# == Custom Parsers
|
6
|
+
#
|
7
|
+
# If you'd like to do your own custom parsing, subclassing HTTParty::Parser
|
8
|
+
# will make that process much easier. There are a few different ways you can
|
9
|
+
# utilize HTTParty::Parser as a superclass.
|
10
|
+
#
|
11
|
+
# @example Intercept the parsing for all formats
|
12
|
+
# class SimpleParser < HTTParty::Parser
|
13
|
+
# def parse
|
14
|
+
# perform_parsing
|
15
|
+
# end
|
16
|
+
# end
|
17
|
+
#
|
18
|
+
# @example Add the atom format and parsing method to the default parser
|
19
|
+
# class AtomParsingIncluded < HTTParty::Parser
|
20
|
+
# SupportedFormats.merge!(
|
21
|
+
# {"application/atom+xml" => :atom}
|
22
|
+
# )
|
23
|
+
#
|
24
|
+
# def atom
|
25
|
+
# perform_atom_parsing
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
#
|
29
|
+
# @example Only support the atom format
|
30
|
+
# class ParseOnlyAtom < HTTParty::Parser
|
31
|
+
# SupportedFormats = {"application/atom+xml" => :atom}
|
32
|
+
#
|
33
|
+
# def atom
|
34
|
+
# perform_atom_parsing
|
35
|
+
# end
|
36
|
+
# end
|
37
|
+
#
|
38
|
+
# @abstract Read the Custom Parsers section for more information.
|
39
|
+
class Parser
|
40
|
+
SupportedFormats = {
|
41
|
+
'text/xml' => :xml,
|
42
|
+
'application/xml' => :xml,
|
43
|
+
'application/json' => :json,
|
44
|
+
'application/vnd.api+json' => :json,
|
45
|
+
'application/hal+json' => :json,
|
46
|
+
'text/json' => :json,
|
47
|
+
'application/javascript' => :plain,
|
48
|
+
'text/javascript' => :plain,
|
49
|
+
'text/html' => :html,
|
50
|
+
'text/plain' => :plain,
|
51
|
+
'text/csv' => :csv,
|
52
|
+
'application/csv' => :csv,
|
53
|
+
'text/comma-separated-values' => :csv
|
54
|
+
}
|
55
|
+
|
56
|
+
# The response body of the request
|
57
|
+
# @return [String]
|
58
|
+
attr_reader :body
|
59
|
+
|
60
|
+
# The intended parsing format for the request
|
61
|
+
# @return [Symbol] e.g. :json
|
62
|
+
attr_reader :format
|
63
|
+
|
64
|
+
# Instantiate the parser and call {#parse}.
|
65
|
+
# @param [String] body the response body
|
66
|
+
# @param [Symbol] format the response format
|
67
|
+
# @return parsed response
|
68
|
+
def self.call(body, format)
|
69
|
+
new(body, format).parse
|
70
|
+
end
|
71
|
+
|
72
|
+
# @return [Hash] the SupportedFormats hash
|
73
|
+
def self.formats
|
74
|
+
const_get(:SupportedFormats)
|
75
|
+
end
|
76
|
+
|
77
|
+
# @param [String] mimetype response MIME type
|
78
|
+
# @return [Symbol]
|
79
|
+
# @return [nil] mime type not supported
|
80
|
+
def self.format_from_mimetype(mimetype)
|
81
|
+
formats[formats.keys.detect {|k| mimetype.include?(k)}]
|
82
|
+
end
|
83
|
+
|
84
|
+
# @return [Array<Symbol>] list of supported formats
|
85
|
+
def self.supported_formats
|
86
|
+
formats.values.uniq
|
87
|
+
end
|
88
|
+
|
89
|
+
# @param [Symbol] format e.g. :json, :xml
|
90
|
+
# @return [Boolean]
|
91
|
+
def self.supports_format?(format)
|
92
|
+
supported_formats.include?(format)
|
93
|
+
end
|
94
|
+
|
95
|
+
def initialize(body, format)
|
96
|
+
@body = body
|
97
|
+
@format = format
|
98
|
+
end
|
99
|
+
|
100
|
+
# @return [Object] the parsed body
|
101
|
+
# @return [nil] when the response body is nil, an empty string, spaces only or "null"
|
102
|
+
def parse
|
103
|
+
return nil if body.nil?
|
104
|
+
return nil if body == "null"
|
105
|
+
return nil if body.valid_encoding? && body.strip.empty?
|
106
|
+
if body.valid_encoding? && body.encoding == Encoding::UTF_8
|
107
|
+
@body = body.gsub(/\A#{UTF8_BOM}/, '')
|
108
|
+
end
|
109
|
+
if supports_format?
|
110
|
+
parse_supported_format
|
111
|
+
else
|
112
|
+
body
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
protected
|
117
|
+
|
118
|
+
def xml
|
119
|
+
MultiXml.parse(body)
|
120
|
+
end
|
121
|
+
|
122
|
+
UTF8_BOM = "\xEF\xBB\xBF".freeze
|
123
|
+
|
124
|
+
def json
|
125
|
+
JSON.parse(body, :quirks_mode => true, :allow_nan => true)
|
126
|
+
end
|
127
|
+
|
128
|
+
def csv
|
129
|
+
CSV.parse(body)
|
130
|
+
end
|
131
|
+
|
132
|
+
def html
|
133
|
+
body
|
134
|
+
end
|
135
|
+
|
136
|
+
def plain
|
137
|
+
body
|
138
|
+
end
|
139
|
+
|
140
|
+
def supports_format?
|
141
|
+
self.class.supports_format?(format)
|
142
|
+
end
|
143
|
+
|
144
|
+
def parse_supported_format
|
145
|
+
send(format)
|
146
|
+
rescue NoMethodError => e
|
147
|
+
raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|