pauldix-typhoeus 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,196 @@
1
+ module Typhoeus
2
+ class Easy
3
+ attr_reader :response_body, :response_header, :method, :headers, :url
4
+ CURLINFO_STRING = 1048576
5
+ OPTION_VALUES = {
6
+ :CURLOPT_URL => 10002,
7
+ :CURLOPT_HTTPGET => 80,
8
+ :CURLOPT_HTTPPOST => 10024,
9
+ :CURLOPT_UPLOAD => 46,
10
+ :CURLOPT_CUSTOMREQUEST => 10036,
11
+ :CURLOPT_POSTFIELDS => 10015,
12
+ :CURLOPT_POSTFIELDSIZE => 60,
13
+ :CURLOPT_USERAGENT => 10018,
14
+ :CURLOPT_TIMEOUT_MS => 155,
15
+ :CURLOPT_NOSIGNAL => 99,
16
+ :CURLOPT_HTTPHEADER => 10023,
17
+ :CURLOPT_FOLLOWLOCATION => 52
18
+ }
19
+ INFO_VALUES = {
20
+ :CURLINFO_RESPONSE_CODE => 2097154,
21
+ :CURLINFO_TOTAL_TIME => 3145731
22
+ }
23
+
24
+ def initialize
25
+ @method = :get
26
+ @post_dat_set = nil
27
+ @headers = {}
28
+ end
29
+
30
+ def total_time_taken
31
+ get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
32
+ end
33
+
34
+ def response_code
35
+ get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
36
+ end
37
+
38
+ def follow_location=(boolean)
39
+ if boolean
40
+ set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 1)
41
+ else
42
+ set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 0)
43
+ end
44
+ end
45
+
46
+ def timeout=(milliseconds)
47
+ @timeout = milliseconds
48
+ set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
49
+ set_option(OPTION_VALUES[:CURLOPT_TIMEOUT_MS], milliseconds)
50
+ end
51
+
52
+ def timed_out?
53
+ @timeout && total_time_taken > @timeout && response_code == 0
54
+ end
55
+
56
+ def request_body=(request_body)
57
+ @request_body = request_body
58
+ if @method == :put
59
+ easy_set_request_body(@request_body)
60
+ headers["Transfer-Encoding"] = ""
61
+ headers["Expect"] = ""
62
+ else
63
+ self.post_data = request_body
64
+ end
65
+ end
66
+
67
+ def user_agent=(user_agent)
68
+ set_option(OPTION_VALUES[:CURLOPT_USERAGENT], user_agent)
69
+ end
70
+
71
+ def url=(url)
72
+ @url = url
73
+ set_option(OPTION_VALUES[:CURLOPT_URL], url)
74
+ end
75
+
76
+ def method=(method)
77
+ @method = method
78
+ if method == :get
79
+ set_option(OPTION_VALUES[:CURLOPT_HTTPGET], 1)
80
+ elsif method == :post
81
+ set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], 1)
82
+ self.post_data = ""
83
+ elsif method == :put
84
+ set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
85
+ else
86
+ set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], "DELETE")
87
+ end
88
+ end
89
+
90
+ def post_data=(data)
91
+ @post_data_set = true
92
+ set_option(OPTION_VALUES[:CURLOPT_POSTFIELDS], data)
93
+ set_option(OPTION_VALUES[:CURLOPT_POSTFIELDSIZE], data.length)
94
+ end
95
+
96
+ def params=(params)
97
+ params_string = params.keys.collect do |k|
98
+ value = params[k]
99
+ if value.is_a? Hash
100
+ value.keys.collect {|sk| CGI.escape("#{k}[#{sk}]") + "=" + CGI.escape(value[sk].to_s)}
101
+ else
102
+ "#{CGI.escape(k.to_s)}=#{CGI.escape(params[k].to_s)}"
103
+ end
104
+ end.flatten.join("&")
105
+
106
+ if method == :get
107
+ self.url = "#{url}?#{params_string}"
108
+ elsif method == :post
109
+ self.post_data = params_string
110
+ end
111
+ end
112
+
113
+ def set_option(option, value)
114
+ if value.class == String
115
+ easy_setopt_string(option, value)
116
+ else
117
+ easy_setopt_long(option, value)
118
+ end
119
+ end
120
+
121
+ def perform
122
+ headers.each_pair do |key, value|
123
+ easy_add_header("#{key}: #{value}")
124
+ end
125
+ easy_set_headers() unless headers.empty?
126
+ easy_perform()
127
+ response_code()
128
+ end
129
+
130
+ # gets called when finished and response code is 200-299
131
+ def success
132
+ @success.call(self) if @success
133
+ end
134
+
135
+ def on_success(&block)
136
+ @success = block
137
+ end
138
+
139
+ def on_success=(block)
140
+ @success = block
141
+ end
142
+
143
+ # gets called when finished and response code is 300-599
144
+ def failure
145
+ @failure.call(self) if @failure
146
+ end
147
+
148
+ def on_failure(&block)
149
+ @failure = block
150
+ end
151
+
152
+ def on_failure=(block)
153
+ @failure = block
154
+ end
155
+
156
+ def retries
157
+ @retries ||= 0
158
+ end
159
+
160
+ def increment_retries
161
+ @retries ||= 0
162
+ @retries += 1
163
+ end
164
+
165
+ def max_retries
166
+ @max_retries ||= 40
167
+ end
168
+
169
+ def max_retries?
170
+ retries >= max_retries
171
+ end
172
+
173
+ def reset
174
+ @response_code = 0
175
+ @response_header = ""
176
+ @response_body = ""
177
+ easy_reset()
178
+ end
179
+
180
+ def get_info_string(option)
181
+ easy_getinfo_string(option)
182
+ end
183
+
184
+ def get_info_long(option)
185
+ easy_getinfo_long(option)
186
+ end
187
+
188
+ def get_info_double(option)
189
+ easy_getinfo_double(option)
190
+ end
191
+
192
+ def curl_version
193
+ version
194
+ end
195
+ end
196
+ end
@@ -0,0 +1,28 @@
1
+ module Typhoeus
2
+ class Filter
3
+ attr_reader :method_name
4
+
5
+ def initialize(method_name, options = {})
6
+ @method_name = method_name
7
+ @options = options
8
+ end
9
+
10
+ def apply_filter?(method_name)
11
+ if @options[:only]
12
+ if @options[:only].instance_of? Symbol
13
+ @options[:only] == method_name
14
+ else
15
+ @options[:only].include?(method_name)
16
+ end
17
+ elsif @options[:except]
18
+ if @options[:except].instance_of? Symbol
19
+ @options[:except] != method_name
20
+ else
21
+ !@options[:except].include?(method_name)
22
+ end
23
+ else
24
+ true
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,21 @@
1
+ module Typhoeus
2
+ class Multi
3
+ def remove(easy)
4
+ multi_remove_handle(easy)
5
+ end
6
+
7
+ def add(easy)
8
+ multi_add_handle(easy)
9
+ end
10
+
11
+ def perform()
12
+ while active_handle_count > 0 do
13
+ multi_perform
14
+ end
15
+ end
16
+
17
+ def cleanup()
18
+ multi_cleanup
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,126 @@
1
+ module Typhoeus
2
+ USER_AGENT = "Typhoeus - http://github.com/pauldix/typhoeus/tree/master"
3
+
4
+ def self.included(base)
5
+ base.extend ClassMethods
6
+ end
7
+
8
+ module ClassMethods
9
+ def get(url, options = {})
10
+ remote_proxy_object(url, :get, options)
11
+ end
12
+
13
+ def post(url, options = {}, &block)
14
+ remote_proxy_object(url, :post, options)
15
+ end
16
+
17
+ def put(url, options = {}, &block)
18
+ remote_proxy_object(url, :put, options)
19
+ end
20
+
21
+ def delete(url, options = {}, &block)
22
+ remote_proxy_object(url, :delete, options)
23
+ end
24
+
25
+ def remote_proxy_object(url, method, options)
26
+ easy = Typhoeus.get_easy_object
27
+
28
+ easy.url = url
29
+ easy.method = method
30
+ easy.headers["User-Agent"] = (options[:user_agent] || Typhoeus::USER_AGENT)
31
+ easy.params = options[:params] if options[:params]
32
+ easy.request_body = options[:body] if options[:body]
33
+ easy.timeout = options[:timeout] if options[:timeout]
34
+
35
+ easy
36
+
37
+ proxy = Typhoeus::RemoteProxyObject.new(clear_memoized_proxy_objects, easy, options)
38
+ set_memoized_proxy_object(method, url, options, proxy)
39
+ end
40
+
41
+ def remote_defaults(options)
42
+ @remote_defaults = options
43
+ end
44
+
45
+ def call_remote_method(method_name, args)
46
+ m = @remote_methods[method_name]
47
+
48
+ base_uri = args.delete(:base_uri) || m.base_uri || ""
49
+
50
+ if args.has_key? :path
51
+ path = args.delete(:path)
52
+ else
53
+ path = m.interpolate_path_with_arguments(args)
54
+ end
55
+ path ||= ""
56
+
57
+ http_method = m.http_method
58
+ url = base_uri + path
59
+ options = m.merge_options(args)
60
+
61
+ # proxy_object = memoized_proxy_object(http_method, url, options)
62
+ # return proxy_object unless proxy_object.nil?
63
+ #
64
+ # if m.cache_responses?
65
+ # object = @cache.get(get_memcache_response_key(method_name, args))
66
+ # if object
67
+ # set_memoized_proxy_object(http_method, url, options, object)
68
+ # return object
69
+ # end
70
+ # end
71
+
72
+ proxy = memoized_proxy_object(http_method, url, options)
73
+ unless proxy
74
+ if m.cache_responses?
75
+ options[:cache] = @cache
76
+ options[:cache_key] = get_memcache_response_key(method_name, args)
77
+ options[:cache_timeout] = m.cache_ttl
78
+ end
79
+ proxy = send(http_method, url, options)
80
+ end
81
+ proxy
82
+ end
83
+
84
+ def set_memoized_proxy_object(http_method, url, options, object)
85
+ @memoized_proxy_objects ||= {}
86
+ @memoized_proxy_objects["#{http_method}_#{url}_#{options.to_s}"] = object
87
+ end
88
+
89
+ def memoized_proxy_object(http_method, url, options)
90
+ @memoized_proxy_objects ||= {}
91
+ @memoized_proxy_objects["#{http_method}_#{url}_#{options.to_s}"]
92
+ end
93
+
94
+ def clear_memoized_proxy_objects
95
+ lambda { @memoized_proxy_objects = {} }
96
+ end
97
+
98
+ def get_memcache_response_key(remote_method_name, args)
99
+ result = "#{remote_method_name.to_s}-#{args.to_s}"
100
+ (Digest::SHA2.new << result).to_s
101
+ end
102
+
103
+ def cache=(cache)
104
+ @cache = cache
105
+ end
106
+
107
+ def define_remote_method(name, args = {})
108
+ @remote_defaults ||= {}
109
+ args[:method] ||= @remote_defaults[:method]
110
+ args[:on_success] ||= @remote_defaults[:on_success]
111
+ args[:on_failure] ||= @remote_defaults[:on_failure]
112
+ args[:base_uri] ||= @remote_defaults[:base_uri]
113
+ args[:path] ||= @remote_defaults[:path]
114
+ m = RemoteMethod.new(args)
115
+
116
+ @remote_methods ||= {}
117
+ @remote_methods[name] = m
118
+
119
+ class_eval <<-SRC
120
+ def self.#{name.to_s}(args = {})
121
+ call_remote_method(:#{name.to_s}, args)
122
+ end
123
+ SRC
124
+ end
125
+ end # ClassMethods
126
+ end
@@ -0,0 +1,107 @@
1
+ module Typhoeus
2
+ class RemoteMethod
3
+ attr_accessor :http_method, :options, :base_uri, :path, :on_success, :on_failure, :cache_ttl
4
+
5
+ def initialize(options = {})
6
+ @http_method = options.delete(:method) || :get
7
+ @options = options
8
+ @base_uri = options.delete(:base_uri)
9
+ @path = options.delete(:path)
10
+ @on_success = options[:on_success]
11
+ @on_failure = options[:on_failure]
12
+ @cache_responses = options.delete(:cache_responses)
13
+ @memoize_responses = options.delete(:memoize_responses) || @cache_responses
14
+ @cache_ttl = @cache_responses == true ? 0 : @cache_responses
15
+
16
+ clear_cache
17
+ end
18
+
19
+ def cache_responses?
20
+ @cache_responses
21
+ end
22
+
23
+ def memoize_responses?
24
+ @memoize_responses
25
+ end
26
+
27
+ def args_options_key(args, options)
28
+ "#{args.to_s}+#{options.to_s}"
29
+ end
30
+
31
+ def calling(args, options)
32
+ @called_methods[args_options_key(args, options)] = true
33
+ end
34
+
35
+ def already_called?(args, options)
36
+ @called_methods.has_key? args_options_key(args, options)
37
+ end
38
+
39
+ def add_response_block(block, args, options)
40
+ @response_blocks[args_options_key(args, options)] << block
41
+ end
42
+
43
+ def call_response_blocks(result, args, options)
44
+ key = args_options_key(args, options)
45
+ @response_blocks[key].each {|block| block.call(result)}
46
+ @response_blocks.delete(key)
47
+ @called_methods.delete(key)
48
+ end
49
+
50
+ def clear_cache
51
+ @response_blocks = Hash.new {|h, k| h[k] = []}
52
+ @called_methods = {}
53
+ end
54
+
55
+ def merge_options(new_options)
56
+ merged = options.merge(new_options)
57
+ if options.has_key?(:params) && new_options.has_key?(:params)
58
+ merged[:params] = options[:params].merge(new_options[:params])
59
+ end
60
+ argument_names.each {|a| merged.delete(a)}
61
+ merged.delete(:on_success) if merged[:on_success].nil?
62
+ merged.delete(:on_failure) if merged[:on_failure].nil?
63
+ merged
64
+ end
65
+
66
+ def interpolate_path_with_arguments(args)
67
+ interpolated_path = @path
68
+ argument_names.each do |arg|
69
+ interpolated_path = interpolated_path.gsub(":#{arg}", args[arg].to_s)
70
+ end
71
+ interpolated_path
72
+ end
73
+
74
+ def argument_names
75
+ return @keys if @keys
76
+ pattern, keys = compile(@path)
77
+ @keys = keys.collect {|k| k.to_sym}
78
+ end
79
+
80
+ # rippped from Sinatra. clean up stuff we don't need later
81
+ def compile(path)
82
+ path ||= ""
83
+ keys = []
84
+ if path.respond_to? :to_str
85
+ special_chars = %w{. + ( )}
86
+ pattern =
87
+ path.gsub(/((:\w+)|[\*#{special_chars.join}])/) do |match|
88
+ case match
89
+ when "*"
90
+ keys << 'splat'
91
+ "(.*?)"
92
+ when *special_chars
93
+ Regexp.escape(match)
94
+ else
95
+ keys << $2[1..-1]
96
+ "([^/?&#]+)"
97
+ end
98
+ end
99
+ [/^#{pattern}$/, keys]
100
+ elsif path.respond_to? :match
101
+ [path, keys]
102
+ else
103
+ raise TypeError, path
104
+ end
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,42 @@
1
+ module Typhoeus
2
+ class RemoteProxyObject
3
+ instance_methods.each { |m| undef_method m unless m =~ /^__/ }
4
+
5
+ def initialize(clear_memoized_store_proc, easy, options = {})
6
+ @clear_memoized_store_proc = clear_memoized_store_proc
7
+ @easy = easy
8
+ @success = options[:on_success]
9
+ @failure = options[:on_failure]
10
+ @cache = options.delete(:cache)
11
+ @cache_key = options.delete(:cache_key)
12
+ @timeout = options.delete(:cache_timeout)
13
+ Typhoeus.add_easy_request(@easy)
14
+ end
15
+
16
+ def method_missing(sym, *args, &block)
17
+ unless @proxied_object
18
+ if @cache && @cache_key
19
+ @proxied_object = @cache.get(@cache_key)
20
+ end
21
+
22
+ unless @proxied_object
23
+ Typhoeus.perform_easy_requests
24
+ response = Response.new(@easy.response_code, @easy.response_header, @easy.response_body, @easy.total_time_taken)
25
+ if @easy.response_code >= 200 && @easy.response_code < 300
26
+ Typhoeus.release_easy_object(@easy)
27
+ @proxied_object = @success.nil? ? response : @success.call(response)
28
+
29
+ if @cache && @cache_key
30
+ @cache.set(@cache_key, @proxied_object, @timeout)
31
+ end
32
+ else
33
+ @proxied_object = @failure.nil? ? response : @failure.call(response)
34
+ end
35
+ @clear_memoized_store_proc.call
36
+ end
37
+ end
38
+
39
+ @proxied_object.__send__(sym, *args, &block)
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,12 @@
1
+ module Typhoeus
2
+ class Response
3
+ attr_reader :code, :headers, :body, :time
4
+
5
+ def initialize(response_code, response_headers, response_body, request_time)
6
+ @code = response_code
7
+ @headers = response_headers
8
+ @body = response_body
9
+ @time = request_time
10
+ end
11
+ end
12
+ end
data/lib/typhoeus.rb ADDED
@@ -0,0 +1,48 @@
1
+ $LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
2
+
3
+ require 'cgi'
4
+ require 'digest/sha2'
5
+ require 'typhoeus/easy'
6
+ require 'typhoeus/multi'
7
+ require 'typhoeus/native'
8
+ require 'typhoeus/filter'
9
+ require 'typhoeus/remote_method'
10
+ require 'typhoeus/remote'
11
+ require 'typhoeus/remote_proxy_object'
12
+ require 'typhoeus/response'
13
+
14
+ module Typhoeus
15
+ VERSION = "0.0.8"
16
+
17
+ def self.easy_object_pool
18
+ @easy_objects ||= []
19
+ end
20
+
21
+ def self.init_easy_object_pool
22
+ 20.times do
23
+ easy_object_pool << Typhoeus::Easy.new
24
+ end
25
+ end
26
+
27
+ def self.release_easy_object(easy)
28
+ easy.reset
29
+ easy_object_pool << easy
30
+ end
31
+
32
+ def self.get_easy_object
33
+ if easy_object_pool.empty?
34
+ Typhoeus::Easy.new
35
+ else
36
+ easy_object_pool.pop
37
+ end
38
+ end
39
+
40
+ def self.add_easy_request(easy_object)
41
+ Thread.current[:curl_multi] ||= Typhoeus::Multi.new
42
+ Thread.current[:curl_multi].add(easy_object)
43
+ end
44
+
45
+ def self.perform_easy_requests
46
+ Thread.current[:curl_multi].perform
47
+ end
48
+ end
@@ -0,0 +1,66 @@
1
+ # this server simply accepts requests and blocks for a passed in interval before returning a passed in reqeust value to
2
+ # the client
3
+ require 'rubygems'
4
+ require 'eventmachine'
5
+ require 'evma_httpserver'
6
+
7
+ class DelayFixtureServer < EventMachine::Connection
8
+ include EventMachine::HttpServer
9
+
10
+ def process_http_request
11
+ EventMachine.stop if ENV["PATH_INFO"] == "/die"
12
+ puts "got a request #{ENV['PATH_INFO']}"
13
+ resp = EventMachine::DelegatedHttpResponse.new( self )
14
+
15
+ # Block which fulfills the request
16
+ operation = proc do
17
+ sleep DelayFixtureServer.response_delay
18
+
19
+ resp.status = 200
20
+ resp.content = "whatever"
21
+ end
22
+
23
+ # Callback block to execute once the request is fulfilled
24
+ callback = proc do |res|
25
+ resp.send_response
26
+ end
27
+
28
+ # Let the thread pool (20 Ruby threads) handle request
29
+ EM.defer(operation, callback)
30
+ end
31
+
32
+ def self.response_fixture
33
+ @response_fixture ||= ""
34
+ end
35
+
36
+ def self.response_fixture=(val)
37
+ @response_fixture = val
38
+ end
39
+
40
+ def self.response_delay
41
+ @response_delay ||= 0
42
+ end
43
+
44
+ def self.response_delay=(val)
45
+ @response_delay = val
46
+ end
47
+
48
+ def self.reponse_number
49
+ @response_number
50
+ end
51
+
52
+ def self.response_number=(val)
53
+ @response_number = val
54
+ end
55
+ end
56
+
57
+ port = (ARGV[0] || 3000).to_i
58
+
59
+ DelayFixtureServer.response_delay = 0.5
60
+ DelayFixtureServer.response_number = 0
61
+ #DelayFixtureServer.response_fixture = File.read(File.dirname(__FILE__) + "/../fixtures/result_set.xml")
62
+
63
+ EventMachine::run {
64
+ EventMachine.epoll
65
+ EventMachine::start_server("0.0.0.0", port, DelayFixtureServer)
66
+ }
@@ -0,0 +1,51 @@
1
+ # this server simply is for testing out the different http methods. it echoes back the passed in info
2
+ require 'rubygems'
3
+ require 'eventmachine'
4
+ require 'evma_httpserver'
5
+
6
+ class MethodServer < EventMachine::Connection
7
+ include EventMachine::HttpServer
8
+
9
+ def process_http_request
10
+ EventMachine.stop if ENV["PATH_INFO"] == "/die"
11
+
12
+ resp = EventMachine::DelegatedHttpResponse.new( self )
13
+
14
+ # Block which fulfills the request
15
+ operation = proc do
16
+ sleep MethodServer.sleep_time
17
+ resp.status = 200
18
+ resp.content = request_params + "\n#{@http_post_content}"
19
+ end
20
+
21
+ # Callback block to execute once the request is fulfilled
22
+ callback = proc do |res|
23
+ resp.send_response
24
+ end
25
+
26
+ # Let the thread pool (20 Ruby threads) handle request
27
+ EM.defer(operation, callback)
28
+ end
29
+
30
+ def request_params
31
+ %w( PATH_INFO QUERY_STRING HTTP_COOKIE IF_NONE_MATCH CONTENT_TYPE REQUEST_METHOD REQUEST_URI ).collect do |param|
32
+ "#{param}=#{ENV[param]}"
33
+ end.join("\n")
34
+ end
35
+
36
+ def self.sleep_time=(val)
37
+ @sleep_time = val
38
+ end
39
+
40
+ def self.sleep_time
41
+ @sleep_time || 0
42
+ end
43
+ end
44
+ #
45
+ # port = (ARGV[0] || 3000).to_i
46
+ # #Process.fork do
47
+ # EventMachine::run {
48
+ # EventMachine.epoll
49
+ # EventMachine::start_server("0.0.0.0", port, MethodServer)
50
+ # }
51
+ # #end
data/spec/spec.opts ADDED
@@ -0,0 +1,2 @@
1
+ --diff
2
+ --color