typhoeus 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,19 @@
1
+ #ifndef TYPHOEUS_EASY
2
+ #define TYPHOEUS_EASY
3
+
4
+ #include <native.h>
5
+
6
+ void init_typhoeus_easy();
7
+ typedef struct {
8
+ const char *memory;
9
+ int size;
10
+ int read;
11
+ } RequestChunk;
12
+
13
+ typedef struct {
14
+ RequestChunk *request_chunk;
15
+ CURL *curl;
16
+ struct curl_slist *headers;
17
+ } CurlEasy;
18
+
19
+ #endif
@@ -0,0 +1,213 @@
1
+ #include <typhoeus_multi.h>
2
+
3
+ static void multi_read_info(VALUE self, CURLM *multi_handle);
4
+
5
+ static void dealloc(CurlMulti *curl_multi) {
6
+ curl_multi_cleanup(curl_multi->multi);
7
+ free(curl_multi);
8
+ }
9
+
10
+ static VALUE multi_add_handle(VALUE self, VALUE easy) {
11
+ CurlEasy *curl_easy;
12
+ Data_Get_Struct(easy, CurlEasy, curl_easy);
13
+ CurlMulti *curl_multi;
14
+ Data_Get_Struct(self, CurlMulti, curl_multi);
15
+ CURLMcode mcode;
16
+
17
+ mcode = curl_multi_add_handle(curl_multi->multi, curl_easy->curl);
18
+ if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
19
+ rb_raise((VALUE)mcode, "An error occured adding the handle");
20
+ }
21
+
22
+ curl_easy_setopt(curl_easy->curl, CURLOPT_PRIVATE, easy);
23
+ curl_multi->active++;
24
+
25
+ if (mcode == CURLM_CALL_MULTI_PERFORM) {
26
+ curl_multi_perform(curl_multi->multi, &(curl_multi->running));
27
+ }
28
+ //
29
+ // if (curl_multi->running) {
30
+ // printf("call read_info on add<br/>");
31
+ // multi_read_info(self, curl_multi->multi);
32
+ // }
33
+
34
+ return easy;
35
+ }
36
+
37
+ static VALUE multi_remove_handle(VALUE self, VALUE easy) {
38
+ CurlEasy *curl_easy;
39
+ Data_Get_Struct(easy, CurlEasy, curl_easy);
40
+ CurlMulti *curl_multi;
41
+ Data_Get_Struct(self, CurlMulti, curl_multi);
42
+
43
+ curl_multi->active--;
44
+ curl_multi_remove_handle(curl_multi->multi, curl_easy->curl);
45
+
46
+ return easy;
47
+ }
48
+
49
+ static void multi_read_info(VALUE self, CURLM *multi_handle) {
50
+ int msgs_left, result;
51
+ CURLMsg *msg;
52
+ CURLcode ecode;
53
+ CURL *easy_handle;
54
+ VALUE easy;
55
+
56
+ /* check for finished easy handles and remove from the multi handle */
57
+ while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
58
+
59
+ if (msg->msg != CURLMSG_DONE) {
60
+ continue;
61
+ }
62
+
63
+ easy_handle = msg->easy_handle;
64
+ result = msg->data.result;
65
+ if (easy_handle) {
66
+ ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &easy);
67
+ if (ecode != 0) {
68
+ rb_raise(ecode, "error getting easy object");
69
+ }
70
+
71
+ long response_code = -1;
72
+ curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &response_code);
73
+
74
+ // TODO: find out what the real problem is here and fix it.
75
+ // this next bit is a horrible hack. For some reason my tests against a local server on my laptop
76
+ // fail intermittently and return this result number. However, it will succeed if you try it a few
77
+ // more times. Also noteworthy is that this doens't happen when hitting an external server. WTF?!
78
+ if (result == 7) {
79
+ VALUE max_retries = rb_funcall(easy, rb_intern("max_retries?"), 0);
80
+ if (max_retries != Qtrue) {
81
+ multi_remove_handle(self, easy);
82
+ multi_add_handle(self, easy);
83
+ CurlMulti *curl_multi;
84
+ Data_Get_Struct(self, CurlMulti, curl_multi);
85
+ curl_multi_perform(curl_multi->multi, &(curl_multi->running));
86
+
87
+ rb_funcall(easy, rb_intern("increment_retries"), 0);
88
+
89
+ continue;
90
+ }
91
+ }
92
+ multi_remove_handle(self, easy);
93
+
94
+ if (result != 0) {
95
+ rb_funcall(easy, rb_intern("failure"), 0);
96
+ }
97
+ else if ((response_code >= 200 && response_code < 300) || response_code == 0) {
98
+ rb_funcall(easy, rb_intern("success"), 0);
99
+ }
100
+ else if (response_code >= 300 && response_code < 600) {
101
+ rb_funcall(easy, rb_intern("failure"), 0);
102
+ }
103
+ }
104
+ }
105
+ }
106
+
107
+ /* called within ruby_curl_multi_perform */
108
+ static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_running) {
109
+ CURLMcode mcode;
110
+
111
+ do {
112
+ mcode = curl_multi_perform(multi_handle, still_running);
113
+ } while (mcode == CURLM_CALL_MULTI_PERFORM);
114
+
115
+ if (mcode != CURLM_OK) {
116
+ rb_raise((VALUE)mcode, "an error occured while running perform");
117
+ }
118
+
119
+ multi_read_info( self, multi_handle );
120
+ }
121
+
122
+ static VALUE multi_perform(VALUE self) {
123
+ CURLMcode mcode;
124
+ CurlMulti *curl_multi;
125
+ int maxfd, rc;
126
+ fd_set fdread, fdwrite, fdexcep;
127
+
128
+ long timeout;
129
+ struct timeval tv = {0, 0};
130
+
131
+ Data_Get_Struct(self, CurlMulti, curl_multi);
132
+
133
+ rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
134
+ while(curl_multi->running) {
135
+ FD_ZERO(&fdread);
136
+ FD_ZERO(&fdwrite);
137
+ FD_ZERO(&fdexcep);
138
+
139
+ /* get the curl suggested time out */
140
+ mcode = curl_multi_timeout(curl_multi->multi, &timeout);
141
+ if (mcode != CURLM_OK) {
142
+ rb_raise((VALUE)mcode, "an error occured getting the timeout");
143
+ }
144
+
145
+ if (timeout == 0) { /* no delay */
146
+ rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
147
+ continue;
148
+ }
149
+ else if (timeout == -1) {
150
+ timeout = 1;
151
+ }
152
+
153
+ tv.tv_sec = timeout / 1000;
154
+ tv.tv_usec = (timeout * 1000) % 1000000;
155
+
156
+ /* load the fd sets from the multi handle */
157
+ mcode = curl_multi_fdset(curl_multi->multi, &fdread, &fdwrite, &fdexcep, &maxfd);
158
+ if (mcode != CURLM_OK) {
159
+ rb_raise((VALUE)mcode, "an error occured getting the fdset");
160
+ }
161
+
162
+ rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
163
+ if (rc < 0) {
164
+ rb_raise(rb_eRuntimeError, "error on thread select");
165
+ }
166
+ rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
167
+
168
+ }
169
+
170
+ return Qnil;
171
+ }
172
+
173
+ static VALUE active_handle_count(VALUE self) {
174
+ CurlMulti *curl_multi;
175
+ Data_Get_Struct(self, CurlMulti, curl_multi);
176
+
177
+ return INT2NUM(curl_multi->active);
178
+ }
179
+
180
+ static VALUE multi_cleanup(VALUE self) {
181
+ CurlMulti *curl_multi;
182
+ Data_Get_Struct(self, CurlMulti, curl_multi);
183
+
184
+ curl_multi_cleanup(curl_multi->multi);
185
+ curl_multi->active = 0;
186
+ curl_multi->running = 0;
187
+
188
+ return Qnil;
189
+ }
190
+
191
+ static VALUE new(int argc, VALUE *argv, VALUE klass) {
192
+ CurlMulti *curl_multi = ALLOC(CurlMulti);
193
+ curl_multi->multi = curl_multi_init();
194
+ curl_multi->active = 0;
195
+ curl_multi->running = 0;
196
+
197
+ VALUE multi = Data_Wrap_Struct(cTyphoeusMulti, 0, dealloc, curl_multi);
198
+
199
+ rb_obj_call_init(multi, argc, argv);
200
+
201
+ return multi;
202
+ }
203
+
204
+ void init_typhoeus_multi() {
205
+ VALUE klass = cTyphoeusMulti = rb_define_class_under(mTyphoeus, "Multi", rb_cObject);
206
+
207
+ rb_define_singleton_method(klass, "new", new, -1);
208
+ rb_define_private_method(klass, "multi_add_handle", multi_add_handle, 1);
209
+ rb_define_private_method(klass, "multi_remove_handle", multi_remove_handle, 1);
210
+ rb_define_private_method(klass, "multi_perform", multi_perform, 0);
211
+ rb_define_private_method(klass, "multi_cleanup", multi_cleanup, 0);
212
+ rb_define_private_method(klass, "active_handle_count", active_handle_count, 0);
213
+ }
@@ -0,0 +1,16 @@
1
+ #ifndef TYPHOEUS_MULTI
2
+ #define TYPHOEUS_MULTI
3
+
4
+ #include <native.h>
5
+ #include <typhoeus_easy.h>
6
+
7
+ VALUE cTyphoeusMulti;
8
+ typedef struct {
9
+ int running;
10
+ int active;
11
+ CURLM *multi;
12
+ } CurlMulti;
13
+
14
+ void init_typhoeus_multi();
15
+
16
+ #endif
data/lib/typhoeus.rb ADDED
@@ -0,0 +1,55 @@
1
+ $LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
2
+
3
+ require 'rack/utils'
4
+ require 'digest/sha2'
5
+ require 'typhoeus/easy'
6
+ require 'typhoeus/multi'
7
+ require 'typhoeus/native'
8
+ require 'typhoeus/filter'
9
+ require 'typhoeus/remote_method'
10
+ require 'typhoeus/remote'
11
+ require 'typhoeus/remote_proxy_object'
12
+ require 'typhoeus/response'
13
+ require 'typhoeus/request'
14
+ require 'typhoeus/hydra'
15
+
16
+ module Typhoeus
17
+ VERSION = "0.1.2"
18
+
19
+ def self.easy_object_pool
20
+ @easy_objects ||= []
21
+ end
22
+
23
+ def self.init_easy_object_pool
24
+ 20.times do
25
+ easy_object_pool << Typhoeus::Easy.new
26
+ end
27
+ end
28
+
29
+ def self.release_easy_object(easy)
30
+ easy.reset
31
+ easy_object_pool << easy
32
+ end
33
+
34
+ def self.get_easy_object
35
+ if easy_object_pool.empty?
36
+ Typhoeus::Easy.new
37
+ else
38
+ easy_object_pool.pop
39
+ end
40
+ end
41
+
42
+ def self.add_easy_request(easy_object)
43
+ Thread.current[:curl_multi] ||= Typhoeus::Multi.new
44
+ Thread.current[:curl_multi].add(easy_object)
45
+ end
46
+
47
+ def self.perform_easy_requests
48
+ multi = Thread.current[:curl_multi]
49
+ start_time = Time.now
50
+ multi.easy_handles.each do |easy|
51
+ easy.start_time = start_time
52
+ end
53
+ multi.perform
54
+ end
55
+ end
@@ -0,0 +1,210 @@
1
+ module Typhoeus
2
+ class Easy
3
+ attr_reader :response_body, :response_header, :method, :headers, :url
4
+ attr_accessor :start_time
5
+
6
+ CURLINFO_STRING = 1048576
7
+ OPTION_VALUES = {
8
+ :CURLOPT_URL => 10002,
9
+ :CURLOPT_HTTPGET => 80,
10
+ :CURLOPT_HTTPPOST => 10024,
11
+ :CURLOPT_UPLOAD => 46,
12
+ :CURLOPT_CUSTOMREQUEST => 10036,
13
+ :CURLOPT_POSTFIELDS => 10015,
14
+ :CURLOPT_POSTFIELDSIZE => 60,
15
+ :CURLOPT_USERAGENT => 10018,
16
+ :CURLOPT_TIMEOUT_MS => 155,
17
+ :CURLOPT_NOSIGNAL => 99,
18
+ :CURLOPT_HTTPHEADER => 10023,
19
+ :CURLOPT_FOLLOWLOCATION => 52
20
+ }
21
+ INFO_VALUES = {
22
+ :CURLINFO_RESPONSE_CODE => 2097154,
23
+ :CURLINFO_TOTAL_TIME => 3145731
24
+ }
25
+
26
+ def initialize
27
+ @method = :get
28
+ @post_dat_set = nil
29
+ @headers = {}
30
+ end
31
+
32
+ def headers=(hash)
33
+ @headers = hash
34
+ end
35
+
36
+ def total_time_taken
37
+ get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
38
+ end
39
+
40
+ def response_code
41
+ get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
42
+ end
43
+
44
+ def follow_location=(boolean)
45
+ if boolean
46
+ set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 1)
47
+ else
48
+ set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 0)
49
+ end
50
+ end
51
+
52
+ def timeout=(milliseconds)
53
+ @timeout = milliseconds
54
+ set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
55
+ set_option(OPTION_VALUES[:CURLOPT_TIMEOUT_MS], milliseconds)
56
+ end
57
+
58
+ def timed_out?
59
+ @timeout && total_time_taken > @timeout && response_code == 0
60
+ end
61
+
62
+ def request_body=(request_body)
63
+ @request_body = request_body
64
+ if @method == :put
65
+ easy_set_request_body(@request_body)
66
+ headers["Transfer-Encoding"] = ""
67
+ headers["Expect"] = ""
68
+ else
69
+ self.post_data = request_body
70
+ end
71
+ end
72
+
73
+ def user_agent=(user_agent)
74
+ set_option(OPTION_VALUES[:CURLOPT_USERAGENT], user_agent)
75
+ end
76
+
77
+ def url=(url)
78
+ @url = url
79
+ set_option(OPTION_VALUES[:CURLOPT_URL], url)
80
+ end
81
+
82
+ def method=(method)
83
+ @method = method
84
+ if method == :get
85
+ set_option(OPTION_VALUES[:CURLOPT_HTTPGET], 1)
86
+ elsif method == :post
87
+ set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], 1)
88
+ self.post_data = ""
89
+ elsif method == :put
90
+ set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
91
+ self.request_body = "" unless @request_body
92
+ else
93
+ set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], "DELETE")
94
+ end
95
+ end
96
+
97
+ def post_data=(data)
98
+ @post_data_set = true
99
+ set_option(OPTION_VALUES[:CURLOPT_POSTFIELDS], data)
100
+ set_option(OPTION_VALUES[:CURLOPT_POSTFIELDSIZE], data.length)
101
+ end
102
+
103
+ def params=(params)
104
+ params_string = params.keys.collect do |k|
105
+ value = params[k]
106
+ if value.is_a? Hash
107
+ value.keys.collect {|sk| Rack::Utils.escape("#{k}[#{sk}]") + "=" + Rack::Utils.escape(value[sk].to_s)}
108
+ elsif value.is_a? Array
109
+ key = Rack::Utils.escape(k.to_s)
110
+ value.collect { |v| "#{key}=#{Rack::Utils.escape(v.to_s)}" }.join('&')
111
+ else
112
+ "#{Rack::Utils.escape(k.to_s)}=#{Rack::Utils.escape(params[k].to_s)}"
113
+ end
114
+ end.flatten.join("&")
115
+
116
+ if method == :post
117
+ self.post_data = params_string
118
+ else
119
+ self.url = "#{url}?#{params_string}"
120
+ end
121
+ end
122
+
123
+ def set_option(option, value)
124
+ if value.class == String
125
+ easy_setopt_string(option, value)
126
+ else
127
+ easy_setopt_long(option, value)
128
+ end
129
+ end
130
+
131
+ def perform
132
+ set_headers()
133
+ easy_perform()
134
+ response_code()
135
+ end
136
+
137
+ def set_headers
138
+ headers.each_pair do |key, value|
139
+ easy_add_header("#{key}: #{value}")
140
+ end
141
+ easy_set_headers() unless headers.empty?
142
+ end
143
+
144
+ # gets called when finished and response code is 200-299
145
+ def success
146
+ @success.call(self) if @success
147
+ end
148
+
149
+ def on_success(&block)
150
+ @success = block
151
+ end
152
+
153
+ def on_success=(block)
154
+ @success = block
155
+ end
156
+
157
+ # gets called when finished and response code is 300-599
158
+ def failure
159
+ @failure.call(self) if @failure
160
+ end
161
+
162
+ def on_failure(&block)
163
+ @failure = block
164
+ end
165
+
166
+ def on_failure=(block)
167
+ @failure = block
168
+ end
169
+
170
+ def retries
171
+ @retries ||= 0
172
+ end
173
+
174
+ def increment_retries
175
+ @retries ||= 0
176
+ @retries += 1
177
+ end
178
+
179
+ def max_retries
180
+ @max_retries ||= 40
181
+ end
182
+
183
+ def max_retries?
184
+ retries >= max_retries
185
+ end
186
+
187
+ def reset
188
+ @response_code = 0
189
+ @response_header = ""
190
+ @response_body = ""
191
+ easy_reset()
192
+ end
193
+
194
+ def get_info_string(option)
195
+ easy_getinfo_string(option)
196
+ end
197
+
198
+ def get_info_long(option)
199
+ easy_getinfo_long(option)
200
+ end
201
+
202
+ def get_info_double(option)
203
+ easy_getinfo_double(option)
204
+ end
205
+
206
+ def curl_version
207
+ version
208
+ end
209
+ end
210
+ end