curb 0.7.15 → 0.7.16

Sign up to get free protection for your applications and to get access to all the features.
data/ext/curb_easy.h CHANGED
@@ -56,6 +56,7 @@ typedef struct {
56
56
  long ssl_version;
57
57
  long use_ssl;
58
58
  long ftp_filemethod;
59
+ unsigned short resolve_mode;
59
60
 
60
61
  /* bool flags */
61
62
  char proxy_tunnel;
data/ext/curb_errors.c CHANGED
@@ -28,6 +28,7 @@ VALUE eCurlErrUnsupportedProtocol;
28
28
  VALUE eCurlErrFailedInit;
29
29
  VALUE eCurlErrMalformedURL;
30
30
  VALUE eCurlErrMalformedURLUser;
31
+ VALUE eCurlErrNotBuiltIn;
31
32
  VALUE eCurlErrProxyResolution;
32
33
  VALUE eCurlErrHostResolution;
33
34
  VALUE eCurlErrConnectFailed;
@@ -141,9 +142,15 @@ VALUE rb_curl_easy_error(CURLcode code) {
141
142
  case CURLE_URL_MALFORMAT: /* 3 */
142
143
  exclz = eCurlErrMalformedURL;
143
144
  break;
145
+ #ifdef HAVE_CURLE_NOT_BUILT_IN
146
+ case CURLE_NOT_BUILT_IN: /* 4 - [was obsoleted in August 2007 for 7.17.0, reused in April 2011 for 7.21.5] */
147
+ exclz = eCurlErrNotBuiltIn;
148
+ break;
149
+ #else
144
150
  case CURLE_URL_MALFORMAT_USER: /* 4 (NOT USED) */
145
151
  exclz = eCurlErrMalformedURLUser;
146
152
  break;
153
+ #endif
147
154
  case CURLE_COULDNT_RESOLVE_PROXY: /* 5 */
148
155
  exclz = eCurlErrProxyResolution;
149
156
  break;
@@ -518,6 +525,7 @@ void init_curb_errors() {
518
525
  eCurlErrUnsupportedProtocol = rb_define_class_under(mCurlErr, "UnsupportedProtocolError", eCurlErrError);
519
526
  eCurlErrFailedInit = rb_define_class_under(mCurlErr, "FailedInitError", eCurlErrError);
520
527
  eCurlErrMalformedURL = rb_define_class_under(mCurlErr, "MalformedURLError", eCurlErrError);
528
+ eCurlErrNotBuiltIn = rb_define_class_under(mCurlErr, "NotBuiltInError", eCurlErrError);
521
529
  eCurlErrMalformedURLUser = rb_define_class_under(mCurlErr, "MalformedURLUserError", eCurlErrError);
522
530
  eCurlErrProxyResolution = rb_define_class_under(mCurlErr, "ProxyResolutionError", eCurlErrError);
523
531
  eCurlErrHostResolution = rb_define_class_under(mCurlErr, "HostResolutionError", eCurlErrError);
data/ext/curb_macros.h CHANGED
@@ -8,6 +8,7 @@
8
8
  #ifndef __CURB_MACROS_H
9
9
  #define __CURB_MACROS_H
10
10
 
11
+ #define rb_easy_sym(sym) ID2SYM(rb_intern(sym))
11
12
  #define rb_easy_hkey(key) ID2SYM(rb_intern(key))
12
13
  #define rb_easy_set(key,val) rb_hash_aset(rbce->opts, rb_easy_hkey(key) , val)
13
14
  #define rb_easy_get(key) rb_hash_aref(rbce->opts, rb_easy_hkey(key))
@@ -152,4 +153,7 @@
152
153
  return INT2FIX(ptr->attr); \
153
154
  }
154
155
 
156
+ #define CURB_DEFINE(name) \
157
+ rb_define_const(mCurl, #name, INT2FIX(name))
158
+
155
159
  #endif
data/ext/curb_multi.c CHANGED
@@ -3,7 +3,6 @@
3
3
  * Licensed under the Ruby License. See LICENSE for details.
4
4
  *
5
5
  */
6
-
7
6
  #include "curb_config.h"
8
7
  #ifdef HAVE_RUBY19_ST_H
9
8
  #include <ruby.h>
@@ -39,13 +38,12 @@ static void rb_curl_multi_remove(ruby_curl_multi *rbcm, VALUE easy);
39
38
  static void rb_curl_multi_read_info(VALUE self, CURLM *mptr);
40
39
  static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_running);
41
40
 
42
- static void rb_curl_multi_mark_all_easy(VALUE key, VALUE rbeasy, ruby_curl_multi *rbcm) {
43
- rb_gc_mark(rbeasy);
44
- }
41
+ static VALUE callback_exception(VALUE unused) {
42
+ return Qfalse;
43
+ }
45
44
 
46
45
  static void curl_multi_mark(ruby_curl_multi *rbcm) {
47
46
  rb_gc_mark(rbcm->requests);
48
- rb_hash_foreach( rbcm->requests, (int (*)())rb_curl_multi_mark_all_easy, (VALUE)rbcm );
49
47
  }
50
48
 
51
49
  static void curl_multi_flush_easy(VALUE key, VALUE easy, ruby_curl_multi *rbcm) {
@@ -245,8 +243,6 @@ VALUE ruby_curl_multi_add(VALUE self, VALUE easy) {
245
243
 
246
244
  rb_hash_aset( rbcm->requests, easy, easy );
247
245
 
248
- rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
249
-
250
246
  return self;
251
247
  }
252
248
 
@@ -323,11 +319,20 @@ static VALUE ruby_curl_multi_cancel(VALUE self) {
323
319
  return self;
324
320
  }
325
321
 
322
+ // on_success, on_failure, on_complete
323
+ static VALUE call_status_handler1(VALUE ary) {
324
+ return rb_funcall(rb_ary_entry(ary, 0), idCall, 1, rb_ary_entry(ary, 1));
325
+ }
326
+ static VALUE call_status_handler2(VALUE ary) {
327
+ return rb_funcall(rb_ary_entry(ary, 0), idCall, 2, rb_ary_entry(ary, 1), rb_ary_entry(ary, 2));
328
+ }
329
+
326
330
  static void rb_curl_mutli_handle_complete(VALUE self, CURL *easy_handle, int result) {
327
331
 
328
332
  long response_code = -1;
329
333
  VALUE easy;
330
334
  ruby_curl_easy *rbce = NULL;
335
+ VALUE callargs;
331
336
 
332
337
  CURLcode ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, (char**)&easy);
333
338
 
@@ -348,24 +353,32 @@ static void rb_curl_mutli_handle_complete(VALUE self, CURL *easy_handle, int res
348
353
  }
349
354
 
350
355
  if (!rb_easy_nil("complete_proc")) {
351
- rb_funcall( rb_easy_get("complete_proc"), idCall, 1, easy );
356
+ callargs = rb_ary_new3(2, rb_easy_get("complete_proc"), easy);
357
+ rb_rescue(call_status_handler1, callargs, callback_exception, Qnil);
358
+ //rb_funcall( rb_easy_get("complete_proc"), idCall, 1, easy );
352
359
  }
353
360
 
354
361
  curl_easy_getinfo(rbce->curl, CURLINFO_RESPONSE_CODE, &response_code);
355
362
 
356
363
  if (result != 0) {
357
364
  if (!rb_easy_nil("failure_proc")) {
358
- rb_funcall( rb_easy_get("failure_proc"), idCall, 2, easy, rb_curl_easy_error(result) );
365
+ callargs = rb_ary_new3(3, rb_easy_get("failure_proc"), easy, rb_curl_easy_error(result));
366
+ rb_rescue(call_status_handler2, callargs, callback_exception, Qnil);
367
+ //rb_funcall( rb_easy_get("failure_proc"), idCall, 2, easy, rb_curl_easy_error(result) );
359
368
  }
360
369
  }
361
370
  else if (!rb_easy_nil("success_proc") &&
362
371
  ((response_code >= 200 && response_code < 300) || response_code == 0)) {
363
372
  /* NOTE: we allow response_code == 0, in the case of non http requests e.g. reading from disk */
364
- rb_funcall( rb_easy_get("success_proc"), idCall, 1, easy );
373
+ callargs = rb_ary_new3(2, rb_easy_get("success_proc"), easy);
374
+ rb_rescue(call_status_handler1, callargs, callback_exception, Qnil);
375
+ //rb_funcall( rb_easy_get("success_proc"), idCall, 1, easy );
365
376
  }
366
377
  else if (!rb_easy_nil("failure_proc") &&
367
378
  (response_code >= 300 && response_code <= 999)) {
368
- rb_funcall( rb_easy_get("failure_proc"), idCall, 2, easy, rb_curl_easy_error(result) );
379
+ callargs = rb_ary_new3(3, rb_easy_get("failure_proc"), easy, rb_curl_easy_error(result));
380
+ rb_rescue(call_status_handler2, callargs, callback_exception, Qnil);
381
+ //rb_funcall( rb_easy_get("failure_proc"), idCall, 2, easy, rb_curl_easy_error(result) );
369
382
  }
370
383
 
371
384
  }
@@ -465,6 +478,8 @@ VALUE ruby_curl_multi_perform(int argc, VALUE *argv, VALUE self) {
465
478
  timeout_milliseconds = cCurlMutiDefaulttimeout;
466
479
 
467
480
  rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
481
+ rb_curl_multi_read_info( self, rbcm->handle );
482
+ if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self); }
468
483
 
469
484
  do {
470
485
  while (rbcm->running) {
data/ext/extconf.rb CHANGED
@@ -35,16 +35,17 @@ end
35
35
  # puts "Selected arch: #{archs.first}"
36
36
  #end
37
37
 
38
- def define(s)
39
- $defs.push( format("-D HAVE_%s", s.to_s.upcase) )
38
+ def define(s, v=1)
39
+ $defs.push( format("-D HAVE_%s=%d", s.to_s.upcase, v) )
40
40
  end
41
41
 
42
42
  def have_constant(name)
43
+ sname = name.is_a?(Symbol) ? name.to_s : name.upcase
43
44
  checking_for name do
44
45
  src = %{
45
46
  #include <curl/curl.h>
46
47
  int main() {
47
- int test = (int)#{name.upcase};
48
+ int test = (int)#{sname};
48
49
  return 0;
49
50
  }
50
51
  }
@@ -52,6 +53,7 @@ def have_constant(name)
52
53
  define name
53
54
  true
54
55
  else
56
+ #define name, 0
55
57
  false
56
58
  end
57
59
  end
@@ -130,6 +132,213 @@ have_func("curl_multi_timeout")
130
132
  have_func("curl_multi_fdset")
131
133
  have_func("curl_multi_perform")
132
134
 
135
+ # constants
136
+ have_constant "curlopt_interleavefunction"
137
+ have_constant "curlopt_interleavedata"
138
+ have_constant "curlopt_chunk_bgn_function"
139
+ have_constant "curlopt_chunk_end_function"
140
+ have_constant "curlopt_chunk_data"
141
+ have_constant "curlopt_fnmatch_function"
142
+ have_constant "curlopt_fnmatch_data"
143
+ have_constant "curlopt_errorbuffer"
144
+ have_constant "curlopt_stderr"
145
+ have_constant "curlopt_failonerror"
146
+ have_constant "curlopt_url"
147
+ have_constant "curlopt_protocols"
148
+ have_constant "curlopt_redir_protocols"
149
+ have_constant "curlopt_proxy"
150
+ have_constant "curlopt_proxyport"
151
+ have_constant "curlopt_proxytype"
152
+ have_constant "curlopt_noproxy"
153
+ have_constant "curlopt_httpproxytunnel"
154
+ have_constant "curlopt_socks5_gssapi_service"
155
+ have_constant "curlopt_socks5_gssapi_nec"
156
+ have_constant "curlopt_interface"
157
+ have_constant "curlopt_localport"
158
+ have_constant "curlopt_dns_cache_timeout"
159
+ have_constant "curlopt_dns_use_global_cache"
160
+ have_constant "curlopt_buffersize"
161
+ have_constant "curlopt_port"
162
+ have_constant "curlopt_tcp_nodelay"
163
+ have_constant "curlopt_address_scope"
164
+ have_constant "curlopt_netrc"
165
+ have_constant "curl_netrc_optional"
166
+ have_constant "curl_netrc_ignored"
167
+ have_constant "curl_netrc_required"
168
+ have_constant "curlopt_netrc_file"
169
+ have_constant "curlopt_userpwd"
170
+ have_constant "curlopt_proxyuserpwd"
171
+ have_constant "curlopt_username"
172
+ have_constant "curlopt_password"
173
+ have_constant "curlopt_password"
174
+ have_constant "curlopt_password"
175
+ have_constant "curlopt_httpauth"
176
+ have_constant "curlauth_digest_ie"
177
+ have_constant "curlauth_only"
178
+ have_constant "curlopt_tlsauth_type"
179
+ have_constant "curlopt_tlsauth_srp"
180
+ have_constant "curlopt_tlsauth_username"
181
+ have_constant "curlopt_tlsauth_password"
182
+ have_constant "curlopt_proxyauth"
183
+ have_constant "curlopt_autoreferer"
184
+ have_constant "curlopt_encoding"
185
+ have_constant "curlopt_followlocation"
186
+ have_constant "curlopt_unrestricted_auth"
187
+ have_constant "curlopt_maxredirs"
188
+ have_constant "curlopt_postredir"
189
+ have_constant "curlopt_put"
190
+ have_constant "curlopt_post"
191
+ have_constant "curlopt_postfields"
192
+ have_constant "curlopt_postfieldsize"
193
+ have_constant "curlopt_postfieldsize_large"
194
+ have_constant "curlopt_copypostfields"
195
+ have_constant "curlopt_httppost"
196
+ have_constant "curlopt_referer"
197
+ have_constant "curlopt_useragent"
198
+ have_constant "curlopt_httpheader"
199
+ have_constant "curlopt_http200aliases"
200
+ have_constant "curlopt_cookie"
201
+ have_constant "curlopt_cookiefile"
202
+ have_constant "curlopt_cookiejar"
203
+ have_constant "curlopt_cookiesession"
204
+ have_constant "curlopt_cookielist"
205
+ have_constant "curlopt_httpget"
206
+ have_constant "curlopt_http_version"
207
+ have_constant "curl_http_version_none"
208
+ have_constant "curl_http_version_1_0"
209
+ have_constant "curl_http_version_1_1"
210
+ have_constant "curlopt_ignore_content_length"
211
+ have_constant "curlopt_http_content_decoding"
212
+ have_constant "curlopt_http_transfer_decoding"
213
+ have_constant "curlopt_mail_from"
214
+ have_constant "curlopt_mail_rcpt"
215
+ have_constant "curlopt_tftp_blksize"
216
+ have_constant "curlopt_ftpport"
217
+ have_constant "curlopt_quote"
218
+ have_constant "curlopt_postquote"
219
+ have_constant "curlopt_prequote"
220
+ have_constant "curlopt_dirlistonly"
221
+ have_constant "curlopt_append"
222
+ have_constant "curlopt_ftp_use_eprt"
223
+ have_constant "curlopt_ftp_use_epsv"
224
+ have_constant "curlopt_ftp_use_pret"
225
+ have_constant "curlopt_ftp_create_missing_dirs"
226
+ have_constant "curlopt_ftp_response_timeout"
227
+ have_constant "curlopt_ftp_alternative_to_user"
228
+ have_constant "curlopt_ftp_skip_pasv_ip"
229
+ have_constant "curlopt_ftpsslauth"
230
+ have_constant "curlftpauth_default"
231
+ have_constant "curlftpauth_ssl"
232
+ have_constant "curlftpauth_tls"
233
+ have_constant "curlopt_ftp_ssl_ccc"
234
+ have_constant "curlftpssl_ccc_none"
235
+ have_constant "curlftpssl_ccc_passive"
236
+ have_constant "curlftpssl_ccc_active"
237
+ have_constant "curlopt_ftp_account"
238
+ have_constant "curlopt_ftp_filemethod"
239
+ have_constant "curlftpmethod_multicwd"
240
+ have_constant "curlftpmethod_nocwd"
241
+ have_constant "curlftpmethod_singlecwd"
242
+ have_constant "curlopt_rtsp_request"
243
+ have_constant "curl_rtspreq_options"
244
+ have_constant "curl_rtspreq_describe"
245
+ have_constant "curl_rtspreq_announce"
246
+ have_constant "curl_rtspreq_setup"
247
+ have_constant "curl_rtspreq_play"
248
+ have_constant "curl_rtspreq_pause"
249
+ have_constant "curl_rtspreq_teardown"
250
+ have_constant "curl_rtspreq_get_parameter"
251
+ have_constant "curl_rtspreq_set_parameter"
252
+ have_constant "curl_rtspreq_record"
253
+ have_constant "curl_rtspreq_receive"
254
+ have_constant "curlopt_rtsp_session_id"
255
+ have_constant "curlopt_rtsp_stream_uri"
256
+ have_constant "curlopt_rtsp_transport"
257
+ have_constant "curlopt_rtsp_header"
258
+ have_constant "curlopt_rtsp_client_cseq"
259
+ have_constant "curlopt_rtsp_server_cseq"
260
+ have_constant "curlopt_transfertext"
261
+ have_constant "curlopt_proxy_transfer_mode"
262
+ have_constant "curlopt_crlf"
263
+ have_constant "curlopt_range"
264
+ have_constant "curlopt_resume_from"
265
+ have_constant "curlopt_resume_from_large"
266
+ have_constant "curlopt_customrequest"
267
+ have_constant "curlopt_filetime"
268
+ have_constant "curlopt_nobody"
269
+ have_constant "curlopt_infilesize"
270
+ have_constant "curlopt_infilesize_large"
271
+ have_constant "curlopt_upload"
272
+ have_constant "curlopt_maxfilesize"
273
+ have_constant "curlopt_maxfilesize_large"
274
+ have_constant "curlopt_timecondition"
275
+ have_constant "curlopt_timevalue"
276
+ have_constant "curlopt_timeout"
277
+ have_constant "curlopt_timeout_ms"
278
+ have_constant "curlopt_low_speed_limit"
279
+ have_constant "curlopt_low_speed_time"
280
+ have_constant "curlopt_max_send_speed_large"
281
+ have_constant "curlopt_max_recv_speed_large"
282
+ have_constant "curlopt_maxconnects"
283
+ have_constant "curlopt_closepolicy"
284
+ have_constant "curlopt_fresh_connect"
285
+ have_constant "curlopt_forbid_reuse"
286
+ have_constant "curlopt_connecttimeout"
287
+ have_constant "curlopt_connecttimeout_ms"
288
+ have_constant "curlopt_ipresolve"
289
+ have_constant "curl_ipresolve_whatever"
290
+ have_constant "curl_ipresolve_v4"
291
+ have_constant "curl_ipresolve_v6"
292
+ have_constant "curlopt_connect_only"
293
+ have_constant "curlopt_use_ssl"
294
+ have_constant "curlusessl_none"
295
+ have_constant "curlusessl_try"
296
+ have_constant "curlusessl_control"
297
+ have_constant "curlusessl_all"
298
+ have_constant "curlopt_resolve"
299
+ have_constant "curlopt_sslcert"
300
+ have_constant "curlopt_sslcerttype"
301
+ have_constant "curlopt_sslkey"
302
+ have_constant "curlopt_sslkeytype"
303
+ have_constant "curlopt_keypasswd"
304
+ have_constant "curlopt_sslengine"
305
+ have_constant "curlopt_sslengine_default"
306
+ have_constant "curlopt_sslversion"
307
+ have_constant "curl_sslversion_default"
308
+ have_constant :CURL_SSLVERSION_TLSv1
309
+ have_constant :CURL_SSLVERSION_SSLv2
310
+ have_constant :CURL_SSLVERSION_SSLv3
311
+ have_constant "curlopt_ssl_verifypeer"
312
+ have_constant "curlopt_cainfo"
313
+ have_constant "curlopt_issuercert"
314
+ have_constant "curlopt_capath"
315
+ have_constant "curlopt_crlfile"
316
+ have_constant "curlopt_ssl_verifyhost"
317
+ have_constant "curlopt_certinfo"
318
+ have_constant "curlopt_random_file"
319
+ have_constant "curlopt_egdsocket"
320
+ have_constant "curlopt_ssl_cipher_list"
321
+ have_constant "curlopt_ssl_sessionid_cache"
322
+ have_constant "curlopt_krblevel"
323
+ have_constant "curlopt_ssh_auth_types"
324
+ have_constant "curlopt_ssh_host_public_key_md5"
325
+ have_constant "curlopt_ssh_public_keyfile"
326
+ have_constant "curlopt_ssh_private_keyfile"
327
+ have_constant "curlopt_ssh_knownhosts"
328
+ have_constant "curlopt_ssh_keyfunction"
329
+ have_constant "curlkhstat_fine_add_to_file"
330
+ have_constant "curlkhstat_fine"
331
+ have_constant "curlkhstat_reject"
332
+ have_constant "curlkhstat_defer"
333
+ have_constant "curlopt_ssh_keydata"
334
+ have_constant "curlopt_private"
335
+ have_constant "curlopt_share"
336
+ have_constant "curlopt_new_file_perms"
337
+ have_constant "curlopt_new_directory_perms"
338
+ have_constant "curlopt_telnetoptions"
339
+ # was obsoleted in August 2007 for 7.17.0, reused in April 2011 for 7.21.5
340
+ have_constant "curle_not_built_in"
341
+
133
342
  if try_compile('int main() { return 0; }','-Wall')
134
343
  $CFLAGS << ' -Wall'
135
344
  end
data/lib/curb.rb CHANGED
@@ -1,308 +1,3 @@
1
1
  require 'curb_core'
2
-
3
- module Curl
4
-
5
- class Easy
6
- class << self
7
-
8
- # call-seq:
9
- # Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
10
- #
11
- # Stream the specified url (via perform) and save the data directly to the
12
- # supplied filename (defaults to the last component of the URL path, which will
13
- # usually be the filename most simple urls).
14
- #
15
- # If a block is supplied, it will be passed the curl instance prior to the
16
- # perform call.
17
- #
18
- # *Note* that the semantics of the on_body handler are subtly changed when using
19
- # download, to account for the automatic routing of data to the specified file: The
20
- # data string is passed to the handler *before* it is written
21
- # to the file, allowing the handler to perform mutative operations where
22
- # necessary. As usual, the transfer will be aborted if the on_body handler
23
- # returns a size that differs from the data chunk size - in this case, the
24
- # offending chunk will *not* be written to the file, the file will be closed,
25
- # and a Curl::Err::AbortedByCallbackError will be raised.
26
- def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
27
- curl = Curl::Easy.new(url, &blk)
28
-
29
- output = if filename.is_a? IO
30
- filename.binmode if filename.respond_to?(:binmode)
31
- filename
32
- else
33
- File.open(filename, 'wb')
34
- end
35
-
36
- begin
37
- old_on_body = curl.on_body do |data|
38
- result = old_on_body ? old_on_body.call(data) : data.length
39
- output << data if result == data.length
40
- result
41
- end
42
- curl.perform
43
- ensure
44
- output.close rescue IOError
45
- end
46
-
47
- return curl
48
- end
49
- end
50
-
51
- # Allow the incoming cert string to be file:password
52
- # but be careful to not use a colon from a windows file path
53
- # as the split point. Mimic what curl's main does
54
- alias_method :native_cert=, :cert=
55
- def cert=(cert_file)
56
- pos = cert_file.rindex(':')
57
- if pos && pos > 1
58
- self.native_cert= cert_file[0..pos-1]
59
- self.certpassword= cert_file[pos+1..-1]
60
- else
61
- self.native_cert= cert_file
62
- end
63
- self.cert
64
- end
65
- end
66
-
67
- class Multi
68
- class << self
69
- # call-seq:
70
- # Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
71
- # easy
72
- # end
73
- #
74
- # Blocking call to fetch multiple url's in parallel.
75
- def get(urls, easy_options={}, multi_options={}, &blk)
76
- url_confs = []
77
- urls.each do|url|
78
- url_confs << {:url => url, :method => :get}.merge(easy_options)
79
- end
80
- self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk }
81
- end
82
-
83
- # call-seq:
84
- #
85
- # Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
86
- # {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
87
- # {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
88
- # { :follow_location => true, :multipart_form_post => true },
89
- # {:pipeline => true }) do|easy|
90
- # easy_handle_on_request_complete
91
- # end
92
- #
93
- # Blocking call to POST multiple form's in parallel.
94
- #
95
- # urls_with_config: is a hash of url's pointing to the postfields to send
96
- # easy_options: are a set of common options to set on all easy handles
97
- # multi_options: options to set on the Curl::Multi handle
98
- #
99
- def post(urls_with_config, easy_options={}, multi_options={}, &blk)
100
- url_confs = []
101
- urls_with_config.each do|uconf|
102
- url_confs << uconf.merge(:method => :post).merge(easy_options)
103
- end
104
- self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
105
- end
106
-
107
- # call-seq:
108
- #
109
- # Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
110
- # {:url => 'url2', :put_data => IO.read('filepath')},
111
- # {:url => 'url3', :put_data => "maybe another string or socket?"],
112
- # {:follow_location => true},
113
- # {:pipeline => true }) do|easy|
114
- # easy_handle_on_request_complete
115
- # end
116
- #
117
- # Blocking call to POST multiple form's in parallel.
118
- #
119
- # urls_with_config: is a hash of url's pointing to the postfields to send
120
- # easy_options: are a set of common options to set on all easy handles
121
- # multi_options: options to set on the Curl::Multi handle
122
- #
123
- def put(urls_with_config, easy_options={}, multi_options={}, &blk)
124
- url_confs = []
125
- urls_with_config.each do|uconf|
126
- url_confs << uconf.merge(:method => :put).merge(easy_options)
127
- end
128
- self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
129
- end
130
-
131
-
132
- # call-seq:
133
- #
134
- # Curl::Multi.http( [
135
- # { :url => 'url1', :method => :post,
136
- # :post_fields => {'field1' => 'value1', 'field2' => 'value2'} },
137
- # { :url => 'url2', :method => :get,
138
- # :follow_location => true, :max_redirects => 3 },
139
- # { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
140
- # { :url => 'url4', :method => :head }
141
- # ], {:pipeline => true})
142
- #
143
- # Blocking call to issue multiple HTTP requests with varying verb's.
144
- #
145
- # urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used
146
- # multi_options: options for the multi handle
147
- # blk: a callback, that yeilds when a handle is completed
148
- #
149
- def http(urls_with_config, multi_options={}, &blk)
150
- m = Curl::Multi.new
151
-
152
- # maintain a sane number of easy handles
153
- multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10
154
-
155
- free_handles = [] # keep a list of free easy handles
156
-
157
- # configure the multi handle
158
- multi_options.each { |k,v| m.send("#{k}=", v) }
159
- callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]
160
-
161
- add_free_handle = proc do|conf, easy|
162
- c = conf.dup # avoid being destructive to input
163
- url = c.delete(:url)
164
- method = c.delete(:method)
165
- headers = c.delete(:headers)
166
-
167
- easy = Curl::Easy.new if easy.nil?
168
-
169
- easy.url = url
170
-
171
- # assign callbacks
172
- callbacks.each do |cb|
173
- cbproc = c.delete(cb)
174
- easy.send(cb,&cbproc) if cbproc
175
- end
176
-
177
- case method
178
- when :post
179
- fields = c.delete(:post_fields)
180
- # set the post post using the url fields
181
- easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
182
- when :put
183
- easy.put_data = c.delete(:put_data)
184
- when :head
185
- easy.head = true
186
- when :delete
187
- easy.delete = true
188
- when :get
189
- else
190
- # XXX: nil is treated like a GET
191
- end
192
-
193
- # headers is a special key
194
- headers.each {|k,v| easy.headers[k] = v } if headers
195
-
196
- #
197
- # use the remaining options as specific configuration to the easy handle
198
- # bad options should raise an undefined method error
199
- #
200
- c.each { |k,v| easy.send("#{k}=",v) }
201
-
202
- easy.on_complete {|curl,code|
203
- free_handles << curl
204
- blk.call(curl,code,method) if blk
205
- }
206
- m.add(easy)
207
- end
208
-
209
- max_connects.times do
210
- conf = urls_with_config.pop
211
- add_free_handle.call conf, nil
212
- break if urls_with_config.empty?
213
- end
214
-
215
- consume_free_handles = proc do
216
- # as we idle consume free handles
217
- if urls_with_config.size > 0 && free_handles.size > 0
218
- easy = free_handles.pop
219
- conf = urls_with_config.pop
220
- add_free_handle.call conf, easy
221
- end
222
- end
223
-
224
- until urls_with_config.empty?
225
- m.perform do
226
- consume_free_handles.call
227
- end
228
- consume_free_handles.call
229
- end
230
- free_handles = nil
231
- end
232
-
233
- # call-seq:
234
- #
235
- # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
236
- #
237
- # will create 2 new files file1.txt and file2.txt
238
- #
239
- # 2 files will be opened, and remain open until the call completes
240
- #
241
- # when using the :post or :put method, urls should be a hash, including the individual post fields per post
242
- #
243
- def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
244
- errors = []
245
- procs = []
246
- files = []
247
- urls_with_config = []
248
- url_to_download_paths = {}
249
-
250
- urls.each_with_index do|urlcfg,i|
251
- if urlcfg.is_a?(Hash)
252
- url = url[:url]
253
- else
254
- url = urlcfg
255
- end
256
-
257
- if download_paths and download_paths[i]
258
- download_path = download_paths[i]
259
- else
260
- download_path = File.basename(url)
261
- end
262
-
263
- file = lambda do|dp|
264
- file = File.open(dp,"wb")
265
- procs << (lambda {|data| file.write data; data.size })
266
- files << file
267
- file
268
- end.call(download_path)
269
-
270
- if urlcfg.is_a?(Hash)
271
- urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
272
- else
273
- urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
274
- end
275
- url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
276
- end
277
-
278
- if blk
279
- # when injecting the block, ensure file is closed before yielding
280
- Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
281
- info = url_to_download_paths[c.url]
282
- begin
283
- file = info[:file]
284
- files.reject!{|f| f == file }
285
- file.close
286
- rescue => e
287
- errors << e
288
- end
289
- blk.call(c,info[:path])
290
- end
291
- else
292
- Curl::Multi.http(urls_with_config, multi_options)
293
- end
294
-
295
- ensure
296
- files.each {|f|
297
- begin
298
- f.close
299
- rescue => e
300
- errors << e
301
- end
302
- }
303
- raise errors unless errors.empty?
304
- end
305
-
306
- end
307
- end
308
- end
2
+ require 'curl/easy'
3
+ require 'curl/multi'