curb 0.7.15 → 0.7.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/curl/easy.rb ADDED
@@ -0,0 +1,385 @@
1
+ module Curl
2
+ class Easy
3
+
4
+ #
5
+ # call-seq:
6
+ # easy.set :sym|Fixnum, value
7
+ #
8
+ # set options on the curl easy handle see http://curl.haxx.se/libcurl/c/curl_easy_setopt.html
9
+ #
10
+ def set(opt,val)
11
+ if opt.is_a?(Symbol)
12
+ setopt(sym2curl(opt), val)
13
+ else
14
+ setopt(opt.to_i, val)
15
+ end
16
+ end
17
+
18
+ #
19
+ # call-seq:
20
+ # easy.sym2curl :symbol => Fixnum
21
+ #
22
+ # translates ruby symbols to libcurl options
23
+ #
24
+ def sym2curl(opt)
25
+ Curl.const_get("CURLOPT_#{opt.to_s.upcase}")
26
+ end
27
+
28
+ #
29
+ # call-seq:
30
+ # easy.perform => true
31
+ #
32
+ # Transfer the currently configured URL using the options set for this
33
+ # Curl::Easy instance. If this is an HTTP URL, it will be transferred via
34
+ # the configured HTTP Verb.
35
+ #
36
+ def perform
37
+ self.multi = Curl::Multi.new if self.multi.nil?
38
+ self.multi.add self
39
+ ret = self.multi.perform
40
+
41
+ if self.last_result != 0 && self.on_failure.nil?
42
+ error = Curl::Easy.error(self.last_result)
43
+ raise error.first
44
+ end
45
+
46
+ ret
47
+ end
48
+
49
+ #
50
+ # call-seq:
51
+ #
52
+ # easy = Curl::Easy.new
53
+ # easy.nosignal = true
54
+ #
55
+ def nosignal=(onoff)
56
+ set :nosignal, !!onoff
57
+ end
58
+
59
+ #
60
+ # call-seq:
61
+ # easy = Curl::Easy.new("url") do|c|
62
+ # c.delete = true
63
+ # end
64
+ # easy.perform
65
+ #
66
+ def delete=(onoff)
67
+ set :customrequest, onoff ? 'delete' : nil
68
+ onoff
69
+ end
70
+ #
71
+ # call-seq:
72
+ #
73
+ # easy = Curl::Easy.new("url")
74
+ # easy.version = Curl::HTTP_1_1
75
+ # easy.version = Curl::HTTP_1_0
76
+ # easy.version = Curl::HTTP_NONE
77
+ #
78
+ def version=(http_version)
79
+ set :http_version, http_version
80
+ end
81
+
82
+ #
83
+ # call-seq:
84
+ # easy.url = "http://some.url/" => "http://some.url/"
85
+ #
86
+ # Set the URL for subsequent calls to +perform+. It is acceptable
87
+ # (and even recommended) to reuse Curl::Easy instances by reassigning
88
+ # the URL between calls to +perform+.
89
+ #
90
+ def url=(u)
91
+ set :url, u
92
+ end
93
+
94
+ #
95
+ # call-seq:
96
+ # easy.proxy_url = string => string
97
+ #
98
+ # Set the URL of the HTTP proxy to use for subsequent calls to +perform+.
99
+ # The URL should specify the the host name or dotted IP address. To specify
100
+ # port number in this string, append :[port] to the end of the host name.
101
+ # The proxy string may be prefixed with [protocol]:// since any such prefix
102
+ # will be ignored. The proxy's port number may optionally be specified with
103
+ # the separate option proxy_port .
104
+ #
105
+ # When you tell the library to use an HTTP proxy, libcurl will transparently
106
+ # convert operations to HTTP even if you specify an FTP URL etc. This may have
107
+ # an impact on what other features of the library you can use, such as
108
+ # FTP specifics that don't work unless you tunnel through the HTTP proxy. Such
109
+ # tunneling is activated with proxy_tunnel = true.
110
+ #
111
+ # libcurl respects the environment variables *http_proxy*, *ftp_proxy*,
112
+ # *all_proxy* etc, if any of those is set. The proxy_url option does however
113
+ # override any possibly set environment variables.
114
+ #
115
+ # Starting with libcurl 7.14.1, the proxy host string given in environment
116
+ # variables can be specified the exact same way as the proxy can be set with
117
+ # proxy_url, including protocol prefix (http://) and embedded user + password.
118
+ #
119
+ def proxy_url=(url)
120
+ set :proxy, url
121
+ end
122
+
123
+ def ssl_verify_host=(value)
124
+ value = 1 if value.class == TrueClass
125
+ value = 0 if value.class == FalseClass
126
+ self.ssl_verify_host_integer=value
127
+ end
128
+
129
+ #
130
+ # call-seq:
131
+ # easy.ssl_verify_host? => boolean
132
+ #
133
+ # Deprecated: call easy.ssl_verify_host instead
134
+ # can be one of [0,1,2]
135
+ #
136
+ # Determine whether this Curl instance will verify that the server cert
137
+ # is for the server it is known as.
138
+ #
139
+ def ssl_verify_host?
140
+ ssl_verify_host.nil? ? false : (ssl_verify_host > 0)
141
+ end
142
+
143
+ #
144
+ # call-seq:
145
+ # easy.interface = string => string
146
+ #
147
+ # Set the interface name to use as the outgoing network interface.
148
+ # The name can be an interface name, an IP address or a host name.
149
+ #
150
+ def interface=(value)
151
+ set :interface, value
152
+ end
153
+
154
+ #
155
+ # call-seq:
156
+ # easy.userpwd = string => string
157
+ #
158
+ # Set the username/password string to use for subsequent calls to +perform+.
159
+ # The supplied string should have the form "username:password"
160
+ #
161
+ def userpwd=(value)
162
+ set :userpwd, value
163
+ end
164
+
165
+ #
166
+ # call-seq:
167
+ # easy.proxypwd = string => string
168
+ #
169
+ # Set the username/password string to use for proxy connection during
170
+ # subsequent calls to +perform+. The supplied string should have the
171
+ # form "username:password"
172
+ #
173
+ def proxypwd=(value)
174
+ set :proxyuserpwd, value
175
+ end
176
+
177
+ #
178
+ # call-seq:
179
+ # easy.cookies = "name1=content1; name2=content2;" => string
180
+ #
181
+ # Set cookies to be sent by this Curl::Easy instance. The format of the string should
182
+ # be NAME=CONTENTS, where NAME is the cookie name and CONTENTS is what the cookie should contain.
183
+ # Set multiple cookies in one string like this: "name1=content1; name2=content2;" etc.
184
+ #
185
+ def cookies=(value)
186
+ set :cookie, value
187
+ end
188
+
189
+ #
190
+ # call-seq:
191
+ # easy.cookiefile = string => string
192
+ #
193
+ # Set a file that contains cookies to be sent in subsequent requests by this Curl::Easy instance.
194
+ #
195
+ # *Note* that you must set enable_cookies true to enable the cookie
196
+ # engine, or this option will be ignored.
197
+ #
198
+ def cookiefile=(value)
199
+ set :cookiefile, value
200
+ end
201
+
202
+ #
203
+ # call-seq:
204
+ # easy.cookiejar = string => string
205
+ #
206
+ # Set a cookiejar file to use for this Curl::Easy instance.
207
+ # Cookies from the response will be written into this file.
208
+ #
209
+ # *Note* that you must set enable_cookies true to enable the cookie
210
+ # engine, or this option will be ignored.
211
+ #
212
+ def cookiejar=(value)
213
+ set :cookiejar, value
214
+ end
215
+
216
+ class << self
217
+
218
+ #
219
+ # call-seq:
220
+ # Curl::Easy.perform(url) { |easy| ... } => #&lt;Curl::Easy...&gt;
221
+ #
222
+ # Convenience method that creates a new Curl::Easy instance with
223
+ # the specified URL and calls the general +perform+ method, before returning
224
+ # the new instance. For HTTP URLs, this is equivalent to calling +http_get+.
225
+ #
226
+ # If a block is supplied, the new instance will be yielded just prior to
227
+ # the +http_get+ call.
228
+ #
229
+ def perform(*args)
230
+ c = Curl::Easy.new *args
231
+ yield c if block_given?
232
+ c.perform
233
+ c
234
+ end
235
+
236
+ #
237
+ # call-seq:
238
+ # Curl::Easy.http_get(url) { |easy| ... } => #&lt;Curl::Easy...&gt;
239
+ #
240
+ # Convenience method that creates a new Curl::Easy instance with
241
+ # the specified URL and calls +http_get+, before returning the new instance.
242
+ #
243
+ # If a block is supplied, the new instance will be yielded just prior to
244
+ # the +http_get+ call.
245
+ #
246
+ def http_get(*args)
247
+ c = Curl::Easy.new *args
248
+ yield c if block_given?
249
+ c.http_get
250
+ c
251
+ end
252
+
253
+ #
254
+ # call-seq:
255
+ # Curl::Easy.http_head(url) { |easy| ... } => #&lt;Curl::Easy...&gt;
256
+ #
257
+ # Convenience method that creates a new Curl::Easy instance with
258
+ # the specified URL and calls +http_head+, before returning the new instance.
259
+ #
260
+ # If a block is supplied, the new instance will be yielded just prior to
261
+ # the +http_head+ call.
262
+ #
263
+ def http_head(*args)
264
+ c = Curl::Easy.new *args
265
+ yield c if block_given?
266
+ c.http_head
267
+ c
268
+ end
269
+
270
+ #
271
+ # call-seq:
272
+ # Curl::Easy.http_put(url, data) {|c| ... }
273
+ #
274
+ # see easy.http_put
275
+ #
276
+ def http_put(url, data)
277
+ c = Curl::Easy.new url
278
+ yield c if block_given?
279
+ c.http_put data
280
+ c
281
+ end
282
+
283
+ #
284
+ # call-seq:
285
+ # Curl::Easy.http_post(url, "some=urlencoded%20form%20data&and=so%20on") => true
286
+ # Curl::Easy.http_post(url, "some=urlencoded%20form%20data", "and=so%20on", ...) => true
287
+ # Curl::Easy.http_post(url, "some=urlencoded%20form%20data", Curl::PostField, "and=so%20on", ...) => true
288
+ # Curl::Easy.http_post(url, Curl::PostField, Curl::PostField ..., Curl::PostField) => true
289
+ #
290
+ # POST the specified formdata to the currently configured URL using
291
+ # the current options set for this Curl::Easy instance. This method
292
+ # always returns true, or raises an exception (defined under
293
+ # Curl::Err) on error.
294
+ #
295
+ # If you wish to use multipart form encoding, you'll need to supply a block
296
+ # in order to set multipart_form_post true. See #http_post for more
297
+ # information.
298
+ #
299
+ def http_post(*args)
300
+ url = args.shift
301
+ c = Curl::Easy.new url
302
+ yield c if block_given?
303
+ c.http_post *args
304
+ c
305
+ end
306
+
307
+ #
308
+ # call-seq:
309
+ # Curl::Easy.http_delete(url) { |easy| ... } => #&lt;Curl::Easy...&gt;
310
+ #
311
+ # Convenience method that creates a new Curl::Easy instance with
312
+ # the specified URL and calls +http_delete+, before returning the new instance.
313
+ #
314
+ # If a block is supplied, the new instance will be yielded just prior to
315
+ # the +http_delete+ call.
316
+ #
317
+ def http_delete(*args)
318
+ c = Curl::Easy.new *args
319
+ yield c if block_given?
320
+ c.http_delete
321
+ c
322
+ end
323
+
324
+ # call-seq:
325
+ # Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
326
+ #
327
+ # Stream the specified url (via perform) and save the data directly to the
328
+ # supplied filename (defaults to the last component of the URL path, which will
329
+ # usually be the filename most simple urls).
330
+ #
331
+ # If a block is supplied, it will be passed the curl instance prior to the
332
+ # perform call.
333
+ #
334
+ # *Note* that the semantics of the on_body handler are subtly changed when using
335
+ # download, to account for the automatic routing of data to the specified file: The
336
+ # data string is passed to the handler *before* it is written
337
+ # to the file, allowing the handler to perform mutative operations where
338
+ # necessary. As usual, the transfer will be aborted if the on_body handler
339
+ # returns a size that differs from the data chunk size - in this case, the
340
+ # offending chunk will *not* be written to the file, the file will be closed,
341
+ # and a Curl::Err::AbortedByCallbackError will be raised.
342
+ def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
343
+ curl = Curl::Easy.new(url, &blk)
344
+
345
+ output = if filename.is_a? IO
346
+ filename.binmode if filename.respond_to?(:binmode)
347
+ filename
348
+ else
349
+ File.open(filename, 'wb')
350
+ end
351
+
352
+ begin
353
+ old_on_body = curl.on_body do |data|
354
+ result = old_on_body ? old_on_body.call(data) : data.length
355
+ output << data if result == data.length
356
+ result
357
+ end
358
+ curl.perform
359
+ ensure
360
+ output.close rescue IOError
361
+ end
362
+
363
+ return curl
364
+ end
365
+ end
366
+
367
+ # Allow the incoming cert string to be file:password
368
+ # but be careful to not use a colon from a windows file path
369
+ # as the split point. Mimic what curl's main does
370
+ if respond_to?(:cert=)
371
+ alias_method :native_cert=, :cert=
372
+ def cert=(cert_file)
373
+ pos = cert_file.rindex(':')
374
+ if pos && pos > 1
375
+ self.native_cert= cert_file[0..pos-1]
376
+ self.certpassword= cert_file[pos+1..-1]
377
+ else
378
+ self.native_cert= cert_file
379
+ end
380
+ self.cert
381
+ end
382
+ end
383
+
384
+ end
385
+ end
data/lib/curl/multi.rb ADDED
@@ -0,0 +1,244 @@
1
+ module Curl
2
+
3
+ class Multi
4
+ class << self
5
+ # call-seq:
6
+ # Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
7
+ # easy
8
+ # end
9
+ #
10
+ # Blocking call to fetch multiple url's in parallel.
11
+ def get(urls, easy_options={}, multi_options={}, &blk)
12
+ url_confs = []
13
+ urls.each do|url|
14
+ url_confs << {:url => url, :method => :get}.merge(easy_options)
15
+ end
16
+ self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk }
17
+ end
18
+
19
+ # call-seq:
20
+ #
21
+ # Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
22
+ # {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
23
+ # {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
24
+ # { :follow_location => true, :multipart_form_post => true },
25
+ # {:pipeline => true }) do|easy|
26
+ # easy_handle_on_request_complete
27
+ # end
28
+ #
29
+ # Blocking call to POST multiple form's in parallel.
30
+ #
31
+ # urls_with_config: is a hash of url's pointing to the postfields to send
32
+ # easy_options: are a set of common options to set on all easy handles
33
+ # multi_options: options to set on the Curl::Multi handle
34
+ #
35
+ def post(urls_with_config, easy_options={}, multi_options={}, &blk)
36
+ url_confs = []
37
+ urls_with_config.each do|uconf|
38
+ url_confs << uconf.merge(:method => :post).merge(easy_options)
39
+ end
40
+ self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
41
+ end
42
+
43
+ # call-seq:
44
+ #
45
+ # Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
46
+ # {:url => 'url2', :put_data => IO.read('filepath')},
47
+ # {:url => 'url3', :put_data => "maybe another string or socket?"],
48
+ # {:follow_location => true},
49
+ # {:pipeline => true }) do|easy|
50
+ # easy_handle_on_request_complete
51
+ # end
52
+ #
53
+ # Blocking call to POST multiple form's in parallel.
54
+ #
55
+ # urls_with_config: is a hash of url's pointing to the postfields to send
56
+ # easy_options: are a set of common options to set on all easy handles
57
+ # multi_options: options to set on the Curl::Multi handle
58
+ #
59
+ def put(urls_with_config, easy_options={}, multi_options={}, &blk)
60
+ url_confs = []
61
+ urls_with_config.each do|uconf|
62
+ url_confs << uconf.merge(:method => :put).merge(easy_options)
63
+ end
64
+ self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
65
+ end
66
+
67
+
68
+ # call-seq:
69
+ #
70
+ # Curl::Multi.http( [
71
+ # { :url => 'url1', :method => :post,
72
+ # :post_fields => {'field1' => 'value1', 'field2' => 'value2'} },
73
+ # { :url => 'url2', :method => :get,
74
+ # :follow_location => true, :max_redirects => 3 },
75
+ # { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
76
+ # { :url => 'url4', :method => :head }
77
+ # ], {:pipeline => true})
78
+ #
79
+ # Blocking call to issue multiple HTTP requests with varying verb's.
80
+ #
81
+ # urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used
82
+ # multi_options: options for the multi handle
83
+ # blk: a callback, that yeilds when a handle is completed
84
+ #
85
+ def http(urls_with_config, multi_options={}, &blk)
86
+ m = Curl::Multi.new
87
+
88
+ # maintain a sane number of easy handles
89
+ multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10
90
+
91
+ free_handles = [] # keep a list of free easy handles
92
+
93
+ # configure the multi handle
94
+ multi_options.each { |k,v| m.send("#{k}=", v) }
95
+ callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]
96
+
97
+ add_free_handle = proc do|conf, easy|
98
+ c = conf.dup # avoid being destructive to input
99
+ url = c.delete(:url)
100
+ method = c.delete(:method)
101
+ headers = c.delete(:headers)
102
+
103
+ easy = Curl::Easy.new if easy.nil?
104
+
105
+ easy.url = url
106
+
107
+ # assign callbacks
108
+ callbacks.each do |cb|
109
+ cbproc = c.delete(cb)
110
+ easy.send(cb,&cbproc) if cbproc
111
+ end
112
+
113
+ case method
114
+ when :post
115
+ fields = c.delete(:post_fields)
116
+ # set the post post using the url fields
117
+ easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
118
+ when :put
119
+ easy.put_data = c.delete(:put_data)
120
+ when :head
121
+ easy.head = true
122
+ when :delete
123
+ easy.delete = true
124
+ when :get
125
+ else
126
+ # XXX: nil is treated like a GET
127
+ end
128
+
129
+ # headers is a special key
130
+ headers.each {|k,v| easy.headers[k] = v } if headers
131
+
132
+ #
133
+ # use the remaining options as specific configuration to the easy handle
134
+ # bad options should raise an undefined method error
135
+ #
136
+ c.each { |k,v| easy.send("#{k}=",v) }
137
+
138
+ easy.on_complete {|curl,code|
139
+ free_handles << curl
140
+ blk.call(curl,code,method) if blk
141
+ }
142
+ m.add(easy)
143
+ end
144
+
145
+ max_connects.times do
146
+ conf = urls_with_config.pop
147
+ add_free_handle.call conf, nil
148
+ break if urls_with_config.empty?
149
+ end
150
+
151
+ consume_free_handles = proc do
152
+ # as we idle consume free handles
153
+ if urls_with_config.size > 0 && free_handles.size > 0
154
+ easy = free_handles.pop
155
+ conf = urls_with_config.pop
156
+ add_free_handle.call conf, easy
157
+ end
158
+ end
159
+
160
+ until urls_with_config.empty?
161
+ m.perform do
162
+ consume_free_handles.call
163
+ end
164
+ consume_free_handles.call
165
+ end
166
+ free_handles = nil
167
+ end
168
+
169
+ # call-seq:
170
+ #
171
+ # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
172
+ #
173
+ # will create 2 new files file1.txt and file2.txt
174
+ #
175
+ # 2 files will be opened, and remain open until the call completes
176
+ #
177
+ # when using the :post or :put method, urls should be a hash, including the individual post fields per post
178
+ #
179
+ def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
180
+ errors = []
181
+ procs = []
182
+ files = []
183
+ urls_with_config = []
184
+ url_to_download_paths = {}
185
+
186
+ urls.each_with_index do|urlcfg,i|
187
+ if urlcfg.is_a?(Hash)
188
+ url = url[:url]
189
+ else
190
+ url = urlcfg
191
+ end
192
+
193
+ if download_paths and download_paths[i]
194
+ download_path = download_paths[i]
195
+ else
196
+ download_path = File.basename(url)
197
+ end
198
+
199
+ file = lambda do|dp|
200
+ file = File.open(dp,"wb")
201
+ procs << (lambda {|data| file.write data; data.size })
202
+ files << file
203
+ file
204
+ end.call(download_path)
205
+
206
+ if urlcfg.is_a?(Hash)
207
+ urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
208
+ else
209
+ urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
210
+ end
211
+ url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
212
+ end
213
+
214
+ if blk
215
+ # when injecting the block, ensure file is closed before yielding
216
+ Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
217
+ info = url_to_download_paths[c.url]
218
+ begin
219
+ file = info[:file]
220
+ files.reject!{|f| f == file }
221
+ file.close
222
+ rescue => e
223
+ errors << e
224
+ end
225
+ blk.call(c,info[:path])
226
+ end
227
+ else
228
+ Curl::Multi.http(urls_with_config, multi_options)
229
+ end
230
+
231
+ ensure
232
+ files.each {|f|
233
+ begin
234
+ f.close
235
+ rescue => e
236
+ errors << e
237
+ end
238
+ }
239
+ raise errors unless errors.empty?
240
+ end
241
+
242
+ end
243
+ end
244
+ end