hhry-typhoeus 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
2
+
3
+ require 'rspec/core/rake_task'
4
+ RSpec::Core::RakeTask.new do |t|
5
+ end
6
+
7
+ task :install do
8
+ rm_rf "*.gem"
9
+ puts `gem build typhoeus.gemspec`
10
+ puts `gem install typhoeus-*.gem`
11
+ end
12
+
13
+ desc "Start up the test servers"
14
+ task :start_test_servers do
15
+ require 'spec/support/typhoeus_localhost_server'
16
+ begin
17
+ TyphoeusLocalhostServer.start_servers!(:rake)
18
+ rescue Exception
19
+ end
20
+ end
21
+
22
+ desc "Build Typhoeus and run all the tests."
23
+ task :default => [:spec]
@@ -0,0 +1,57 @@
1
+ require 'digest/sha2'
2
+ require 'active_support/core_ext'
3
+
4
+ require 'typhoeus/utils'
5
+ require 'typhoeus/header'
6
+ require 'typhoeus/curl'
7
+ require 'typhoeus/easy'
8
+ require 'typhoeus/form'
9
+ require 'typhoeus/multi'
10
+ require 'typhoeus/filter'
11
+ require 'typhoeus/param_processor'
12
+ require 'typhoeus/remote'
13
+ require 'typhoeus/remote_proxy_object'
14
+ require 'typhoeus/response'
15
+ require 'typhoeus/request'
16
+ require 'typhoeus/hydra'
17
+ require 'typhoeus/hydra_mock'
18
+ require 'typhoeus/version'
19
+
20
+ module Typhoeus
21
+ def self.easy_object_pool
22
+ @easy_objects ||= []
23
+ end
24
+
25
+ def self.init_easy_object_pool
26
+ 20.times do
27
+ easy_object_pool << Typhoeus::Easy.new
28
+ end
29
+ end
30
+
31
+ def self.release_easy_object(easy)
32
+ easy.reset
33
+ easy_object_pool << easy
34
+ end
35
+
36
+ def self.get_easy_object
37
+ if easy_object_pool.empty?
38
+ Typhoeus::Easy.new
39
+ else
40
+ easy_object_pool.pop
41
+ end
42
+ end
43
+
44
+ def self.add_easy_request(easy_object)
45
+ Thread.current[:curl_multi] ||= Typhoeus::Multi.new
46
+ Thread.current[:curl_multi].add(easy_object)
47
+ end
48
+
49
+ def self.perform_easy_requests
50
+ multi = Thread.current[:curl_multi]
51
+ start_time = Time.now
52
+ multi.easy_handles.each do |easy|
53
+ easy.start_time = start_time
54
+ end
55
+ multi.perform
56
+ end
57
+ end
@@ -0,0 +1,453 @@
1
+ require 'ffi'
2
+ require 'rbconfig'
3
+ require 'thread'
4
+
5
+ module Typhoeus
6
+ module Curl
7
+ # this does not implement the full curl lib. just what is needed for typhoeus
8
+
9
+ def Curl.windows?
10
+ !(RbConfig::CONFIG['host_os'] !~ /mingw|mswin|bccwin/)
11
+ end
12
+
13
+ extend ::FFI::Library
14
+
15
+ VERSION_NOW = 3
16
+
17
+ GLOBAL_SSL = 0x01
18
+ GLOBAL_WIN32 = 0x02
19
+ GLOBAL_ALL = (GLOBAL_SSL | GLOBAL_WIN32)
20
+ GLOBAL_DEFAULT = GLOBAL_ALL
21
+
22
+ EasyCode = enum :easy_code, [
23
+ :ok,
24
+ :unsupported_protocol,
25
+ :failed_init,
26
+ :url_malformat,
27
+ :not_built_in,
28
+ :couldnt_resolve_proxy,
29
+ :couldnt_resolve_host,
30
+ :couldnt_connect,
31
+ :ftp_weird_server_reply,
32
+ :remote_access_denied,
33
+ :ftp_accept_failed,
34
+ :ftp_weird_pass_reply,
35
+ :ftp_accept_timeout,
36
+ :ftp_weird_pasv_reply,
37
+ :ftp_weird_227_format,
38
+ :ftp_cant_get_host,
39
+ :obsolete16,
40
+ :ftp_couldnt_set_type,
41
+ :partial_file,
42
+ :ftp_couldnt_retr_file,
43
+ :obsolete20,
44
+ :quote_error,
45
+ :http_returned_error,
46
+ :write_error,
47
+ :obsolete24,
48
+ :upload_failed,
49
+ :read_error,
50
+ :out_of_memory,
51
+ :operation_timedout,
52
+ :obsolete29,
53
+ :ftp_port_failed,
54
+ :ftp_couldnt_use_rest,
55
+ :obsolete32,
56
+ :range_error,
57
+ :http_post_error,
58
+ :ssl_connect_error,
59
+ :bad_download_resume,
60
+ :file_couldnt_read_file,
61
+ :ldap_cannot_bind,
62
+ :ldap_search_failed,
63
+ :obsolete40,
64
+ :function_not_found,
65
+ :aborted_by_callback,
66
+ :bad_function_argument,
67
+ :obsolete44,
68
+ :interface_failed,
69
+ :obsolete46,
70
+ :too_many_redirects ,
71
+ :unknown_option,
72
+ :telnet_option_syntax ,
73
+ :obsolete50,
74
+ :peer_failed_verification,
75
+ :got_nothing,
76
+ :ssl_engine_notfound,
77
+ :ssl_engine_setfailed,
78
+ :send_error,
79
+ :recv_error,
80
+ :obsolete57,
81
+ :ssl_certproblem,
82
+ :ssl_cipher,
83
+ :ssl_cacert,
84
+ :bad_content_encoding,
85
+ :ldap_invalid_url,
86
+ :filesize_exceeded,
87
+ :use_ssl_failed,
88
+ :send_fail_rewind,
89
+ :ssl_engine_initfailed,
90
+ :login_denied,
91
+ :tftp_notfound,
92
+ :tftp_perm,
93
+ :remote_disk_full,
94
+ :tftp_illegal,
95
+ :tftp_unknownid,
96
+ :remote_file_exists,
97
+ :tftp_nosuchuser,
98
+ :conv_failed,
99
+ :conv_reqd,
100
+ :ssl_cacert_badfile,
101
+ :remote_file_not_found,
102
+ :ssh,
103
+ :ssl_shutdown_failed,
104
+ :again,
105
+ :ssl_crl_badfile,
106
+ :ssl_issuer_error,
107
+ :ftp_pret_failed,
108
+ :rtsp_cseq_error,
109
+ :rtsp_session_error,
110
+ :ftp_bad_file_list,
111
+ :chunk_failed,
112
+ :last]
113
+
114
+ MultiCode = enum :multi_code, [
115
+ :call_multi_perform, -1,
116
+ :ok,
117
+ :bad_handle,
118
+ :bad_easy_handle,
119
+ :out_of_memory,
120
+ :internal_error,
121
+ :bad_socket,
122
+ :unknown_option,
123
+ :last]
124
+
125
+ OptionType = enum [
126
+ :long, 0,
127
+ :object_point, 10000,
128
+ :function_point, 20000,
129
+ :off_t, 30000]
130
+
131
+ Option = enum :option, [
132
+ :file, OptionType[:object_point] + 1,
133
+ :writedata, OptionType[:object_point] + 1,
134
+ :url, OptionType[:object_point] + 2,
135
+ :port, OptionType[:long] + 3,
136
+ :proxy, OptionType[:object_point] + 4,
137
+ :userpwd, OptionType[:object_point] + 5,
138
+ :proxyuserpwd, OptionType[:object_point] + 6,
139
+ :range, OptionType[:object_point] + 7,
140
+ :infile, OptionType[:object_point] + 9,
141
+ :readdata, OptionType[:object_point] + 9,
142
+ :errorbuffer, OptionType[:object_point] + 10,
143
+ :writefunction, OptionType[:function_point] + 11,
144
+ :readfunction, OptionType[:function_point] + 12,
145
+ :timeout, OptionType[:long] + 13,
146
+ :infilesize, OptionType[:long] + 14,
147
+ :postfields, OptionType[:object_point] + 15,
148
+ :referer, OptionType[:object_point] + 16,
149
+ :ftpport, OptionType[:object_point] + 17,
150
+ :useragent, OptionType[:object_point] + 18,
151
+ :low_speed_time, OptionType[:long] + 20,
152
+ :resume_from, OptionType[:long] + 21,
153
+ :cookie, OptionType[:object_point] + 22,
154
+ :httpheader, OptionType[:object_point] + 23,
155
+ :httppost, OptionType[:object_point] + 24,
156
+ :sslcert, OptionType[:object_point] + 25,
157
+ :sslcertpasswd, OptionType[:object_point] + 26,
158
+ :sslkeypasswd, OptionType[:object_point] + 26,
159
+ :crlf, OptionType[:long] + 27,
160
+ :quote, OptionType[:object_point] + 28,
161
+ :writeheader, OptionType[:object_point] + 29,
162
+ :headerdata, OptionType[:object_point] + 29,
163
+ :cookiefile, OptionType[:object_point] + 31,
164
+ :sslversion, OptionType[:long] + 32,
165
+ :timecondition, OptionType[:long] + 33,
166
+ :timevalue, OptionType[:long] + 34,
167
+ :customrequest, OptionType[:object_point] + 36,
168
+ :stderr, OptionType[:object_point] + 37,
169
+ :postquote, OptionType[:object_point] + 39,
170
+ :writeinfo, OptionType[:object_point] + 40,
171
+ :verbose, OptionType[:long] + 41,
172
+ :header, OptionType[:long] + 42,
173
+ :noprogress, OptionType[:long] + 43,
174
+ :nobody, OptionType[:long] + 44,
175
+ :failonerror, OptionType[:long] + 45,
176
+ :upload, OptionType[:long] + 46,
177
+ :post, OptionType[:long] + 47,
178
+ :ftplistonly, OptionType[:long] + 48,
179
+ :ftpappend, OptionType[:long] + 50,
180
+ :netrc, OptionType[:long] + 51,
181
+ :followlocation, OptionType[:long] + 52,
182
+ :transfertext, OptionType[:long] + 53,
183
+ :put, OptionType[:long] + 54,
184
+ :progressfunction, OptionType[:function_point] + 56,
185
+ :progressdata, OptionType[:object_point] + 57,
186
+ :autoreferer, OptionType[:long] + 58,
187
+ :proxyport, OptionType[:long] + 59,
188
+ :postfieldsize, OptionType[:long] + 60,
189
+ :httpproxytunnel, OptionType[:long] + 61,
190
+ :interface, OptionType[:object_point] + 62,
191
+ :ssl_verifypeer, OptionType[:long] + 64,
192
+ :cainfo, OptionType[:object_point] + 65,
193
+ :maxredirs, OptionType[:long] + 68,
194
+ :filetime, OptionType[:long] + 69,
195
+ :telnetoptions, OptionType[:object_point] + 70,
196
+ :maxconnects, OptionType[:long] + 71,
197
+ :closepolicy, OptionType[:long] + 72,
198
+ :fresh_connect, OptionType[:long] + 74,
199
+ :forbid_reuse, OptionType[:long] + 75,
200
+ :random_file, OptionType[:object_point] + 76,
201
+ :egdsocket, OptionType[:object_point] + 77,
202
+ :connecttimeout, OptionType[:long] + 78,
203
+ :headerfunction, OptionType[:function_point] + 79,
204
+ :httpget, OptionType[:long] + 80,
205
+ :ssl_verifyhost, OptionType[:long] + 81,
206
+ :cookiejar, OptionType[:object_point] + 82,
207
+ :ssl_cipher_list, OptionType[:object_point] + 83,
208
+ :http_version, OptionType[:long] + 84,
209
+ :ftp_use_epsv, OptionType[:long] + 85,
210
+ :sslcerttype, OptionType[:object_point] + 86,
211
+ :sslkey, OptionType[:object_point] + 87,
212
+ :sslkeytype, OptionType[:object_point] + 88,
213
+ :sslengine, OptionType[:object_point] + 89,
214
+ :sslengine_default, OptionType[:long] + 90,
215
+ :dns_use_global_cache, OptionType[:long] + 91,
216
+ :dns_cache_timeout, OptionType[:long] + 92,
217
+ :prequote, OptionType[:object_point] + 93,
218
+ :debugfunction, OptionType[:function_point] + 94,
219
+ :debugdata, OptionType[:object_point] + 95,
220
+ :cookiesession, OptionType[:long] + 96,
221
+ :capath, OptionType[:object_point] + 97,
222
+ :buffersize, OptionType[:long] + 98,
223
+ :nosignal, OptionType[:long] + 99,
224
+ :share, OptionType[:object_point] + 100,
225
+ :proxytype, OptionType[:long] + 101,
226
+ :encoding, OptionType[:object_point] + 102,
227
+ :private, OptionType[:object_point] + 103,
228
+ :unrestricted_auth, OptionType[:long] + 105,
229
+ :ftp_use_eprt, OptionType[:long] + 106,
230
+ :httpauth, OptionType[:long] + 107,
231
+ :ssl_ctx_function, OptionType[:function_point] + 108,
232
+ :ssl_ctx_data, OptionType[:object_point] + 109,
233
+ :ftp_create_missing_dirs, OptionType[:long] + 110,
234
+ :proxyauth, OptionType[:long] + 111,
235
+ :ipresolve, OptionType[:long] + 113,
236
+ :maxfilesize, OptionType[:long] + 114,
237
+ :infilesize_large, OptionType[:off_t] + 115,
238
+ :resume_from_large, OptionType[:off_t] + 116,
239
+ :maxfilesize_large, OptionType[:off_t] + 117,
240
+ :netrc_file, OptionType[:object_point] + 118,
241
+ :ftp_ssl, OptionType[:long] + 119,
242
+ :postfieldsize_large, OptionType[:off_t] + 120,
243
+ :tcp_nodelay, OptionType[:long] + 121,
244
+ :ftpsslauth, OptionType[:long] + 129,
245
+ :ioctlfunction, OptionType[:function_point] + 130,
246
+ :ioctldata, OptionType[:object_point] + 131,
247
+ :ftp_account, OptionType[:object_point] + 134,
248
+ :cookielist, OptionType[:object_point] + 135,
249
+ :ignore_content_length, OptionType[:long] + 136,
250
+ :ftp_skip_pasv_ip, OptionType[:long] + 137,
251
+ :ftp_filemethod, OptionType[:long] + 138,
252
+ :localport, OptionType[:long] + 139,
253
+ :localportrange, OptionType[:long] + 140,
254
+ :connect_only, OptionType[:long] + 141,
255
+ :conv_from_network_function, OptionType[:function_point] + 142,
256
+ :conv_to_network_function, OptionType[:function_point] + 143,
257
+ :max_send_speed_large, OptionType[:off_t] + 145,
258
+ :max_recv_speed_large, OptionType[:off_t] + 146,
259
+ :ftp_alternative_to_user, OptionType[:object_point] + 147,
260
+ :sockoptfunction, OptionType[:function_point] + 148,
261
+ :sockoptdata, OptionType[:object_point] + 149,
262
+ :ssl_sessionid_cache, OptionType[:long] + 150,
263
+ :ssh_auth_types, OptionType[:long] + 151,
264
+ :ssh_public_keyfile, OptionType[:object_point] + 152,
265
+ :ssh_private_keyfile, OptionType[:object_point] + 153,
266
+ :ftp_ssl_ccc, OptionType[:long] + 154,
267
+ :timeout_ms, OptionType[:long] + 155,
268
+ :connecttimeout_ms, OptionType[:long] + 156,
269
+ :http_transfer_decoding, OptionType[:long] + 157,
270
+ :http_content_decoding, OptionType[:long] + 158,
271
+ :copypostfields, OptionType[:object_point] + 165]
272
+
273
+ InfoType = enum [
274
+ :string, 0x100000,
275
+ :long, 0x200000,
276
+ :double, 0x300000,
277
+ :slist, 0x400000]
278
+
279
+ Info = enum :info, [
280
+ :effective_url, InfoType[:string] + 1,
281
+ :response_code, InfoType[:long] + 2,
282
+ :total_time, InfoType[:double] + 3,
283
+ :namelookup_time, InfoType[:double] + 4,
284
+ :connect_time, InfoType[:double] + 5,
285
+ :pretransfer_time, InfoType[:double] + 6,
286
+ :size_upload, InfoType[:double] + 7,
287
+ :size_download, InfoType[:double] + 8,
288
+ :speed_download, InfoType[:double] + 9,
289
+ :speed_upload, InfoType[:double] + 10,
290
+ :header_size, InfoType[:long] + 11,
291
+ :request_size, InfoType[:long] + 12,
292
+ :ssl_verifyresult, InfoType[:long] + 13,
293
+ :filetime, InfoType[:long] + 14,
294
+ :content_length_download, InfoType[:double] + 15,
295
+ :content_length_upload, InfoType[:double] + 16,
296
+ :starttransfer_time, InfoType[:double] + 17,
297
+ :content_type, InfoType[:string] + 18,
298
+ :redirect_time, InfoType[:double] + 19,
299
+ :redirect_count, InfoType[:long] + 20,
300
+ :private, InfoType[:string] + 21,
301
+ :http_connectcode, InfoType[:long] + 22,
302
+ :httpauth_avail, InfoType[:long] + 23,
303
+ :proxyauth_avail, InfoType[:long] + 24,
304
+ :os_errno, InfoType[:long] + 25,
305
+ :num_connects, InfoType[:long] + 26,
306
+ :ssl_engines, InfoType[:slist] + 27,
307
+ :cookielist, InfoType[:slist] + 28,
308
+ :lastsocket, InfoType[:long] + 29,
309
+ :ftp_entry_path, InfoType[:string] + 30,
310
+ :redirect_url, InfoType[:string] + 31,
311
+ :primary_ip, InfoType[:string] + 32,
312
+ :appconnect_time, InfoType[:double] + 33,
313
+ :certinfo, InfoType[:slist] + 34,
314
+ :condition_unmet, InfoType[:long] + 35,
315
+ :rtsp_session_id, InfoType[:string] + 36,
316
+ :rtsp_client_cseq, InfoType[:long] + 37,
317
+ :rtsp_server_cseq, InfoType[:long] + 38,
318
+ :rtsp_cseq_recv, InfoType[:long] + 39,
319
+ :primary_port, InfoType[:long] + 40,
320
+ :local_ip, InfoType[:string] + 41,
321
+ :local_port, InfoType[:long] + 42,
322
+ :last, 42]
323
+
324
+ FormOption = enum :form_option, [
325
+ :none,
326
+ :copyname,
327
+ :ptrname,
328
+ :namelength,
329
+ :copycontents,
330
+ :ptrcontents,
331
+ :contentslength,
332
+ :filecontent,
333
+ :array,
334
+ :obsolete,
335
+ :file,
336
+ :buffer,
337
+ :bufferptr,
338
+ :bufferlength,
339
+ :contenttype,
340
+ :contentheader,
341
+ :filename,
342
+ :end,
343
+ :obsolete2,
344
+ :stream,
345
+ :last]
346
+
347
+ Auth = enum [
348
+ :basic, 0x01,
349
+ :digest, 0x02,
350
+ :gssnegotiate, 0x04,
351
+ :ntlm, 0x08,
352
+ :digest_ie, 0x10,
353
+ :auto, 0x1f] # all options or'd together
354
+
355
+ Proxy = enum [
356
+ :http, 0,
357
+ :http_1_0, 1,
358
+ :socks4, 4,
359
+ :socks5, 5,
360
+ :socks4a, 6]
361
+
362
+ SSLVersion = enum [
363
+ :default, 0,
364
+ :tlsv1, 1,
365
+ :sslv2, 2,
366
+ :sslv3, 3]
367
+
368
+ MsgCode = enum :msg_code, [:none, :done, :last]
369
+
370
+ class MsgData < ::FFI::Union
371
+ layout :whatever, :pointer,
372
+ :code, :easy_code
373
+ end
374
+
375
+ class Msg < ::FFI::Struct
376
+ layout :code, :msg_code,
377
+ :easy_handle, :pointer,
378
+ :data, MsgData
379
+ end
380
+
381
+ class FDSet < ::FFI::Struct
382
+ # XXX how does this work on non-windows? how can curl know the new size...
383
+ FD_SETSIZE = 524288 # set a higher maximum number of fds. this has never applied to windows, so just use the default there
384
+
385
+ if Curl.windows?
386
+ layout :fd_count, :u_int,
387
+ :fd_array, [:u_int, 64] # 2048 FDs
388
+
389
+ def clear; self[:fd_count] = 0; end
390
+ else
391
+ layout :fds_bits, [:long, FD_SETSIZE / ::FFI::Type::LONG.size]
392
+
393
+ def clear; super; end
394
+ end
395
+ end
396
+
397
+ class Timeval < ::FFI::Struct
398
+ layout :sec, :time_t,
399
+ :usec, :suseconds_t
400
+ end
401
+
402
+ callback :callback, [:pointer, :size_t, :size_t, :pointer], :size_t
403
+
404
+ ffi_lib_flags :now, :global
405
+ ffi_lib 'libcurl'
406
+
407
+ attach_function :global_init, :curl_global_init, [:long], :int
408
+
409
+ attach_function :easy_init, :curl_easy_init, [], :pointer
410
+ attach_function :easy_cleanup, :curl_easy_cleanup, [:pointer], :void
411
+ attach_function :easy_getinfo, :curl_easy_getinfo, [:pointer, :info, :pointer], :easy_code
412
+ attach_function :easy_setopt, :curl_easy_setopt, [:pointer, :option, :pointer], :easy_code
413
+ attach_function :easy_setopt_string, :curl_easy_setopt, [:pointer, :option, :string], :easy_code
414
+ attach_function :easy_setopt_long, :curl_easy_setopt, [:pointer, :option, :long], :easy_code
415
+ attach_function :easy_setopt_callback, :curl_easy_setopt, [:pointer, :option, :callback], :easy_code
416
+ attach_function :easy_perform, :curl_easy_perform, [:pointer], :easy_code
417
+ attach_function :easy_strerror, :curl_easy_strerror, [:int], :string
418
+ attach_function :easy_escape, :curl_easy_escape, [:pointer, :pointer, :int], :string
419
+ attach_function :easy_reset, :curl_easy_reset, [:pointer], :void
420
+
421
+ attach_function :formadd, :curl_formadd, [:pointer, :pointer, :varargs], :int
422
+
423
+ attach_function :multi_init, :curl_multi_init, [], :pointer
424
+ attach_function :multi_add_handle, :curl_multi_add_handle, [:pointer, :pointer], :multi_code
425
+ attach_function :multi_remove_handle, :curl_multi_remove_handle, [:pointer, :pointer], :multi_code
426
+ attach_function :multi_info_read, :curl_multi_info_read, [:pointer, :pointer], Msg.ptr
427
+ attach_function :multi_perform, :curl_multi_perform, [:pointer, :pointer], :multi_code
428
+ attach_function :multi_timeout, :curl_multi_timeout, [:pointer, :pointer], :multi_code
429
+ attach_function :multi_fdset, :curl_multi_fdset, [:pointer, FDSet.ptr, FDSet.ptr, FDSet.ptr, :pointer], :multi_code
430
+ attach_function :multi_strerror, :curl_multi_strerror, [:int], :string
431
+
432
+ attach_function :version, :curl_version, [], :string
433
+ attach_function :slist_append, :curl_slist_append, [:pointer, :string], :pointer
434
+ attach_function :slist_free_all, :curl_slist_free_all, [:pointer], :void
435
+
436
+ ffi_lib (windows? ? 'ws2_32' : ::FFI::Library::LIBC)
437
+ @blocking = true
438
+ attach_function :select, [:int, FDSet.ptr, FDSet.ptr, FDSet.ptr, Timeval.ptr], :int
439
+
440
+ @@initialized = false
441
+ @@init_mutex = Mutex.new
442
+
443
+ def Curl.init
444
+ # ensure curl lib is initialised. not thread-safe so must be wrapped in a mutex
445
+ @@init_mutex.synchronize {
446
+ if not @@initialized
447
+ raise RuntimeError.new('curl failed to initialise') if Curl.global_init(GLOBAL_ALL) != 0
448
+ @@initialized = true
449
+ end
450
+ }
451
+ end
452
+ end
453
+ end