chimera 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/History.txt +4 -0
  2. data/Manifest.txt +57 -0
  3. data/PostInstall.txt +7 -0
  4. data/README.rdoc +114 -0
  5. data/Rakefile +30 -0
  6. data/doc/NOTES +11 -0
  7. data/doc/examples/config.yml +16 -0
  8. data/doc/redis6379.conf +132 -0
  9. data/lib/chimera.rb +33 -0
  10. data/lib/chimera/associations.rb +146 -0
  11. data/lib/chimera/attributes.rb +52 -0
  12. data/lib/chimera/base.rb +95 -0
  13. data/lib/chimera/config.rb +9 -0
  14. data/lib/chimera/error.rb +12 -0
  15. data/lib/chimera/finders.rb +49 -0
  16. data/lib/chimera/geo_indexes.rb +76 -0
  17. data/lib/chimera/indexes.rb +177 -0
  18. data/lib/chimera/persistence.rb +70 -0
  19. data/lib/chimera/redis_objects.rb +345 -0
  20. data/lib/redis.rb +373 -0
  21. data/lib/redis/counter.rb +94 -0
  22. data/lib/redis/dist_redis.rb +149 -0
  23. data/lib/redis/hash_ring.rb +135 -0
  24. data/lib/redis/helpers/core_commands.rb +46 -0
  25. data/lib/redis/helpers/serialize.rb +25 -0
  26. data/lib/redis/list.rb +122 -0
  27. data/lib/redis/lock.rb +83 -0
  28. data/lib/redis/objects.rb +100 -0
  29. data/lib/redis/objects/counters.rb +132 -0
  30. data/lib/redis/objects/lists.rb +45 -0
  31. data/lib/redis/objects/locks.rb +71 -0
  32. data/lib/redis/objects/sets.rb +46 -0
  33. data/lib/redis/objects/values.rb +56 -0
  34. data/lib/redis/pipeline.rb +21 -0
  35. data/lib/redis/set.rb +156 -0
  36. data/lib/redis/value.rb +35 -0
  37. data/lib/riak_raw.rb +100 -0
  38. data/lib/typhoeus.rb +55 -0
  39. data/lib/typhoeus/.gitignore +1 -0
  40. data/lib/typhoeus/easy.rb +253 -0
  41. data/lib/typhoeus/filter.rb +28 -0
  42. data/lib/typhoeus/hydra.rb +210 -0
  43. data/lib/typhoeus/multi.rb +34 -0
  44. data/lib/typhoeus/remote.rb +306 -0
  45. data/lib/typhoeus/remote_method.rb +108 -0
  46. data/lib/typhoeus/remote_proxy_object.rb +48 -0
  47. data/lib/typhoeus/request.rb +124 -0
  48. data/lib/typhoeus/response.rb +39 -0
  49. data/lib/typhoeus/service.rb +20 -0
  50. data/script/console +10 -0
  51. data/script/destroy +14 -0
  52. data/script/generate +14 -0
  53. data/test/models.rb +49 -0
  54. data/test/test_chimera.rb +238 -0
  55. data/test/test_helper.rb +7 -0
  56. metadata +243 -0
@@ -0,0 +1,55 @@
1
+ $LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
2
+
3
+ require 'rack/utils'
4
+ require 'digest/sha2'
5
+ require 'typhoeus/easy'
6
+ require 'typhoeus/multi'
7
+ require 'typhoeus/native'
8
+ require 'typhoeus/filter'
9
+ require 'typhoeus/remote_method'
10
+ require 'typhoeus/remote'
11
+ require 'typhoeus/remote_proxy_object'
12
+ require 'typhoeus/response'
13
+ require 'typhoeus/request'
14
+ require 'typhoeus/hydra'
15
+
16
+ module Typhoeus
17
+ VERSION = "0.1.22"
18
+
19
+ def self.easy_object_pool
20
+ @easy_objects ||= []
21
+ end
22
+
23
+ def self.init_easy_object_pool
24
+ 20.times do
25
+ easy_object_pool << Typhoeus::Easy.new
26
+ end
27
+ end
28
+
29
+ def self.release_easy_object(easy)
30
+ easy.reset
31
+ easy_object_pool << easy
32
+ end
33
+
34
+ def self.get_easy_object
35
+ if easy_object_pool.empty?
36
+ Typhoeus::Easy.new
37
+ else
38
+ easy_object_pool.pop
39
+ end
40
+ end
41
+
42
+ def self.add_easy_request(easy_object)
43
+ Thread.current[:curl_multi] ||= Typhoeus::Multi.new
44
+ Thread.current[:curl_multi].add(easy_object)
45
+ end
46
+
47
+ def self.perform_easy_requests
48
+ multi = Thread.current[:curl_multi]
49
+ start_time = Time.now
50
+ multi.easy_handles.each do |easy|
51
+ easy.start_time = start_time
52
+ end
53
+ multi.perform
54
+ end
55
+ end
@@ -0,0 +1 @@
1
+ native.bundle
@@ -0,0 +1,253 @@
1
+ module Typhoeus
2
+ class Easy
3
+ attr_reader :response_body, :response_header, :method, :headers, :url
4
+ attr_accessor :start_time
5
+
6
+ CURLINFO_STRING = 1048576
7
+ OPTION_VALUES = {
8
+ :CURLOPT_URL => 10002,
9
+ :CURLOPT_HTTPGET => 80,
10
+ :CURLOPT_HTTPPOST => 10024,
11
+ :CURLOPT_UPLOAD => 46,
12
+ :CURLOPT_CUSTOMREQUEST => 10036,
13
+ :CURLOPT_POSTFIELDS => 10015,
14
+ :CURLOPT_POSTFIELDSIZE => 60,
15
+ :CURLOPT_USERAGENT => 10018,
16
+ :CURLOPT_TIMEOUT_MS => 155,
17
+ :CURLOPT_NOSIGNAL => 99,
18
+ :CURLOPT_HTTPHEADER => 10023,
19
+ :CURLOPT_FOLLOWLOCATION => 52,
20
+ :CURLOPT_MAXREDIRS => 68,
21
+ :CURLOPT_HTTPAUTH => 107,
22
+ :CURLOPT_USERPWD => 10000 + 5,
23
+ :CURLOPT_VERBOSE => 41,
24
+ :CURLOPT_PROXY => 10004,
25
+ :CURLOPT_VERIFYPEER => 64,
26
+ :CURLOPT_NOBODY => 44
27
+ }
28
+ INFO_VALUES = {
29
+ :CURLINFO_RESPONSE_CODE => 2097154,
30
+ :CURLINFO_TOTAL_TIME => 3145731,
31
+ :CURLINFO_HTTPAUTH_AVAIL => 0x200000 + 23
32
+ }
33
+ AUTH_TYPES = {
34
+ :CURLAUTH_BASIC => 1,
35
+ :CURLAUTH_DIGEST => 2,
36
+ :CURLAUTH_GSSNEGOTIATE => 4,
37
+ :CURLAUTH_NTLM => 8,
38
+ :CURLAUTH_DIGEST_IE => 16
39
+ }
40
+
41
+ def initialize
42
+ @method = :get
43
+ @post_dat_set = nil
44
+ @headers = {}
45
+ end
46
+
47
+ def headers=(hash)
48
+ @headers = hash
49
+ end
50
+
51
+ def proxy=(proxy)
52
+ set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy)
53
+ end
54
+
55
+ def auth=(authinfo)
56
+ set_option(OPTION_VALUES[:CURLOPT_USERPWD], "#{authinfo[:username]}:#{authinfo[:password]}")
57
+ set_option(OPTION_VALUES[:CURLOPT_HTTPAUTH], authinfo[:method]) if authinfo[:method]
58
+ end
59
+
60
+ def auth_methods
61
+ get_info_long(INFO_VALUES[:CURLINFO_HTTPAUTH_AVAIL])
62
+ end
63
+
64
+ def verbose=(boolean)
65
+ set_option(OPTION_VALUES[:CURLOPT_VERBOSE], !!boolean ? 1 : 0)
66
+ end
67
+
68
+ def total_time_taken
69
+ get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
70
+ end
71
+
72
+ def response_code
73
+ get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
74
+ end
75
+
76
+ def follow_location=(boolean)
77
+ if boolean
78
+ set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 1)
79
+ else
80
+ set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 0)
81
+ end
82
+ end
83
+
84
+ def max_redirects=(redirects)
85
+ set_option(OPTION_VALUES[:CURLOPT_MAXREDIRS], redirects)
86
+ end
87
+
88
+ def timeout=(milliseconds)
89
+ @timeout = milliseconds
90
+ set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
91
+ set_option(OPTION_VALUES[:CURLOPT_TIMEOUT_MS], milliseconds)
92
+ end
93
+
94
+ def timed_out?
95
+ @timeout && total_time_taken > @timeout && response_code == 0
96
+ end
97
+
98
+ def request_body=(request_body)
99
+ @request_body = request_body
100
+ if @method == :put
101
+ easy_set_request_body(@request_body)
102
+ headers["Transfer-Encoding"] = ""
103
+ headers["Expect"] = ""
104
+ else
105
+ self.post_data = request_body
106
+ end
107
+ end
108
+
109
+ def user_agent=(user_agent)
110
+ set_option(OPTION_VALUES[:CURLOPT_USERAGENT], user_agent)
111
+ end
112
+
113
+ def url=(url)
114
+ @url = url
115
+ set_option(OPTION_VALUES[:CURLOPT_URL], url)
116
+ end
117
+
118
+ def disable_ssl_peer_verification
119
+ set_option(OPTION_VALUES[:CURLOPT_VERIFYPEER], 0)
120
+ end
121
+
122
+ def method=(method)
123
+ @method = method
124
+ if method == :get
125
+ set_option(OPTION_VALUES[:CURLOPT_HTTPGET], 1)
126
+ elsif method == :post
127
+ set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], 1)
128
+ self.post_data = ""
129
+ elsif method == :put
130
+ set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
131
+ self.request_body = "" unless @request_body
132
+ elsif method == :head
133
+ set_option(OPTION_VALUES[:CURLOPT_NOBODY], 1)
134
+ else
135
+ set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], "DELETE")
136
+ end
137
+ end
138
+
139
+ def post_data=(data)
140
+ @post_data_set = true
141
+ set_option(OPTION_VALUES[:CURLOPT_POSTFIELDS], data)
142
+ set_option(OPTION_VALUES[:CURLOPT_POSTFIELDSIZE], data.length)
143
+ end
144
+
145
+ def params=(params)
146
+ params_string = params.keys.collect do |k|
147
+ value = params[k]
148
+ if value.is_a? Hash
149
+ value.keys.collect {|sk| Rack::Utils.escape("#{k}[#{sk}]") + "=" + Rack::Utils.escape(value[sk].to_s)}
150
+ elsif value.is_a? Array
151
+ key = Rack::Utils.escape(k.to_s)
152
+ value.collect { |v| "#{key}=#{Rack::Utils.escape(v.to_s)}" }.join('&')
153
+ else
154
+ "#{Rack::Utils.escape(k.to_s)}=#{Rack::Utils.escape(params[k].to_s)}"
155
+ end
156
+ end.flatten.join("&")
157
+
158
+ if method == :post
159
+ self.post_data = params_string
160
+ else
161
+ self.url = "#{url}?#{params_string}"
162
+ end
163
+ end
164
+
165
+ def set_option(option, value)
166
+ if value.class == String
167
+ easy_setopt_string(option, value)
168
+ else
169
+ easy_setopt_long(option, value)
170
+ end
171
+ end
172
+
173
+ def perform
174
+ set_headers()
175
+ easy_perform()
176
+ response_code()
177
+ end
178
+
179
+ def set_headers
180
+ headers.each_pair do |key, value|
181
+ easy_add_header("#{key}: #{value}")
182
+ end
183
+ easy_set_headers() unless headers.empty?
184
+ end
185
+
186
+ # gets called when finished and response code is 200-299
187
+ def success
188
+ @success.call(self) if @success
189
+ end
190
+
191
+ def on_success(&block)
192
+ @success = block
193
+ end
194
+
195
+ def on_success=(block)
196
+ @success = block
197
+ end
198
+
199
+ # gets called when finished and response code is 300-599
200
+ def failure
201
+ @failure.call(self) if @failure
202
+ end
203
+
204
+ def on_failure(&block)
205
+ @failure = block
206
+ end
207
+
208
+ def on_failure=(block)
209
+ @failure = block
210
+ end
211
+
212
+ def retries
213
+ @retries ||= 0
214
+ end
215
+
216
+ def increment_retries
217
+ @retries ||= 0
218
+ @retries += 1
219
+ end
220
+
221
+ def max_retries
222
+ @max_retries ||= 40
223
+ end
224
+
225
+ def max_retries?
226
+ retries >= max_retries
227
+ end
228
+
229
+ def reset
230
+ @retries = 0
231
+ @response_code = 0
232
+ @response_header = ""
233
+ @response_body = ""
234
+ easy_reset()
235
+ end
236
+
237
+ def get_info_string(option)
238
+ easy_getinfo_string(option)
239
+ end
240
+
241
+ def get_info_long(option)
242
+ easy_getinfo_long(option)
243
+ end
244
+
245
+ def get_info_double(option)
246
+ easy_getinfo_double(option)
247
+ end
248
+
249
+ def curl_version
250
+ version
251
+ end
252
+ end
253
+ end
@@ -0,0 +1,28 @@
1
+ module Typhoeus
2
+ class Filter
3
+ attr_reader :method_name
4
+
5
+ def initialize(method_name, options = {})
6
+ @method_name = method_name
7
+ @options = options
8
+ end
9
+
10
+ def apply_filter?(method_name)
11
+ if @options[:only]
12
+ if @options[:only].instance_of? Symbol
13
+ @options[:only] == method_name
14
+ else
15
+ @options[:only].include?(method_name)
16
+ end
17
+ elsif @options[:except]
18
+ if @options[:except].instance_of? Symbol
19
+ @options[:except] != method_name
20
+ else
21
+ !@options[:except].include?(method_name)
22
+ end
23
+ else
24
+ true
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,210 @@
1
+ module Typhoeus
2
+ class Hydra
3
+ def initialize(options = {})
4
+ @memoize_requests = true
5
+ @multi = Multi.new
6
+ @easy_pool = []
7
+ initial_pool_size = options[:initial_pool_size] || 10
8
+ @max_concurrency = options[:max_concurrency] || 200
9
+ initial_pool_size.times { @easy_pool << Easy.new }
10
+ @stubs = []
11
+ @memoized_requests = {}
12
+ @retrieved_from_cache = {}
13
+ @queued_requests = []
14
+ @running_requests = 0
15
+ end
16
+
17
+ def self.hydra
18
+ @hydra ||= new
19
+ end
20
+
21
+ def self.hydra=(val)
22
+ @hydra = val
23
+ end
24
+
25
+ def clear_stubs
26
+ @stubs = []
27
+ end
28
+
29
+ def fire_and_forget
30
+ @queued_requests.each {|r| queue(r, false)}
31
+ @multi.fire_and_forget
32
+ end
33
+
34
+ def queue(request, obey_concurrency_limit = true)
35
+ return if assign_to_stub(request)
36
+
37
+ if @running_requests >= @max_concurrency && obey_concurrency_limit
38
+ @queued_requests << request
39
+ else
40
+ if request.method == :get
41
+ if @memoize_requests && @memoized_requests.has_key?(request.url)
42
+ if response = @retrieved_from_cache[request.url]
43
+ request.response = response
44
+ request.call_handlers
45
+ else
46
+ @memoized_requests[request.url] << request
47
+ end
48
+ else
49
+ @memoized_requests[request.url] = [] if @memoize_requests
50
+ get_from_cache_or_queue(request)
51
+ end
52
+ else
53
+ get_from_cache_or_queue(request)
54
+ end
55
+ end
56
+ end
57
+
58
+ def run
59
+ @stubs.each do |m|
60
+ m.requests.each do |request|
61
+ m.response.request = request
62
+ handle_request(request, m.response)
63
+ end
64
+ end
65
+ @multi.perform
66
+ @memoized_requests = {}
67
+ @retrieved_from_cache = {}
68
+ end
69
+
70
+ def disable_memoization
71
+ @memoize_requests = false
72
+ end
73
+
74
+ def cache_getter(&block)
75
+ @cache_getter = block
76
+ end
77
+
78
+ def cache_setter(&block)
79
+ @cache_setter = block
80
+ end
81
+
82
+ def on_complete(&block)
83
+ @on_complete = block
84
+ end
85
+
86
+ def on_complete=(proc)
87
+ @on_complete = proc
88
+ end
89
+
90
+ def stub(method, url)
91
+ @stubs << HydraMock.new(url, method)
92
+ @stubs.last
93
+ end
94
+
95
+ def assign_to_stub(request)
96
+ m = @stubs.detect {|stub| stub.matches? request}
97
+ m && m.add_request(request)
98
+ end
99
+ private :assign_to_stub
100
+
101
+ def get_from_cache_or_queue(request)
102
+ if @cache_getter
103
+ val = @cache_getter.call(request)
104
+ if val
105
+ @retrieved_from_cache[request.url] = val
106
+ handle_request(request, val, false)
107
+ else
108
+ @multi.add(get_easy_object(request))
109
+ end
110
+ else
111
+ @multi.add(get_easy_object(request))
112
+ end
113
+ end
114
+ private :get_from_cache_or_queue
115
+
116
+ def get_easy_object(request)
117
+ @running_requests += 1
118
+ easy = @easy_pool.pop || Easy.new
119
+ easy.url = request.url
120
+ easy.method = request.method
121
+ easy.params = request.params if request.method == :post && !request.params.nil?
122
+ easy.headers = request.headers if request.headers
123
+ easy.request_body = request.body if request.body
124
+ easy.timeout = request.timeout if request.timeout
125
+ easy.follow_location = request.follow_location if request.follow_location
126
+ easy.max_redirects = request.max_redirects if request.max_redirects
127
+ easy.proxy = request.proxy if request.proxy
128
+ easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
129
+
130
+ easy.on_success do |easy|
131
+ queue_next
132
+ handle_request(request, response_from_easy(easy, request))
133
+ release_easy_object(easy)
134
+ end
135
+ easy.on_failure do |easy|
136
+ queue_next
137
+ handle_request(request, response_from_easy(easy, request))
138
+ release_easy_object(easy)
139
+ end
140
+ easy.set_headers
141
+ easy
142
+ end
143
+ private :get_easy_object
144
+
145
+ def queue_next
146
+ @running_requests -= 1
147
+ queue(@queued_requests.pop) unless @queued_requests.empty?
148
+ end
149
+ private :queue_next
150
+
151
+ def release_easy_object(easy)
152
+ easy.reset
153
+ @easy_pool.push easy
154
+ end
155
+ private :release_easy_object
156
+
157
+ def handle_request(request, response, live_request = true)
158
+ request.response = response
159
+
160
+ if live_request && request.cache_timeout && @cache_setter
161
+ @cache_setter.call(request)
162
+ end
163
+ @on_complete.call(response) if @on_complete
164
+
165
+ request.call_handlers
166
+ if requests = @memoized_requests[request.url]
167
+ requests.each do |r|
168
+ r.response = response
169
+ r.call_handlers
170
+ end
171
+ end
172
+ end
173
+ private :handle_request
174
+
175
+ def response_from_easy(easy, request)
176
+ Response.new(:code => easy.response_code,
177
+ :headers => easy.response_header,
178
+ :body => easy.response_body,
179
+ :time => easy.total_time_taken,
180
+ :request => request)
181
+ end
182
+ private :response_from_easy
183
+ end
184
+
185
+ class HydraMock
186
+ attr_reader :url, :method, :response, :requests
187
+
188
+ def initialize(url, method)
189
+ @url = url
190
+ @method = method
191
+ @requests = []
192
+ end
193
+
194
+ def add_request(request)
195
+ @requests << request
196
+ end
197
+
198
+ def and_return(val)
199
+ @response = val
200
+ end
201
+
202
+ def matches?(request)
203
+ if url.kind_of?(String)
204
+ request.method == method && request.url == url
205
+ else
206
+ request.method == method && url =~ request.url
207
+ end
208
+ end
209
+ end
210
+ end