em-riak 0.2.8 → 0.2.91
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +2 -4
- data/em-riak.gemspec +2 -2
- data/lib/em-riak/basic.rb +211 -85
- data/lib/em-riak/configurations.rb +4 -0
- data/lib/em-riak/httpi_hack.rb +23 -0
- data/lib/em-riak/map_reduce.rb +3 -0
- data/lib/em-riak/version.rb +1 -1
- data/lib/em-riak.rb +1 -0
- data/spec/unit/basic.rb +6 -2
- data/spec/unit/core.rb +58 -4
- metadata +13 -12
data/README.md
CHANGED
@@ -116,8 +116,6 @@ Use it directly as following :
|
|
116
116
|
* Create new Pull Request
|
117
117
|
|
118
118
|
### TO-DO
|
119
|
-
Full Text Seacrh
|
119
|
+
Full Text Seacrh (This will not going to implement because Basho will replace search with yakozuna)
|
120
120
|
Deployment & Management
|
121
|
-
|
122
|
-
ORM with Model
|
123
|
-
Hadoop MapReduce Support
|
121
|
+
ORM with Model
|
data/em-riak.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'em-riak'
|
3
|
-
s.version = '0.2.
|
4
|
-
s.date = '
|
3
|
+
s.version = '0.2.91'
|
4
|
+
s.date = '2013-03-08'
|
5
5
|
s.summary = "Riak client for eventmachine"
|
6
6
|
s.description = "An extremely fast and convenient riak client for eventmachine."
|
7
7
|
s.authors = ["Von"]
|
data/lib/em-riak/basic.rb
CHANGED
@@ -7,24 +7,31 @@ require 'httpi'
|
|
7
7
|
require 'yajl'
|
8
8
|
|
9
9
|
module EmRiak
|
10
|
-
attr_accessor :bucket, :interface, :adapter, :hosts, :cluster, :async, :http_client, :vclock, :replication, :debug
|
10
|
+
attr_accessor :bucket, :interface, :adapter, :hosts, :weight, :cluster, :async, :http_client, :vclock, :replication, :debug, :em_enqueue_pool, :em_enqueue_pool_max
|
11
11
|
extend self
|
12
12
|
|
13
|
+
|
13
14
|
# A easy interface to create connections
|
14
15
|
class Connection
|
15
16
|
def initialize(*opts)
|
16
17
|
if opts && opts.length>0
|
17
18
|
opts=opts.first
|
18
|
-
methods=[:bucket, :interface, :adapter, :
|
19
|
+
methods=[:bucket, :interface, :adapter, :cluster, :async, :http_client, :vclock, :repliaction, :debug]
|
19
20
|
opts.each{|key,opt| EmRiak.send "#{key}=".to_sym, opt if opt && methods.index(key.to_sym) }
|
20
21
|
else
|
21
22
|
EmRiak.bucket=RIAKBUCKET
|
22
23
|
end
|
23
|
-
EmRiak.
|
24
|
+
EmRiak.send :handle_em_hosts, opts[:hosts]
|
25
|
+
EmRiak.replication={"w"=>"quorum","r"=>"quorum","n_val"=>3} if EmRiak.replication.nil?
|
26
|
+
|
24
27
|
EmRiak.debug=false if EmRiak.debug.nil?
|
25
28
|
end
|
26
29
|
end
|
27
|
-
|
30
|
+
def replication=replication_setting
|
31
|
+
@replication=convert_params_layers(replication_setting,:to_s)
|
32
|
+
EmRiak.send :change_bucket_replication ,EmRiak.bucket, EmRiak.replication if EmRiak.replication!={"w"=>"quorum","r"=>1,"n_val"=>"quorum"}
|
33
|
+
end
|
34
|
+
|
28
35
|
# -- CRUD
|
29
36
|
class StorageObject < Hash
|
30
37
|
def save(block=nil); EmRiak.save(self,block); end
|
@@ -44,15 +51,7 @@ module EmRiak
|
|
44
51
|
end
|
45
52
|
end
|
46
53
|
def create(key,*opts,&callback)
|
47
|
-
data=
|
48
|
-
if opts && opts.count>0
|
49
|
-
if opts.count==1
|
50
|
-
body=opts.first
|
51
|
-
else
|
52
|
-
opts,body=opts
|
53
|
-
end
|
54
|
-
body.each{|key,value| data[:body][key]=value}
|
55
|
-
end
|
54
|
+
data,opts=handle_opts_for_body(opts)
|
56
55
|
data,callback=handle_callback(key,"save",opts,callback,data)
|
57
56
|
operate_bucket= select_bucket(data)
|
58
57
|
url="/buckets/#{operate_bucket}/keys/#{key_handler(key)}"
|
@@ -71,19 +70,14 @@ module EmRiak
|
|
71
70
|
http("get",key,url,data,callback)
|
72
71
|
end
|
73
72
|
|
74
|
-
def select_bucket(data)
|
75
|
-
return EmRiak.bucket if !data[:opts]
|
76
|
-
return data[:opts][:bucket] ? data[:opts][:bucket] : EmRiak.bucket
|
77
|
-
end
|
78
|
-
|
79
73
|
def save(obj,*opts,&callback)
|
80
|
-
data=
|
81
|
-
obj.each{|key,value| data[:body][key]=value }
|
74
|
+
data,nothing=handle_opts_for_body([obj])
|
82
75
|
data,callback=handle_callback(obj[:riak_key],"save",opts,callback,data)
|
83
76
|
operate_bucket= select_bucket(data)
|
84
77
|
url="/buckets/#{operate_bucket}/keys/#{obj[:riak_key]}"
|
85
78
|
http("put",obj[:riak_key],url,data,callback)
|
86
79
|
end
|
80
|
+
|
87
81
|
def destroy(key,*opts,&callback)
|
88
82
|
obj_key=defined?(obj) ? obj[:riak_key] : (key.class==String ? key : key[:riak_key])
|
89
83
|
data,callback=handle_callback(obj_key,"delete",opts,callback)
|
@@ -93,6 +87,10 @@ module EmRiak
|
|
93
87
|
key=nil
|
94
88
|
end
|
95
89
|
|
90
|
+
def http(method,key,url,data,callback=nil,res=nil)
|
91
|
+
(callback || self.async && ["delete","put","post"].index(method) && key!="mapred") ? em_http(method,key,url,data,callback) : open_http(method,key,url,data)
|
92
|
+
end
|
93
|
+
|
96
94
|
# -- Search
|
97
95
|
def search(method_name,*args,&callback)
|
98
96
|
result_handler=nil
|
@@ -143,28 +141,24 @@ module EmRiak
|
|
143
141
|
response
|
144
142
|
end
|
145
143
|
|
146
|
-
|
147
|
-
(callback || self.async && ["delete","put","post"].index(method) && key!="mapred") ? em_http(method,key,url,data,callback) : open_http(method,key,url,data)
|
148
|
-
end
|
149
|
-
|
150
|
-
protected
|
144
|
+
protected
|
151
145
|
def open_http(method,key,url,data,res={})
|
152
146
|
has_http_instance?
|
153
147
|
data={:body=>{},:head=>{},:query=>{}} if !data
|
154
|
-
|
155
|
-
http_client.url = generate_path(url)
|
148
|
+
http_client.url = generate_path(url)[:url]
|
156
149
|
http_client.headers = key!="mapred" ? convert_params_layers(data[:head],:to_s) : data[:head]
|
157
|
-
http_client.body = key!="mapred" ? convert_params_layers(data[:body],:to_s) : data[:body]
|
158
|
-
http_client.open_timeout =
|
159
|
-
http_client.read_timeout =
|
160
|
-
|
150
|
+
http_client.body = key!="mapred" ? (data[:body].class==String ? data[:body] : convert_params_layers(data[:body],:to_s)) : data[:body]
|
151
|
+
http_client.open_timeout = 500 # seconds. change it if you need
|
152
|
+
http_client.read_timeout = 500 # seconds. change it if you need
|
161
153
|
res=HTTPI.request(method.to_sym,http_client)
|
162
|
-
|
163
|
-
res=handle_response_string_to_object(key,res,method,data)
|
154
|
+
body=res.body if res.class==HTTPI::Response
|
155
|
+
res=handle_response_string_to_object(key,body,res,method,data)
|
164
156
|
|
165
157
|
res=nil if res && res.class!=String && res.count<1
|
166
|
-
|
167
158
|
return res
|
159
|
+
rescue Exception=> e
|
160
|
+
puts "open http error #{e} with key #{key} #{e.backtrace}"
|
161
|
+
return nil
|
168
162
|
end
|
169
163
|
|
170
164
|
def em_http(method,key,url,data,async_callback)
|
@@ -172,40 +166,48 @@ module EmRiak
|
|
172
166
|
begin
|
173
167
|
raise "Retry too much time" if data[:retry]>RETRY_TIMES
|
174
168
|
|
175
|
-
#conn options
|
176
|
-
conn_options = {
|
177
|
-
:connect_timeout => EM_REQUEST_TIMEOUT, # default connection setup timeout
|
178
|
-
:inactivity_timeout => EM_INACTIVITY_TIMEOUT, # default connection inactivity (post-setup) timeout
|
179
|
-
}
|
180
|
-
#data[:body]=data[:body] if data[:body].class==Hash && data[:head]['Content-Type'] && data[:head]['Content-Type']=="application/json"
|
181
|
-
#request options
|
182
169
|
data[:query]="?returnbody=true"
|
183
|
-
request_options=data.reject{|k,v| k==:retry}
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
170
|
+
request_options=data.reject{|k,v| k==:retry} #request options, reject some data before submit
|
171
|
+
path_result=generate_path(url)
|
172
|
+
execute_url = path_result[:url]
|
173
|
+
host = path_result[:host]
|
174
|
+
|
175
|
+
# == Start
|
176
|
+
# These lines should reduce high em-http recreate cost. (especially if we create ssl conn)
|
177
|
+
# you can increase em_enqueue_pool_max when you init EM.connections.
|
178
|
+
# By default, it's 30 for each host. so, for example, you have 2 host as : [127.0.0.1:8080,127.0.0.1:8081]. the max em-http-request alive instance would be 60
|
179
|
+
# It should automatically remove extra and remain 3 alive connections after 30 secs.
|
180
|
+
raise "do it later because pool full" if EmRiak.em_enqueue_pool[host][:working].count>=EmRiak.em_enqueue_pool_max
|
181
|
+
|
182
|
+
if EmRiak.em_enqueue_pool[host][:alives].count<1
|
183
|
+
enqueue=EventMachine::HttpRequest.new(execute_url,{:keepalive=>true}) # keepalive will make all connections alive, so we can avoid heavy cpu load when large amount em_http instance create
|
184
|
+
EmRiak.em_enqueue_pool[host][:working].push(enqueue)
|
185
|
+
else
|
186
|
+
enqueue=EmRiak.em_enqueue_pool[host][:alives].pop
|
187
|
+
EmRiak.em_enqueue_pool[host][:working].push(enqueue)
|
195
188
|
end
|
189
|
+
# == End
|
196
190
|
|
197
|
-
|
191
|
+
# Do job
|
192
|
+
riakHttp = enqueue.send method.to_sym , :connect_timeout => EM_REQUEST_TIMEOUT, :inactivity_timeout => EM_INACTIVITY_TIMEOUT, :keepalive=>true
|
193
|
+
riakHttp.errback{ # Error Callback
|
198
194
|
puts "#{method} - #{key} - riak callback error #{riakHttp.response}" if EmRiak.debug
|
199
195
|
data[:retry]+=1
|
200
|
-
|
196
|
+
EM.add_timer(EM_FAIL_RETRY_DELAY){ em_http(method,key,url,data,async_callback) } # retry after random seconds to avoid too heavy load
|
201
197
|
}
|
198
|
+
riakHttp.callback{ # Success Callback
|
199
|
+
# remove from enqueue pool
|
200
|
+
pool_index=EmRiak.em_enqueue_pool[host][:working].index(enqueue)
|
201
|
+
EmRiak.em_enqueue_pool[host][:alives].push(EmRiak.em_enqueue_pool[host][:working][pool_index])
|
202
|
+
EmRiak.em_enqueue_pool[host][:working].delete(enqueue)
|
203
|
+
alive_counter=EmRiak.em_enqueue_pool[host][:alives].count
|
204
|
+
EM.add_timer(EM_ENQUEUE_POOL_RELEASE_DELAY){ EmRiak.send :release_enqueue_pool, host, alive_counter }
|
202
205
|
|
203
|
-
#Success Callback
|
204
|
-
riakHttp.callback{
|
205
206
|
puts "#{method} - #{url} - #{key} - riak response #{riakHttp.response}" if EmRiak.debug
|
206
207
|
async_callback.call(riakHttp.response) if async_callback
|
207
208
|
}
|
208
209
|
rescue Exception => e
|
210
|
+
EM.add_timer(rand(EM_ENQUEUE_POOL_REDO_DELAY)){ em_http(method,key,url,data,async_callback) } if e.message=="do it later because pool full"
|
209
211
|
puts "em_http - #{method} - #{url} - #{key} error #{e}" if EmRiak.debug
|
210
212
|
end
|
211
213
|
|
@@ -221,18 +223,21 @@ module EmRiak
|
|
221
223
|
res
|
222
224
|
end
|
223
225
|
|
224
|
-
def handle_response_string_to_object(key,res,method,data)
|
226
|
+
def handle_response_string_to_object(key,res,httpi_object,method,data)
|
225
227
|
raise "response can't be nil" if !res
|
226
228
|
return res if key=="mapred"
|
227
|
-
return nil if ["2i"].index(key)
|
229
|
+
return nil if ["2i"].index(key) || ["delete","destroy"].index(method)
|
228
230
|
|
229
|
-
raise "data not found" if ["not found\n"].index(res)
|
230
|
-
raise "method is not match" if !["get","put","post"].index(method)
|
231
|
+
raise "data not found or timeout" if ["not found\n","request timed out\n"].index(res)
|
232
|
+
raise "method is not match with method #{method}" if !["get","put","post","save","delete","destroy"].index(method)
|
231
233
|
raise "data is empty" if method=="get" && res.gsub(" ","").length<1
|
232
234
|
|
233
|
-
|
234
|
-
|
235
|
-
|
235
|
+
if httpi_object && httpi_object.headers["content-type"]=="text/plain"
|
236
|
+
obj=res
|
237
|
+
elsif res=="not found\n"
|
238
|
+
obj=nil
|
239
|
+
elsif res
|
240
|
+
obj=EmRiak::StorageObject.new()
|
236
241
|
if res.class==String
|
237
242
|
obj=return_body_handler(URI.unescape(res.force_encoding("UTF-8")).gsub("+"," ").gsub("=>",":"),obj,"string")
|
238
243
|
elsif res.class==Hash
|
@@ -244,7 +249,7 @@ module EmRiak
|
|
244
249
|
end
|
245
250
|
return obj
|
246
251
|
rescue Exception=>e
|
247
|
-
puts "handle response error reason #{e}" if EmRiak.debug
|
252
|
+
puts "handle response error reason #{e} #{key} #{method}" if EmRiak.debug
|
248
253
|
return nil
|
249
254
|
end
|
250
255
|
|
@@ -269,31 +274,54 @@ module EmRiak
|
|
269
274
|
data[:opts], data[:head], data[:query], callback = option_response
|
270
275
|
end
|
271
276
|
new_proc=Proc.new{|res|
|
272
|
-
obj=handle_response_string_to_object(key,res,method,data)
|
277
|
+
obj=handle_response_string_to_object(key,res,nil,method,data)
|
273
278
|
callback.call(obj)
|
274
279
|
} if callback
|
275
280
|
|
276
281
|
[data,new_proc]
|
277
282
|
end
|
278
|
-
def
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
283
|
+
def handle_opts_for_body(opts,data={:body=>{}})
|
284
|
+
if opts && opts.count>0
|
285
|
+
if opts.count==1
|
286
|
+
body=opts.first
|
287
|
+
else
|
288
|
+
opts,body=opts
|
289
|
+
end
|
290
|
+
|
291
|
+
if body.class==String
|
292
|
+
data[:body]=body
|
293
|
+
else
|
294
|
+
body.each{|key,value| data[:body][key]=value}
|
295
|
+
end
|
291
296
|
end
|
292
|
-
|
297
|
+
[data,opts]
|
298
|
+
end
|
299
|
+
def map_content_type(action,opts)
|
300
|
+
# TODO : need to implement more types
|
301
|
+
|
302
|
+
# case action
|
303
|
+
# when "save","add"
|
304
|
+
# "application/json"
|
305
|
+
# when "get"
|
306
|
+
# "application/json"
|
307
|
+
# when "destroy","delete","remove"
|
308
|
+
# "application/json"
|
309
|
+
# when "add_link"
|
310
|
+
# "application/json"
|
311
|
+
# when "remove_link"
|
312
|
+
# "application/json"
|
313
|
+
# end
|
314
|
+
return "text/plain" if opts.class==String
|
315
|
+
return (opts && opts[:content_type]) ? opts[:content_type] : "application/json"
|
316
|
+
end
|
317
|
+
def handle_options(action,opts,header={},query="",extra_setting={})
|
293
318
|
# Map & overwrite if header specisfic
|
294
319
|
opts=opts.first if opts && opts.class==Array
|
295
320
|
|
296
|
-
|
321
|
+
#Load default header setting
|
322
|
+
header["Content-Type"]=map_content_type(action,opts)
|
323
|
+
|
324
|
+
if opts && opts.class!=String && opts.count>0
|
297
325
|
# Map the options class
|
298
326
|
callback=opts.reject!{|obj| obj.class==Proc}
|
299
327
|
|
@@ -354,8 +382,7 @@ module EmRiak
|
|
354
382
|
res << callback if callback
|
355
383
|
res
|
356
384
|
end
|
357
|
-
|
358
|
-
def key_handler(key,map={"["=>"%5B","]"=>"%5D","+"=>"%2B","/"=>"%2F","("=>"%28",")"=>"%29",":"=>"%3A"})
|
385
|
+
def key_handler(key)
|
359
386
|
# To match the key rule from Basho : If your field contains special characters, such as (‘+’,‘/’,‘[’,‘]’,‘(’,‘)’,‘:’ or space), then either surround the phrase in single quotes, or escape each special character with a backslash.
|
360
387
|
# map.each{|origin,value| key.gsub!(origin,value) }
|
361
388
|
EscapeUtils.escape_url(key.to_s.encode('UTF-8'))
|
@@ -386,12 +413,111 @@ module EmRiak
|
|
386
413
|
end
|
387
414
|
|
388
415
|
private
|
389
|
-
def
|
416
|
+
def change_bucket_replication(bucket_name,replication_setting,counter=0)
|
417
|
+
urls=return_all_hosts_as_urls
|
418
|
+
|
419
|
+
change_replication_proc=Proc.new do
|
420
|
+
has_http_instance?
|
421
|
+
urls.each do |url|
|
422
|
+
http_client.headers = {'Content-Type'=> 'application/json'}
|
423
|
+
http_client.url = "#{url}/riak/#{bucket_name}"
|
424
|
+
http_client.body = json_encode({"props"=>replication_setting})
|
425
|
+
res=HTTPI.request(:put,http_client)
|
426
|
+
end
|
427
|
+
end
|
428
|
+
|
429
|
+
EmRiak.async ? Thread.new{ change_replication_proc.call } : change_replication_proc.call
|
430
|
+
end
|
431
|
+
|
432
|
+
def get_bucket_replication(bucket_name,results=[])
|
433
|
+
urls=return_all_hosts_as_urls
|
434
|
+
|
435
|
+
has_http_instance?
|
436
|
+
urls.each do |url|
|
437
|
+
http_client.headers = {'Content-Type'=> 'application/json'}
|
438
|
+
http_client.url = "#{url}/riak/#{bucket_name}"
|
439
|
+
res=HTTPI.request(:get,http_client)
|
440
|
+
results << json_decode(res.body)
|
441
|
+
end
|
442
|
+
results
|
443
|
+
end
|
444
|
+
|
445
|
+
def return_all_hosts_as_urls(urls=[])
|
446
|
+
EmRiak.hosts.each do |host|
|
447
|
+
data= host.class==Array ? host[1] : host
|
448
|
+
if host[1].class==Array
|
449
|
+
data.each{|url| urls << url }
|
450
|
+
else
|
451
|
+
urls << data
|
452
|
+
end
|
453
|
+
end
|
454
|
+
urls
|
455
|
+
end
|
456
|
+
|
457
|
+
def select_bucket(data)
|
458
|
+
return EmRiak.bucket if !data[:opts]
|
459
|
+
return data[:opts][:bucket] ? data[:opts][:bucket] : EmRiak.bucket
|
460
|
+
end
|
461
|
+
|
462
|
+
def generate_path(url,current_weight=0,host_list=[])
|
463
|
+
if EmRiak.weight
|
464
|
+
selected_weight=rand(EmRiak.weight[:sum])
|
465
|
+
EmRiak.weight[:list].each do |weight_num|
|
466
|
+
current_weight+=weight_num
|
467
|
+
if selected_weight<=current_weight
|
468
|
+
host_list=EmRiak.hosts["weight#{weight_num}".to_sym]
|
469
|
+
break
|
470
|
+
end
|
471
|
+
end
|
472
|
+
host=host_list[rand(host_list.count)]
|
473
|
+
new_url=host+url
|
474
|
+
else
|
475
|
+
host=EmRiak.hosts[rand(EmRiak.hosts.length)]
|
476
|
+
new_url=host+url
|
477
|
+
end
|
478
|
+
{:host=>host,:url=>new_url}
|
479
|
+
end
|
390
480
|
def has_http_instance?
|
391
481
|
if !EmRiak.http_client
|
392
482
|
HTTPI.log=false
|
393
|
-
HTTPI.adapter
|
483
|
+
HTTPI.adapter= EmRiak.adapter ? EmRiak.adapter : :net_http
|
394
484
|
EmRiak.http_client=HTTPI::Request.new
|
395
485
|
end
|
396
486
|
end
|
487
|
+
def handle_em_hosts(emhosts)
|
488
|
+
EmRiak.em_enqueue_pool={}
|
489
|
+
EmRiak.em_enqueue_pool_max=EM_ENQUEUE_POOL_MAX if EmRiak.em_enqueue_pool_max.nil?
|
490
|
+
# We support 2 host methods.
|
491
|
+
# For easier way, send the hosts as array. ex : ["127.0.0.1:8091","127.0.0.1:8092","127.0.0.1:8093"]
|
492
|
+
# For advance way, configurate the host weight, the higher weight would take heavier load.
|
493
|
+
# Fromat : {:weight10=>["127.0.0.1:8091"],:weight5=>["127.0.0.1:8092"],:weight1=>["127.0.0.1:8093"]}
|
494
|
+
if emhosts.class==Hash
|
495
|
+
EmRiak.weight={:list=>[],:sum=>0}
|
496
|
+
emhosts.each{|weight,hosts|
|
497
|
+
weight_num=weight.to_s.split("weight")[1]
|
498
|
+
(hosts).count.times{
|
499
|
+
EmRiak.weight[:list] << weight_num.to_i
|
500
|
+
EmRiak.weight[:sum]+=weight_num.to_i
|
501
|
+
}
|
502
|
+
}
|
503
|
+
EmRiak.weight[:list].sort!
|
504
|
+
# create em enqueue pool here
|
505
|
+
emhosts.each{|weight,hosts| hosts.each{|host| EmRiak.em_enqueue_pool[host]={:alives=>[],:working=>[]} } }
|
506
|
+
else
|
507
|
+
EmRiak.weight=nil
|
508
|
+
# create em enqueue pool here
|
509
|
+
emhosts.each{|host| EmRiak.em_enqueue_pool[host]={:alives=>[],:working=>[]} }
|
510
|
+
end
|
511
|
+
EmRiak.hosts=emhosts
|
512
|
+
end
|
513
|
+
|
514
|
+
def release_enqueue_pool(host,alive_counter)
|
515
|
+
operating_pool=EmRiak.em_enqueue_pool[host][:alives]
|
516
|
+
current_counts=operating_pool.count
|
517
|
+
loop{
|
518
|
+
enqueue=operating_pool.pop
|
519
|
+
enqueue=nil
|
520
|
+
break if operating_pool.count<=3
|
521
|
+
} if current_counts>7 && current_counts>(alive_counter-5)
|
522
|
+
end
|
397
523
|
end
|
@@ -3,6 +3,10 @@ RESERVED_MAPPER=[:bucket]
|
|
3
3
|
SEARCH_SUPPORT=[:secondary_index,:full_text,:map_reduce]
|
4
4
|
RETRY_TIMES=3
|
5
5
|
EMMAX=2000
|
6
|
+
EM_ENQUEUE_POOL_MAX=30
|
7
|
+
EM_ENQUEUE_POOL_RELEASE_DELAY=30
|
8
|
+
EM_ENQUEUE_POOL_REDO_DELAY=10
|
9
|
+
EM_FAIL_RETRY_DELAY=100
|
6
10
|
EM_REQUEST_TIMEOUT=50
|
7
11
|
EM_INACTIVITY_TIMEOUT=0
|
8
12
|
RIAKBUCKET="test_bucket"
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module HTTPI
|
2
|
+
module Adapter
|
3
|
+
class NetHTTP < Base
|
4
|
+
def request(method)
|
5
|
+
unless REQUEST_METHODS.include? method
|
6
|
+
raise NotSupportedError, "Net::HTTP does not support custom HTTP methods"
|
7
|
+
end
|
8
|
+
|
9
|
+
do_request(method) do |http, http_request|
|
10
|
+
http_request.body = @request.body
|
11
|
+
return_data=http.request http_request
|
12
|
+
http.finish
|
13
|
+
return_data
|
14
|
+
end
|
15
|
+
rescue OpenSSL::SSL::SSLError
|
16
|
+
raise SSLError
|
17
|
+
rescue Errno::ECONNREFUSED # connection refused
|
18
|
+
$!.extend ConnectionError
|
19
|
+
raise
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/em-riak/map_reduce.rb
CHANGED
data/lib/em-riak/version.rb
CHANGED
data/lib/em-riak.rb
CHANGED
data/spec/unit/basic.rb
CHANGED
@@ -5,7 +5,7 @@ require 'em-http-request'
|
|
5
5
|
require 'httpi'
|
6
6
|
require 'yajl'
|
7
7
|
require 'em-riak'
|
8
|
-
TEST_RIAK_HOSTS=["http://127.0.0.1:8091"
|
8
|
+
TEST_RIAK_HOSTS=["http://127.0.0.1:8091"]
|
9
9
|
require File.expand_path('../../../lib/em-riak', __FILE__)
|
10
10
|
gem 'minitest'
|
11
11
|
require 'minitest/autorun'
|
@@ -50,21 +50,24 @@ class TestAsyncCRUD < MiniTest::Unit::TestCase
|
|
50
50
|
end
|
51
51
|
def test_async_find
|
52
52
|
EM.run do
|
53
|
+
EmRiak.create("member5",@data)
|
53
54
|
EmRiak.find("member5"){|result|
|
54
55
|
assert_equal @data, result
|
55
56
|
EM.stop
|
56
57
|
}
|
57
58
|
end
|
59
|
+
sleep(3)
|
58
60
|
end
|
59
61
|
def test_async_create
|
60
62
|
EM.run do
|
61
63
|
EmRiak.create("member5",@data){|member|
|
62
|
-
assert_equal
|
64
|
+
assert_equal @data,member[:body] # Because Riak does not response data when create, so it's nil. Will response data with next release.
|
63
65
|
member=EmRiak.find("member5")
|
64
66
|
assert_equal @data, member
|
65
67
|
EM.stop
|
66
68
|
}
|
67
69
|
end
|
70
|
+
sleep(3)
|
68
71
|
end
|
69
72
|
|
70
73
|
def test_async_destroy
|
@@ -76,5 +79,6 @@ class TestAsyncCRUD < MiniTest::Unit::TestCase
|
|
76
79
|
EM.stop
|
77
80
|
}
|
78
81
|
end
|
82
|
+
sleep(3)
|
79
83
|
end
|
80
84
|
end
|
data/spec/unit/core.rb
CHANGED
@@ -5,7 +5,7 @@ require 'em-http-request'
|
|
5
5
|
require 'httpi'
|
6
6
|
require 'yajl'
|
7
7
|
require 'em-riak'
|
8
|
-
TEST_RIAK_HOSTS=["http://127.0.0.1:8091"
|
8
|
+
TEST_RIAK_HOSTS=["http://127.0.0.1:8091"]
|
9
9
|
require File.expand_path('../../../lib/em-riak', __FILE__)
|
10
10
|
gem 'minitest'
|
11
11
|
require 'minitest/autorun'
|
@@ -17,8 +17,11 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
|
|
17
17
|
@data={:name=>"5",:riak_key=>"member5"}
|
18
18
|
end
|
19
19
|
|
20
|
-
def test_hosts_weight
|
21
|
-
|
20
|
+
def test_hosts_weight
|
21
|
+
EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>{:weight10=>["http://127.0.0.1:8091"]}})
|
22
|
+
response=EmRiak.send :generate_path, "/riak/test-member/member5"
|
23
|
+
assert_equal "http://127.0.0.1:8091",response[:host]
|
24
|
+
assert_equal "http://127.0.0.1:8091/riak/test-member/member5",response[:url]
|
22
25
|
end
|
23
26
|
|
24
27
|
def test_independent_bucket
|
@@ -32,7 +35,7 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
|
|
32
35
|
assert_equal @data,@member
|
33
36
|
end
|
34
37
|
|
35
|
-
def
|
38
|
+
def test_data_consistent(results=[],origin={:name=>"MyJobIsToTestReplication",:duty=>"Heavyyyyyyy"})
|
36
39
|
EmRiak.create("member99",{:replication=>{:w=>4}},origin)
|
37
40
|
origin.merge!({:riak_key=>"member99"})
|
38
41
|
sleep(1.5) # Wait for data sync...
|
@@ -74,6 +77,41 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
|
|
74
77
|
should_be_data={:name=>"5",:riak_key=>"member5",:head=>{"x-riak-index-likes_bin"=>"cool, ruby, social, 天天開心, 寫程式好Happy","returnbody"=>"true"}}
|
75
78
|
assert_equal should_be_data,data
|
76
79
|
end
|
80
|
+
|
81
|
+
def test_em_enqueue_pool
|
82
|
+
EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>{:weight10=>["http://127.0.0.1:8091"],:weight5=>["http://127.0.0.1:8092"]}})
|
83
|
+
enqueue_pools={"http://127.0.0.1:8091"=>{:alives=>[],:working=>[]},"http://127.0.0.1:8092"=>{:alives=>[],:working=>[]}}
|
84
|
+
assert_equal enqueue_pools, EmRiak.em_enqueue_pool
|
85
|
+
|
86
|
+
EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>["http://127.0.0.1:8091","http://127.0.0.1:8092"]})
|
87
|
+
assert_equal enqueue_pools, EmRiak.em_enqueue_pool
|
88
|
+
|
89
|
+
counter=0
|
90
|
+
EM.run do
|
91
|
+
(1..100).each{|member_id|
|
92
|
+
EmRiak.create("member#{member_id}",{:id=>member_id}){
|
93
|
+
counter+=1
|
94
|
+
["http://127.0.0.1:8091","http://127.0.0.1:8092"].each do |host|
|
95
|
+
#puts "#{EmRiak.em_enqueue_pool[host][:working].count} - #{EmRiak.em_enqueue_pool[host][:alives].count}"
|
96
|
+
assert_operator EmRiak.em_enqueue_pool[host][:working].count , :<, EmRiak.em_enqueue_pool_max+1
|
97
|
+
end
|
98
|
+
puts "test em_enqueue pool counter #{counter} "
|
99
|
+
EM.stop if counter==100
|
100
|
+
}
|
101
|
+
}
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def test_bucket_replication
|
106
|
+
replication={"n_val"=>2,"r"=>4,"w"=>4}
|
107
|
+
EmRiak.replication={"n_val"=>2,"r"=>4,"w"=>4}
|
108
|
+
assert_equal replication, EmRiak.replication
|
109
|
+
|
110
|
+
props=EmRiak.send :get_bucket_replication, "test-member"
|
111
|
+
props.each{ |prop| replication.each{|key,value| assert_equal value, prop["props"][key] }}
|
112
|
+
|
113
|
+
EmRiak.replication={"n_val"=>3,"r"=>"quorum","w"=>"quorum"}
|
114
|
+
end
|
77
115
|
end
|
78
116
|
|
79
117
|
class TestEmRiakStorageObjcetFunctionality < MiniTest::Unit::TestCase
|
@@ -106,4 +144,20 @@ class TestEmRiakStorageObjcetFunctionality < MiniTest::Unit::TestCase
|
|
106
144
|
assert_equal nil,response
|
107
145
|
assert_equal nil,member
|
108
146
|
end
|
147
|
+
end
|
148
|
+
|
149
|
+
class TestEmRiakVarietyDataType < MiniTest::Unit::TestCase
|
150
|
+
def setup
|
151
|
+
EmRiak::Connection.new({:bucket=>"test-datatype",:async=>false,:hosts=>TEST_RIAK_HOSTS})
|
152
|
+
@data_types_mapper={:Hash=>{:name=>"hihi"},:String=>"hihihi"}
|
153
|
+
@match_data={:Hash=>{:name=>"hihi",:riak_key=>"test-data"},:String=>"hihihi"}
|
154
|
+
end
|
155
|
+
|
156
|
+
def test_data_types
|
157
|
+
@data_types_mapper.each do |data_type,data_body|
|
158
|
+
EmRiak.create("test-data",data_body)
|
159
|
+
data=EmRiak.find("test-data")
|
160
|
+
assert_equal @match_data[data_type],data
|
161
|
+
end
|
162
|
+
end
|
109
163
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: em-riak
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.91
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-03-08 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: escape_utils
|
16
|
-
requirement: &
|
16
|
+
requirement: &70281193511080 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70281193511080
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: yajl-ruby
|
27
|
-
requirement: &
|
27
|
+
requirement: &70281193510640 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70281193510640
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: httpi
|
38
|
-
requirement: &
|
38
|
+
requirement: &70281193510220 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70281193510220
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: eventmachine
|
49
|
-
requirement: &
|
49
|
+
requirement: &70281193509800 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70281193509800
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: em-http-request
|
60
|
-
requirement: &
|
60
|
+
requirement: &70281193509380 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ! '>='
|
@@ -65,7 +65,7 @@ dependencies:
|
|
65
65
|
version: '0'
|
66
66
|
type: :runtime
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *70281193509380
|
69
69
|
description: An extremely fast and convenient riak client for eventmachine.
|
70
70
|
email: von@vonstark.co
|
71
71
|
executables: []
|
@@ -83,6 +83,7 @@ files:
|
|
83
83
|
- lib/em-riak/eventmachine.rb
|
84
84
|
- lib/em-riak/full_text_search.rb
|
85
85
|
- lib/em-riak/grapher.rb
|
86
|
+
- lib/em-riak/httpi_hack.rb
|
86
87
|
- lib/em-riak/luwak.rb
|
87
88
|
- lib/em-riak/map_reduce.rb
|
88
89
|
- lib/em-riak/model.rb
|