em-riak 0.2.8 → 0.2.91

Sign up to get free protection for your applications and to get access to all the features.
data/README.md CHANGED
@@ -116,8 +116,6 @@ Use it directly as following :
116
116
  * Create new Pull Request
117
117
 
118
118
  ### TO-DO
119
- Full Text Seacrh
119
+ Full Text Seacrh (This will not going to implement because Basho will replace search with yakozuna)
120
120
  Deployment & Management
121
- Protocal Buffer Interface
122
- ORM with Model
123
- Hadoop MapReduce Support
121
+ ORM with Model
data/em-riak.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'em-riak'
3
- s.version = '0.2.8'
4
- s.date = '2012-12-28'
3
+ s.version = '0.2.91'
4
+ s.date = '2013-03-08'
5
5
  s.summary = "Riak client for eventmachine"
6
6
  s.description = "An extremely fast and convenient riak client for eventmachine."
7
7
  s.authors = ["Von"]
data/lib/em-riak/basic.rb CHANGED
@@ -7,24 +7,31 @@ require 'httpi'
7
7
  require 'yajl'
8
8
 
9
9
  module EmRiak
10
- attr_accessor :bucket, :interface, :adapter, :hosts, :cluster, :async, :http_client, :vclock, :replication, :debug
10
+ attr_accessor :bucket, :interface, :adapter, :hosts, :weight, :cluster, :async, :http_client, :vclock, :replication, :debug, :em_enqueue_pool, :em_enqueue_pool_max
11
11
  extend self
12
12
 
13
+
13
14
  # A easy interface to create connections
14
15
  class Connection
15
16
  def initialize(*opts)
16
17
  if opts && opts.length>0
17
18
  opts=opts.first
18
- methods=[:bucket, :interface, :adapter, :hosts, :cluster, :async, :http_client, :vclock, :repliaction, :debug]
19
+ methods=[:bucket, :interface, :adapter, :cluster, :async, :http_client, :vclock, :repliaction, :debug]
19
20
  opts.each{|key,opt| EmRiak.send "#{key}=".to_sym, opt if opt && methods.index(key.to_sym) }
20
21
  else
21
22
  EmRiak.bucket=RIAKBUCKET
22
23
  end
23
- EmRiak.replication={"W"=>3,"R"=>1,"n_val"=>3} if EmRiak.replication.nil?
24
+ EmRiak.send :handle_em_hosts, opts[:hosts]
25
+ EmRiak.replication={"w"=>"quorum","r"=>"quorum","n_val"=>3} if EmRiak.replication.nil?
26
+
24
27
  EmRiak.debug=false if EmRiak.debug.nil?
25
28
  end
26
29
  end
27
-
30
+ def replication=replication_setting
31
+ @replication=convert_params_layers(replication_setting,:to_s)
32
+ EmRiak.send :change_bucket_replication ,EmRiak.bucket, EmRiak.replication if EmRiak.replication!={"w"=>"quorum","r"=>1,"n_val"=>"quorum"}
33
+ end
34
+
28
35
  # -- CRUD
29
36
  class StorageObject < Hash
30
37
  def save(block=nil); EmRiak.save(self,block); end
@@ -44,15 +51,7 @@ module EmRiak
44
51
  end
45
52
  end
46
53
  def create(key,*opts,&callback)
47
- data={:body=>{}}
48
- if opts && opts.count>0
49
- if opts.count==1
50
- body=opts.first
51
- else
52
- opts,body=opts
53
- end
54
- body.each{|key,value| data[:body][key]=value}
55
- end
54
+ data,opts=handle_opts_for_body(opts)
56
55
  data,callback=handle_callback(key,"save",opts,callback,data)
57
56
  operate_bucket= select_bucket(data)
58
57
  url="/buckets/#{operate_bucket}/keys/#{key_handler(key)}"
@@ -71,19 +70,14 @@ module EmRiak
71
70
  http("get",key,url,data,callback)
72
71
  end
73
72
 
74
- def select_bucket(data)
75
- return EmRiak.bucket if !data[:opts]
76
- return data[:opts][:bucket] ? data[:opts][:bucket] : EmRiak.bucket
77
- end
78
-
79
73
  def save(obj,*opts,&callback)
80
- data={:body=>{}}
81
- obj.each{|key,value| data[:body][key]=value }
74
+ data,nothing=handle_opts_for_body([obj])
82
75
  data,callback=handle_callback(obj[:riak_key],"save",opts,callback,data)
83
76
  operate_bucket= select_bucket(data)
84
77
  url="/buckets/#{operate_bucket}/keys/#{obj[:riak_key]}"
85
78
  http("put",obj[:riak_key],url,data,callback)
86
79
  end
80
+
87
81
  def destroy(key,*opts,&callback)
88
82
  obj_key=defined?(obj) ? obj[:riak_key] : (key.class==String ? key : key[:riak_key])
89
83
  data,callback=handle_callback(obj_key,"delete",opts,callback)
@@ -93,6 +87,10 @@ module EmRiak
93
87
  key=nil
94
88
  end
95
89
 
90
+ def http(method,key,url,data,callback=nil,res=nil)
91
+ (callback || self.async && ["delete","put","post"].index(method) && key!="mapred") ? em_http(method,key,url,data,callback) : open_http(method,key,url,data)
92
+ end
93
+
96
94
  # -- Search
97
95
  def search(method_name,*args,&callback)
98
96
  result_handler=nil
@@ -143,28 +141,24 @@ module EmRiak
143
141
  response
144
142
  end
145
143
 
146
- def http(method,key,url,data,callback=nil,res=nil)
147
- (callback || self.async && ["delete","put","post"].index(method) && key!="mapred") ? em_http(method,key,url,data,callback) : open_http(method,key,url,data)
148
- end
149
-
150
- protected
144
+ protected
151
145
  def open_http(method,key,url,data,res={})
152
146
  has_http_instance?
153
147
  data={:body=>{},:head=>{},:query=>{}} if !data
154
-
155
- http_client.url = generate_path(url)
148
+ http_client.url = generate_path(url)[:url]
156
149
  http_client.headers = key!="mapred" ? convert_params_layers(data[:head],:to_s) : data[:head]
157
- http_client.body = key!="mapred" ? convert_params_layers(data[:body],:to_s) : data[:body]
158
- http_client.open_timeout = 300 # seconds. change it if you need
159
- http_client.read_timeout = 300 # seconds. change it if you need
160
-
150
+ http_client.body = key!="mapred" ? (data[:body].class==String ? data[:body] : convert_params_layers(data[:body],:to_s)) : data[:body]
151
+ http_client.open_timeout = 500 # seconds. change it if you need
152
+ http_client.read_timeout = 500 # seconds. change it if you need
161
153
  res=HTTPI.request(method.to_sym,http_client)
162
- res=res.body if res.class==HTTPI::Response
163
- res=handle_response_string_to_object(key,res,method,data)
154
+ body=res.body if res.class==HTTPI::Response
155
+ res=handle_response_string_to_object(key,body,res,method,data)
164
156
 
165
157
  res=nil if res && res.class!=String && res.count<1
166
-
167
158
  return res
159
+ rescue Exception=> e
160
+ puts "open http error #{e} with key #{key} #{e.backtrace}"
161
+ return nil
168
162
  end
169
163
 
170
164
  def em_http(method,key,url,data,async_callback)
@@ -172,40 +166,48 @@ module EmRiak
172
166
  begin
173
167
  raise "Retry too much time" if data[:retry]>RETRY_TIMES
174
168
 
175
- #conn options
176
- conn_options = {
177
- :connect_timeout => EM_REQUEST_TIMEOUT, # default connection setup timeout
178
- :inactivity_timeout => EM_INACTIVITY_TIMEOUT, # default connection inactivity (post-setup) timeout
179
- }
180
- #data[:body]=data[:body] if data[:body].class==Hash && data[:head]['Content-Type'] && data[:head]['Content-Type']=="application/json"
181
- #request options
182
169
  data[:query]="?returnbody=true"
183
- request_options=data.reject{|k,v| k==:retry}
184
-
185
- #do job
186
- case method
187
- when "post"
188
- riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).post request_options
189
- when "delete"
190
- riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).delete request_options
191
- when "get"
192
- riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).get request_options
193
- when "put"
194
- riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).put request_options
170
+ request_options=data.reject{|k,v| k==:retry} #request options, reject some data before submit
171
+ path_result=generate_path(url)
172
+ execute_url = path_result[:url]
173
+ host = path_result[:host]
174
+
175
+ # == Start
176
+ # These lines should reduce high em-http recreate cost. (especially if we create ssl conn)
177
+ # you can increase em_enqueue_pool_max when you init EM.connections.
178
+ # By default, it's 30 for each host. so, for example, you have 2 host as : [127.0.0.1:8080,127.0.0.1:8081]. the max em-http-request alive instance would be 60
179
+ # It should automatically remove extra and remain 3 alive connections after 30 secs.
180
+ raise "do it later because pool full" if EmRiak.em_enqueue_pool[host][:working].count>=EmRiak.em_enqueue_pool_max
181
+
182
+ if EmRiak.em_enqueue_pool[host][:alives].count<1
183
+ enqueue=EventMachine::HttpRequest.new(execute_url,{:keepalive=>true}) # keepalive will make all connections alive, so we can avoid heavy cpu load when large amount em_http instance create
184
+ EmRiak.em_enqueue_pool[host][:working].push(enqueue)
185
+ else
186
+ enqueue=EmRiak.em_enqueue_pool[host][:alives].pop
187
+ EmRiak.em_enqueue_pool[host][:working].push(enqueue)
195
188
  end
189
+ # == End
196
190
 
197
- riakHttp.errback{ #Error Callback
191
+ # Do job
192
+ riakHttp = enqueue.send method.to_sym , :connect_timeout => EM_REQUEST_TIMEOUT, :inactivity_timeout => EM_INACTIVITY_TIMEOUT, :keepalive=>true
193
+ riakHttp.errback{ # Error Callback
198
194
  puts "#{method} - #{key} - riak callback error #{riakHttp.response}" if EmRiak.debug
199
195
  data[:retry]+=1
200
- http(method,key,url,data,async_callback)
196
+ EM.add_timer(EM_FAIL_RETRY_DELAY){ em_http(method,key,url,data,async_callback) } # retry after random seconds to avoid too heavy load
201
197
  }
198
+ riakHttp.callback{ # Success Callback
199
+ # remove from enqueue pool
200
+ pool_index=EmRiak.em_enqueue_pool[host][:working].index(enqueue)
201
+ EmRiak.em_enqueue_pool[host][:alives].push(EmRiak.em_enqueue_pool[host][:working][pool_index])
202
+ EmRiak.em_enqueue_pool[host][:working].delete(enqueue)
203
+ alive_counter=EmRiak.em_enqueue_pool[host][:alives].count
204
+ EM.add_timer(EM_ENQUEUE_POOL_RELEASE_DELAY){ EmRiak.send :release_enqueue_pool, host, alive_counter }
202
205
 
203
- #Success Callback
204
- riakHttp.callback{
205
206
  puts "#{method} - #{url} - #{key} - riak response #{riakHttp.response}" if EmRiak.debug
206
207
  async_callback.call(riakHttp.response) if async_callback
207
208
  }
208
209
  rescue Exception => e
210
+ EM.add_timer(rand(EM_ENQUEUE_POOL_REDO_DELAY)){ em_http(method,key,url,data,async_callback) } if e.message=="do it later because pool full"
209
211
  puts "em_http - #{method} - #{url} - #{key} error #{e}" if EmRiak.debug
210
212
  end
211
213
 
@@ -221,18 +223,21 @@ module EmRiak
221
223
  res
222
224
  end
223
225
 
224
- def handle_response_string_to_object(key,res,method,data)
226
+ def handle_response_string_to_object(key,res,httpi_object,method,data)
225
227
  raise "response can't be nil" if !res
226
228
  return res if key=="mapred"
227
- return nil if ["2i"].index(key)
229
+ return nil if ["2i"].index(key) || ["delete","destroy"].index(method)
228
230
 
229
- raise "data not found" if ["not found\n"].index(res)
230
- raise "method is not match" if !["get","put","post"].index(method)
231
+ raise "data not found or timeout" if ["not found\n","request timed out\n"].index(res)
232
+ raise "method is not match with method #{method}" if !["get","put","post","save","delete","destroy"].index(method)
231
233
  raise "data is empty" if method=="get" && res.gsub(" ","").length<1
232
234
 
233
- obj=EmRiak::StorageObject.new()
234
-
235
- if res && res!="not found\n" &&
235
+ if httpi_object && httpi_object.headers["content-type"]=="text/plain"
236
+ obj=res
237
+ elsif res=="not found\n"
238
+ obj=nil
239
+ elsif res
240
+ obj=EmRiak::StorageObject.new()
236
241
  if res.class==String
237
242
  obj=return_body_handler(URI.unescape(res.force_encoding("UTF-8")).gsub("+"," ").gsub("=>",":"),obj,"string")
238
243
  elsif res.class==Hash
@@ -244,7 +249,7 @@ module EmRiak
244
249
  end
245
250
  return obj
246
251
  rescue Exception=>e
247
- puts "handle response error reason #{e}" if EmRiak.debug
252
+ puts "handle response error reason #{e} #{key} #{method}" if EmRiak.debug
248
253
  return nil
249
254
  end
250
255
 
@@ -269,31 +274,54 @@ module EmRiak
269
274
  data[:opts], data[:head], data[:query], callback = option_response
270
275
  end
271
276
  new_proc=Proc.new{|res|
272
- obj=handle_response_string_to_object(key,res,method,data)
277
+ obj=handle_response_string_to_object(key,res,nil,method,data)
273
278
  callback.call(obj)
274
279
  } if callback
275
280
 
276
281
  [data,new_proc]
277
282
  end
278
- def handle_options(action,opts,header={},query="",extra_setting={})
279
- #Load default header setting
280
- case action
281
- when "save","add"
282
- header["Content-Type"]="application/json"
283
- when "get"
284
- header["Content-Type"]="application/json"
285
- when "destroy","delete","remove"
286
- header["Content-Type"]="application/json"
287
- when "add_link"
288
- header["Content-Type"]="application/json"
289
- when "remove_link"
290
- header["Content-Type"]="application/json"
283
+ def handle_opts_for_body(opts,data={:body=>{}})
284
+ if opts && opts.count>0
285
+ if opts.count==1
286
+ body=opts.first
287
+ else
288
+ opts,body=opts
289
+ end
290
+
291
+ if body.class==String
292
+ data[:body]=body
293
+ else
294
+ body.each{|key,value| data[:body][key]=value}
295
+ end
291
296
  end
292
-
297
+ [data,opts]
298
+ end
299
+ def map_content_type(action,opts)
300
+ # TODO : need to implement more types
301
+
302
+ # case action
303
+ # when "save","add"
304
+ # "application/json"
305
+ # when "get"
306
+ # "application/json"
307
+ # when "destroy","delete","remove"
308
+ # "application/json"
309
+ # when "add_link"
310
+ # "application/json"
311
+ # when "remove_link"
312
+ # "application/json"
313
+ # end
314
+ return "text/plain" if opts.class==String
315
+ return (opts && opts[:content_type]) ? opts[:content_type] : "application/json"
316
+ end
317
+ def handle_options(action,opts,header={},query="",extra_setting={})
293
318
  # Map & overwrite if header specisfic
294
319
  opts=opts.first if opts && opts.class==Array
295
320
 
296
- if opts && opts.count>0
321
+ #Load default header setting
322
+ header["Content-Type"]=map_content_type(action,opts)
323
+
324
+ if opts && opts.class!=String && opts.count>0
297
325
  # Map the options class
298
326
  callback=opts.reject!{|obj| obj.class==Proc}
299
327
 
@@ -354,8 +382,7 @@ module EmRiak
354
382
  res << callback if callback
355
383
  res
356
384
  end
357
-
358
- def key_handler(key,map={"["=>"%5B","]"=>"%5D","+"=>"%2B","/"=>"%2F","("=>"%28",")"=>"%29",":"=>"%3A"})
385
+ def key_handler(key)
359
386
  # To match the key rule from Basho : If your field contains special characters, such as (‘+’,‘/’,‘[’,‘]’,‘(’,‘)’,‘:’ or space), then either surround the phrase in single quotes, or escape each special character with a backslash.
360
387
  # map.each{|origin,value| key.gsub!(origin,value) }
361
388
  EscapeUtils.escape_url(key.to_s.encode('UTF-8'))
@@ -386,12 +413,111 @@ module EmRiak
386
413
  end
387
414
 
388
415
  private
389
- def generate_path(url); EmRiak.hosts[rand(EmRiak.hosts.length)]+url; end
416
+ def change_bucket_replication(bucket_name,replication_setting,counter=0)
417
+ urls=return_all_hosts_as_urls
418
+
419
+ change_replication_proc=Proc.new do
420
+ has_http_instance?
421
+ urls.each do |url|
422
+ http_client.headers = {'Content-Type'=> 'application/json'}
423
+ http_client.url = "#{url}/riak/#{bucket_name}"
424
+ http_client.body = json_encode({"props"=>replication_setting})
425
+ res=HTTPI.request(:put,http_client)
426
+ end
427
+ end
428
+
429
+ EmRiak.async ? Thread.new{ change_replication_proc.call } : change_replication_proc.call
430
+ end
431
+
432
+ def get_bucket_replication(bucket_name,results=[])
433
+ urls=return_all_hosts_as_urls
434
+
435
+ has_http_instance?
436
+ urls.each do |url|
437
+ http_client.headers = {'Content-Type'=> 'application/json'}
438
+ http_client.url = "#{url}/riak/#{bucket_name}"
439
+ res=HTTPI.request(:get,http_client)
440
+ results << json_decode(res.body)
441
+ end
442
+ results
443
+ end
444
+
445
+ def return_all_hosts_as_urls(urls=[])
446
+ EmRiak.hosts.each do |host|
447
+ data= host.class==Array ? host[1] : host
448
+ if host[1].class==Array
449
+ data.each{|url| urls << url }
450
+ else
451
+ urls << data
452
+ end
453
+ end
454
+ urls
455
+ end
456
+
457
+ def select_bucket(data)
458
+ return EmRiak.bucket if !data[:opts]
459
+ return data[:opts][:bucket] ? data[:opts][:bucket] : EmRiak.bucket
460
+ end
461
+
462
+ def generate_path(url,current_weight=0,host_list=[])
463
+ if EmRiak.weight
464
+ selected_weight=rand(EmRiak.weight[:sum])
465
+ EmRiak.weight[:list].each do |weight_num|
466
+ current_weight+=weight_num
467
+ if selected_weight<=current_weight
468
+ host_list=EmRiak.hosts["weight#{weight_num}".to_sym]
469
+ break
470
+ end
471
+ end
472
+ host=host_list[rand(host_list.count)]
473
+ new_url=host+url
474
+ else
475
+ host=EmRiak.hosts[rand(EmRiak.hosts.length)]
476
+ new_url=host+url
477
+ end
478
+ {:host=>host,:url=>new_url}
479
+ end
390
480
  def has_http_instance?
391
481
  if !EmRiak.http_client
392
482
  HTTPI.log=false
393
- HTTPI.adapter=:net_http
483
+ HTTPI.adapter= EmRiak.adapter ? EmRiak.adapter : :net_http
394
484
  EmRiak.http_client=HTTPI::Request.new
395
485
  end
396
486
  end
487
+ def handle_em_hosts(emhosts)
488
+ EmRiak.em_enqueue_pool={}
489
+ EmRiak.em_enqueue_pool_max=EM_ENQUEUE_POOL_MAX if EmRiak.em_enqueue_pool_max.nil?
490
+ # We support 2 host methods.
491
+ # For easier way, send the hosts as array. ex : ["127.0.0.1:8091","127.0.0.1:8092","127.0.0.1:8093"]
492
+ # For advance way, configurate the host weight, the higher weight would take heavier load.
493
+ # Fromat : {:weight10=>["127.0.0.1:8091"],:weight5=>["127.0.0.1:8092"],:weight1=>["127.0.0.1:8093"]}
494
+ if emhosts.class==Hash
495
+ EmRiak.weight={:list=>[],:sum=>0}
496
+ emhosts.each{|weight,hosts|
497
+ weight_num=weight.to_s.split("weight")[1]
498
+ (hosts).count.times{
499
+ EmRiak.weight[:list] << weight_num.to_i
500
+ EmRiak.weight[:sum]+=weight_num.to_i
501
+ }
502
+ }
503
+ EmRiak.weight[:list].sort!
504
+ # create em enqueue pool here
505
+ emhosts.each{|weight,hosts| hosts.each{|host| EmRiak.em_enqueue_pool[host]={:alives=>[],:working=>[]} } }
506
+ else
507
+ EmRiak.weight=nil
508
+ # create em enqueue pool here
509
+ emhosts.each{|host| EmRiak.em_enqueue_pool[host]={:alives=>[],:working=>[]} }
510
+ end
511
+ EmRiak.hosts=emhosts
512
+ end
513
+
514
+ def release_enqueue_pool(host,alive_counter)
515
+ operating_pool=EmRiak.em_enqueue_pool[host][:alives]
516
+ current_counts=operating_pool.count
517
+ loop{
518
+ enqueue=operating_pool.pop
519
+ enqueue=nil
520
+ break if operating_pool.count<=3
521
+ } if current_counts>7 && current_counts>(alive_counter-5)
522
+ end
397
523
  end
@@ -3,6 +3,10 @@ RESERVED_MAPPER=[:bucket]
3
3
  SEARCH_SUPPORT=[:secondary_index,:full_text,:map_reduce]
4
4
  RETRY_TIMES=3
5
5
  EMMAX=2000
6
+ EM_ENQUEUE_POOL_MAX=30
7
+ EM_ENQUEUE_POOL_RELEASE_DELAY=30
8
+ EM_ENQUEUE_POOL_REDO_DELAY=10
9
+ EM_FAIL_RETRY_DELAY=100
6
10
  EM_REQUEST_TIMEOUT=50
7
11
  EM_INACTIVITY_TIMEOUT=0
8
12
  RIAKBUCKET="test_bucket"
@@ -0,0 +1,23 @@
1
+ module HTTPI
2
+ module Adapter
3
+ class NetHTTP < Base
4
+ def request(method)
5
+ unless REQUEST_METHODS.include? method
6
+ raise NotSupportedError, "Net::HTTP does not support custom HTTP methods"
7
+ end
8
+
9
+ do_request(method) do |http, http_request|
10
+ http_request.body = @request.body
11
+ return_data=http.request http_request
12
+ http.finish
13
+ return_data
14
+ end
15
+ rescue OpenSSL::SSL::SSLError
16
+ raise SSLError
17
+ rescue Errno::ECONNREFUSED # connection refused
18
+ $!.extend ConnectionError
19
+ raise
20
+ end
21
+ end
22
+ end
23
+ end
@@ -34,6 +34,9 @@ module EmRiak
34
34
  res=callback_handler(res,result_handler) if !callback_proc
35
35
 
36
36
  res
37
+ rescue Exception=>e
38
+ puts "Map Reduce Error #{e} #{e.backtrace}"
39
+ []
37
40
  end
38
41
 
39
42
  private
@@ -1,3 +1,3 @@
1
1
  module EmRiak
2
- VERSION = "0.2.8"
2
+ VERSION = "0.2.91"
3
3
  end
data/lib/em-riak.rb CHANGED
@@ -2,4 +2,5 @@ require 'em-riak/configurations'
2
2
  require 'em-riak/basic'
3
3
  require 'em-riak/secondary_index'
4
4
  require 'em-riak/map_reduce'
5
+ require 'em-riak/httpi_hack'
5
6
  require 'em-riak/utils'
data/spec/unit/basic.rb CHANGED
@@ -5,7 +5,7 @@ require 'em-http-request'
5
5
  require 'httpi'
6
6
  require 'yajl'
7
7
  require 'em-riak'
8
- TEST_RIAK_HOSTS=["http://127.0.0.1:8091","http://127.0.0.1:8092","http://127.0.0.1:8093","http://127.0.0.1:8094"]
8
+ TEST_RIAK_HOSTS=["http://127.0.0.1:8091"]
9
9
  require File.expand_path('../../../lib/em-riak', __FILE__)
10
10
  gem 'minitest'
11
11
  require 'minitest/autorun'
@@ -50,21 +50,24 @@ class TestAsyncCRUD < MiniTest::Unit::TestCase
50
50
  end
51
51
  def test_async_find
52
52
  EM.run do
53
+ EmRiak.create("member5",@data)
53
54
  EmRiak.find("member5"){|result|
54
55
  assert_equal @data, result
55
56
  EM.stop
56
57
  }
57
58
  end
59
+ sleep(3)
58
60
  end
59
61
  def test_async_create
60
62
  EM.run do
61
63
  EmRiak.create("member5",@data){|member|
62
- assert_equal nil,member # Because Riak does not response data when create, so it's nil. Will response data with next release.
64
+ assert_equal @data,member[:body] # Because Riak does not response data when create, so it's nil. Will response data with next release.
63
65
  member=EmRiak.find("member5")
64
66
  assert_equal @data, member
65
67
  EM.stop
66
68
  }
67
69
  end
70
+ sleep(3)
68
71
  end
69
72
 
70
73
  def test_async_destroy
@@ -76,5 +79,6 @@ class TestAsyncCRUD < MiniTest::Unit::TestCase
76
79
  EM.stop
77
80
  }
78
81
  end
82
+ sleep(3)
79
83
  end
80
84
  end
data/spec/unit/core.rb CHANGED
@@ -5,7 +5,7 @@ require 'em-http-request'
5
5
  require 'httpi'
6
6
  require 'yajl'
7
7
  require 'em-riak'
8
- TEST_RIAK_HOSTS=["http://127.0.0.1:8091","http://127.0.0.1:8092","http://127.0.0.1:8093","http://127.0.0.1:8094"]
8
+ TEST_RIAK_HOSTS=["http://127.0.0.1:8091"]
9
9
  require File.expand_path('../../../lib/em-riak', __FILE__)
10
10
  gem 'minitest'
11
11
  require 'minitest/autorun'
@@ -17,8 +17,11 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
17
17
  @data={:name=>"5",:riak_key=>"member5"}
18
18
  end
19
19
 
20
- def test_hosts_weight # Will be available in next release
21
- assert_equal true,true
20
+ def test_hosts_weight
21
+ EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>{:weight10=>["http://127.0.0.1:8091"]}})
22
+ response=EmRiak.send :generate_path, "/riak/test-member/member5"
23
+ assert_equal "http://127.0.0.1:8091",response[:host]
24
+ assert_equal "http://127.0.0.1:8091/riak/test-member/member5",response[:url]
22
25
  end
23
26
 
24
27
  def test_independent_bucket
@@ -32,7 +35,7 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
32
35
  assert_equal @data,@member
33
36
  end
34
37
 
35
- def test_replication(results=[],origin={:name=>"MyJobIsToTestReplication",:duty=>"Heavyyyyyyy"})
38
+ def test_data_consistent(results=[],origin={:name=>"MyJobIsToTestReplication",:duty=>"Heavyyyyyyy"})
36
39
  EmRiak.create("member99",{:replication=>{:w=>4}},origin)
37
40
  origin.merge!({:riak_key=>"member99"})
38
41
  sleep(1.5) # Wait for data sync...
@@ -74,6 +77,41 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
74
77
  should_be_data={:name=>"5",:riak_key=>"member5",:head=>{"x-riak-index-likes_bin"=>"cool, ruby, social, 天天開心, 寫程式好Happy","returnbody"=>"true"}}
75
78
  assert_equal should_be_data,data
76
79
  end
80
+
81
+ def test_em_enqueue_pool
82
+ EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>{:weight10=>["http://127.0.0.1:8091"],:weight5=>["http://127.0.0.1:8092"]}})
83
+ enqueue_pools={"http://127.0.0.1:8091"=>{:alives=>[],:working=>[]},"http://127.0.0.1:8092"=>{:alives=>[],:working=>[]}}
84
+ assert_equal enqueue_pools, EmRiak.em_enqueue_pool
85
+
86
+ EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>["http://127.0.0.1:8091","http://127.0.0.1:8092"]})
87
+ assert_equal enqueue_pools, EmRiak.em_enqueue_pool
88
+
89
+ counter=0
90
+ EM.run do
91
+ (1..100).each{|member_id|
92
+ EmRiak.create("member#{member_id}",{:id=>member_id}){
93
+ counter+=1
94
+ ["http://127.0.0.1:8091","http://127.0.0.1:8092"].each do |host|
95
+ #puts "#{EmRiak.em_enqueue_pool[host][:working].count} - #{EmRiak.em_enqueue_pool[host][:alives].count}"
96
+ assert_operator EmRiak.em_enqueue_pool[host][:working].count , :<, EmRiak.em_enqueue_pool_max+1
97
+ end
98
+ puts "test em_enqueue pool counter #{counter} "
99
+ EM.stop if counter==100
100
+ }
101
+ }
102
+ end
103
+ end
104
+
105
+ def test_bucket_replication
106
+ replication={"n_val"=>2,"r"=>4,"w"=>4}
107
+ EmRiak.replication={"n_val"=>2,"r"=>4,"w"=>4}
108
+ assert_equal replication, EmRiak.replication
109
+
110
+ props=EmRiak.send :get_bucket_replication, "test-member"
111
+ props.each{ |prop| replication.each{|key,value| assert_equal value, prop["props"][key] }}
112
+
113
+ EmRiak.replication={"n_val"=>3,"r"=>"quorum","w"=>"quorum"}
114
+ end
77
115
  end
78
116
 
79
117
  class TestEmRiakStorageObjcetFunctionality < MiniTest::Unit::TestCase
@@ -106,4 +144,20 @@ class TestEmRiakStorageObjcetFunctionality < MiniTest::Unit::TestCase
106
144
  assert_equal nil,response
107
145
  assert_equal nil,member
108
146
  end
147
+ end
148
+
149
+ class TestEmRiakVarietyDataType < MiniTest::Unit::TestCase
150
+ def setup
151
+ EmRiak::Connection.new({:bucket=>"test-datatype",:async=>false,:hosts=>TEST_RIAK_HOSTS})
152
+ @data_types_mapper={:Hash=>{:name=>"hihi"},:String=>"hihihi"}
153
+ @match_data={:Hash=>{:name=>"hihi",:riak_key=>"test-data"},:String=>"hihihi"}
154
+ end
155
+
156
+ def test_data_types
157
+ @data_types_mapper.each do |data_type,data_body|
158
+ EmRiak.create("test-data",data_body)
159
+ data=EmRiak.find("test-data")
160
+ assert_equal @match_data[data_type],data
161
+ end
162
+ end
109
163
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: em-riak
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.8
4
+ version: 0.2.91
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-12-28 00:00:00.000000000Z
12
+ date: 2013-03-08 00:00:00.000000000Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: escape_utils
16
- requirement: &70274801323780 !ruby/object:Gem::Requirement
16
+ requirement: &70281193511080 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70274801323780
24
+ version_requirements: *70281193511080
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: yajl-ruby
27
- requirement: &70274801322140 !ruby/object:Gem::Requirement
27
+ requirement: &70281193510640 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *70274801322140
35
+ version_requirements: *70281193510640
36
36
  - !ruby/object:Gem::Dependency
37
37
  name: httpi
38
- requirement: &70274801321400 !ruby/object:Gem::Requirement
38
+ requirement: &70281193510220 !ruby/object:Gem::Requirement
39
39
  none: false
40
40
  requirements:
41
41
  - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
43
43
  version: '0'
44
44
  type: :runtime
45
45
  prerelease: false
46
- version_requirements: *70274801321400
46
+ version_requirements: *70281193510220
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: eventmachine
49
- requirement: &70274801320120 !ruby/object:Gem::Requirement
49
+ requirement: &70281193509800 !ruby/object:Gem::Requirement
50
50
  none: false
51
51
  requirements:
52
52
  - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
54
54
  version: '0'
55
55
  type: :runtime
56
56
  prerelease: false
57
- version_requirements: *70274801320120
57
+ version_requirements: *70281193509800
58
58
  - !ruby/object:Gem::Dependency
59
59
  name: em-http-request
60
- requirement: &70274801319460 !ruby/object:Gem::Requirement
60
+ requirement: &70281193509380 !ruby/object:Gem::Requirement
61
61
  none: false
62
62
  requirements:
63
63
  - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
65
65
  version: '0'
66
66
  type: :runtime
67
67
  prerelease: false
68
- version_requirements: *70274801319460
68
+ version_requirements: *70281193509380
69
69
  description: An extremely fast and convenient riak client for eventmachine.
70
70
  email: von@vonstark.co
71
71
  executables: []
@@ -83,6 +83,7 @@ files:
83
83
  - lib/em-riak/eventmachine.rb
84
84
  - lib/em-riak/full_text_search.rb
85
85
  - lib/em-riak/grapher.rb
86
+ - lib/em-riak/httpi_hack.rb
86
87
  - lib/em-riak/luwak.rb
87
88
  - lib/em-riak/map_reduce.rb
88
89
  - lib/em-riak/model.rb