em-riak 0.2.8 → 0.2.91

data/README.md

@@ -116,8 +116,6 @@ Use it directly as following :
 * Create new Pull Request  
 
 ### TO-DO
-Full Text Seacrh  
+Full Text Seacrh (This will not going to implement because Basho will replace search with yakozuna)  
 Deployment & Management  
-Protocal Buffer Interface  
-ORM with Model  
-Hadoop MapReduce Support  
+ORM with Model  

data/em-riak.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
     s.name        = 'em-riak'
-    s.version     = '0.2.8'
-    s.date        = '2012-12-28'
+    s.version     = '0.2.91'
+    s.date        = '2013-03-08'
     s.summary     = "Riak client for eventmachine"
     s.description = "An extremely fast and convenient riak client for eventmachine."
     s.authors     = ["Von"]

data/lib/em-riak/basic.rb

@@ -7,24 +7,31 @@ require 'httpi'
 require 'yajl'
 
 module EmRiak
-    attr_accessor :bucket, :interface, :adapter, :hosts, :cluster, :async, :http_client, :vclock, :replication, :debug
+    attr_accessor :bucket, :interface, :adapter, :hosts, :weight, :cluster, :async, :http_client, :vclock, :replication, :debug, :em_enqueue_pool, :em_enqueue_pool_max
     extend self
 
+    
     # A easy interface to create connections
     class Connection
         def initialize(*opts)
             if opts && opts.length>0
                 opts=opts.first
-                methods=[:bucket, :interface, :adapter, :hosts, :cluster, :async, :http_client, :vclock, :repliaction, :debug]
+                methods=[:bucket, :interface, :adapter, :cluster, :async, :http_client, :vclock, :repliaction, :debug]
                 opts.each{|key,opt|  EmRiak.send "#{key}=".to_sym, opt if opt && methods.index(key.to_sym)   }
             else
                 EmRiak.bucket=RIAKBUCKET
             end
-            EmRiak.replication={"W"=>3,"R"=>1,"n_val"=>3} if EmRiak.replication.nil?
+            EmRiak.send :handle_em_hosts, opts[:hosts] 
+            EmRiak.replication={"w"=>"quorum","r"=>"quorum","n_val"=>3}  if EmRiak.replication.nil?
+            
             EmRiak.debug=false if EmRiak.debug.nil?
         end
     end
-
+    def replication=replication_setting
+        @replication=convert_params_layers(replication_setting,:to_s)
+        EmRiak.send :change_bucket_replication ,EmRiak.bucket, EmRiak.replication if EmRiak.replication!={"w"=>"quorum","r"=>1,"n_val"=>"quorum"} 
+    end
+    
     # -- CRUD
     class StorageObject < Hash
         def save(block=nil); EmRiak.save(self,block); end
@@ -44,15 +51,7 @@ module EmRiak
         end
     end
     def create(key,*opts,&callback)
-        data={:body=>{}}
-        if opts && opts.count>0
-            if opts.count==1
-                body=opts.first
-            else
-                opts,body=opts
-            end
-            body.each{|key,value| data[:body][key]=value} 
-        end
+        data,opts=handle_opts_for_body(opts)
         data,callback=handle_callback(key,"save",opts,callback,data)
         operate_bucket= select_bucket(data)
         url="/buckets/#{operate_bucket}/keys/#{key_handler(key)}"
@@ -71,19 +70,14 @@ module EmRiak
         http("get",key,url,data,callback)
     end
 
-    def select_bucket(data)
-        return EmRiak.bucket if !data[:opts]
-        return data[:opts][:bucket] ? data[:opts][:bucket] : EmRiak.bucket
-    end
-
     def save(obj,*opts,&callback)
-        data={:body=>{}}
-        obj.each{|key,value| data[:body][key]=value }
+        data,nothing=handle_opts_for_body([obj])
         data,callback=handle_callback(obj[:riak_key],"save",opts,callback,data)
         operate_bucket= select_bucket(data)
         url="/buckets/#{operate_bucket}/keys/#{obj[:riak_key]}"
         http("put",obj[:riak_key],url,data,callback)
     end
+
     def destroy(key,*opts,&callback)
         obj_key=defined?(obj) ? obj[:riak_key] :  (key.class==String ? key : key[:riak_key])
         data,callback=handle_callback(obj_key,"delete",opts,callback)
@@ -93,6 +87,10 @@ module EmRiak
         key=nil
     end
 
+    def http(method,key,url,data,callback=nil,res=nil)
+        (callback || self.async && ["delete","put","post"].index(method) && key!="mapred") ? em_http(method,key,url,data,callback) : open_http(method,key,url,data)
+    end   
+
     # -- Search
     def search(method_name,*args,&callback)
         result_handler=nil
@@ -143,28 +141,24 @@ module EmRiak
         response
     end
 
-    def http(method,key,url,data,callback=nil,res=nil)
-        (callback || self.async && ["delete","put","post"].index(method) && key!="mapred") ? em_http(method,key,url,data,callback) : open_http(method,key,url,data)
-    end
-
-    protected 
+    protected
     def open_http(method,key,url,data,res={})
         has_http_instance?
         data={:body=>{},:head=>{},:query=>{}} if !data
-
-        http_client.url     = generate_path(url)
+        http_client.url     = generate_path(url)[:url]
         http_client.headers = key!="mapred" ? convert_params_layers(data[:head],:to_s) : data[:head]
-        http_client.body    = key!="mapred" ? convert_params_layers(data[:body],:to_s) : data[:body]
-        http_client.open_timeout = 300 # seconds. change it if you need
-        http_client.read_timeout = 300 # seconds. change it if you need
-
+        http_client.body    = key!="mapred" ? (data[:body].class==String ? data[:body] : convert_params_layers(data[:body],:to_s)) : data[:body]
+        http_client.open_timeout = 500 # seconds. change it if you need
+        http_client.read_timeout = 500 # seconds. change it if you need
         res=HTTPI.request(method.to_sym,http_client)
-        res=res.body if res.class==HTTPI::Response
-        res=handle_response_string_to_object(key,res,method,data)
+        body=res.body if res.class==HTTPI::Response
+        res=handle_response_string_to_object(key,body,res,method,data)
 
         res=nil if res && res.class!=String && res.count<1
-
         return res
+    rescue Exception=> e
+        puts "open http error #{e} with key #{key} #{e.backtrace}"
+        return nil
     end
 
     def em_http(method,key,url,data,async_callback)
@@ -172,40 +166,48 @@ module EmRiak
         begin
             raise "Retry too much time" if data[:retry]>RETRY_TIMES
             
-            #conn options
-            conn_options = {
-              :connect_timeout => EM_REQUEST_TIMEOUT,        # default connection setup timeout
-              :inactivity_timeout => EM_INACTIVITY_TIMEOUT,    # default connection inactivity (post-setup) timeout
-            }
-            #data[:body]=data[:body] if data[:body].class==Hash && data[:head]['Content-Type'] && data[:head]['Content-Type']=="application/json"
-            #request options
             data[:query]="?returnbody=true"
-            request_options=data.reject{|k,v| k==:retry}
-
-            #do job
-            case method
-            when "post"
-                riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).post request_options 
-            when "delete"
-                riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).delete request_options   
-            when "get"
-                riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).get request_options  
-            when "put"
-                riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).put request_options  
+            request_options=data.reject{|k,v| k==:retry} #request options, reject some data before submit
+            path_result=generate_path(url)
+            execute_url  = path_result[:url]
+            host = path_result[:host]
+
+            # == Start
+            # These lines should reduce high em-http recreate cost. (especially if we create ssl conn)
+            # you can increase em_enqueue_pool_max when you init EM.connections. 
+            # By default, it's 30 for each host. so, for example, you have 2 host as : [127.0.0.1:8080,127.0.0.1:8081]. the max em-http-request alive instance would be 60
+            # It should automatically remove extra and remain 3 alive connections after 30 secs.
+            raise "do it later because pool full" if EmRiak.em_enqueue_pool[host][:working].count>=EmRiak.em_enqueue_pool_max
+
+            if EmRiak.em_enqueue_pool[host][:alives].count<1 
+                enqueue=EventMachine::HttpRequest.new(execute_url,{:keepalive=>true}) # keepalive will make all connections alive, so we can avoid heavy cpu load when large amount em_http instance create
+                EmRiak.em_enqueue_pool[host][:working].push(enqueue)
+            else
+                enqueue=EmRiak.em_enqueue_pool[host][:alives].pop
+                EmRiak.em_enqueue_pool[host][:working].push(enqueue)
             end
+            # == End
 
-            riakHttp.errback{ #Error Callback
+            # Do job
+            riakHttp = enqueue.send method.to_sym , :connect_timeout => EM_REQUEST_TIMEOUT, :inactivity_timeout => EM_INACTIVITY_TIMEOUT, :keepalive=>true
+            riakHttp.errback{ # Error Callback
                 puts "#{method} - #{key} - riak callback error #{riakHttp.response}" if EmRiak.debug
                 data[:retry]+=1
-                http(method,key,url,data,async_callback)
+                EM.add_timer(EM_FAIL_RETRY_DELAY){ em_http(method,key,url,data,async_callback) } # retry after random seconds to avoid too heavy load
             }
+            riakHttp.callback{  # Success Callback
+                # remove from enqueue pool
+                pool_index=EmRiak.em_enqueue_pool[host][:working].index(enqueue)
+                EmRiak.em_enqueue_pool[host][:alives].push(EmRiak.em_enqueue_pool[host][:working][pool_index])
+                EmRiak.em_enqueue_pool[host][:working].delete(enqueue)
+                alive_counter=EmRiak.em_enqueue_pool[host][:alives].count
+                EM.add_timer(EM_ENQUEUE_POOL_RELEASE_DELAY){ EmRiak.send :release_enqueue_pool, host, alive_counter }
 
-            #Success Callback
-            riakHttp.callback{ 
                 puts "#{method} - #{url} - #{key} - riak response #{riakHttp.response}" if EmRiak.debug  
                 async_callback.call(riakHttp.response) if async_callback
             }
         rescue Exception => e
+            EM.add_timer(rand(EM_ENQUEUE_POOL_REDO_DELAY)){ em_http(method,key,url,data,async_callback)  }  if e.message=="do it later because pool full"
             puts "em_http - #{method} - #{url} - #{key} error #{e}" if EmRiak.debug
         end
 
@@ -221,18 +223,21 @@ module EmRiak
         res
     end
 
-    def handle_response_string_to_object(key,res,method,data)
+    def handle_response_string_to_object(key,res,httpi_object,method,data)
         raise "response can't be nil" if !res
         return res if key=="mapred"
-        return nil if ["2i"].index(key)
+        return nil if ["2i"].index(key) || ["delete","destroy"].index(method)
 
-        raise "data not found" if ["not found\n"].index(res)
-        raise "method is not match" if !["get","put","post"].index(method)
+        raise "data not found or timeout" if ["not found\n","request timed out\n"].index(res)
+        raise "method is not match with method #{method}" if !["get","put","post","save","delete","destroy"].index(method)
         raise "data is empty" if method=="get" && res.gsub(" ","").length<1
 
-        obj=EmRiak::StorageObject.new()
-
-        if res && res!="not found\n" && 
+        if httpi_object && httpi_object.headers["content-type"]=="text/plain"
+            obj=res
+        elsif res=="not found\n"
+            obj=nil
+        elsif res
+            obj=EmRiak::StorageObject.new()
             if res.class==String
                 obj=return_body_handler(URI.unescape(res.force_encoding("UTF-8")).gsub("+"," ").gsub("=>",":"),obj,"string")
             elsif res.class==Hash
@@ -244,7 +249,7 @@ module EmRiak
         end
         return obj
     rescue Exception=>e
-        puts "handle response error reason #{e}" if EmRiak.debug
+        puts "handle response error reason #{e} #{key} #{method}" if EmRiak.debug
         return nil
     end
 
@@ -269,31 +274,54 @@ module EmRiak
             data[:opts], data[:head], data[:query], callback = option_response
         end
         new_proc=Proc.new{|res|
-            obj=handle_response_string_to_object(key,res,method,data)
+            obj=handle_response_string_to_object(key,res,nil,method,data)
             callback.call(obj)
         } if callback
         
         [data,new_proc]
     end
-    def handle_options(action,opts,header={},query="",extra_setting={})
-        #Load default header setting
-        case action
-        when "save","add"
-            header["Content-Type"]="application/json"
-        when "get"
-            header["Content-Type"]="application/json"
-        when "destroy","delete","remove"
-            header["Content-Type"]="application/json"
-        when "add_link"
-            header["Content-Type"]="application/json"
-        when "remove_link"
-            header["Content-Type"]="application/json"
+    def handle_opts_for_body(opts,data={:body=>{}})
+        if opts && opts.count>0
+            if opts.count==1
+                body=opts.first
+            else
+                opts,body=opts
+            end
+            
+            if body.class==String
+                data[:body]=body
+            else
+                body.each{|key,value| data[:body][key]=value} 
+            end
         end
-
+        [data,opts]
+    end
+    def map_content_type(action,opts) 
+        # TODO : need to implement more types
+
+        # case action
+        # when "save","add"
+        #     "application/json"
+        # when "get"
+        #     "application/json"
+        # when "destroy","delete","remove"
+        #     "application/json"
+        # when "add_link"
+        #     "application/json"
+        # when "remove_link"
+        #     "application/json"
+        # end
+        return "text/plain" if opts.class==String
+        return (opts && opts[:content_type]) ? opts[:content_type] : "application/json"
+    end
+    def handle_options(action,opts,header={},query="",extra_setting={})
         # Map & overwrite if header specisfic
         opts=opts.first if opts && opts.class==Array
 
-        if opts && opts.count>0
+        #Load default header setting
+        header["Content-Type"]=map_content_type(action,opts)
+
+        if opts && opts.class!=String && opts.count>0
             # Map the options class
             callback=opts.reject!{|obj| obj.class==Proc}
 
@@ -354,8 +382,7 @@ module EmRiak
         res << callback if callback
         res
     end
-
-    def key_handler(key,map={"["=>"%5B","]"=>"%5D","+"=>"%2B","/"=>"%2F","("=>"%28",")"=>"%29",":"=>"%3A"})
+    def key_handler(key)
         # To match the key rule from Basho : If your field contains special characters, such as (‘+’,‘/’,‘[’,‘]’,‘(’,‘)’,‘:’ or space), then either surround the phrase in single quotes, or escape each special character with a backslash.
         # map.each{|origin,value| key.gsub!(origin,value) }
         EscapeUtils.escape_url(key.to_s.encode('UTF-8'))
@@ -386,12 +413,111 @@ module EmRiak
     end
 
     private
-    def generate_path(url); EmRiak.hosts[rand(EmRiak.hosts.length)]+url; end
+    def change_bucket_replication(bucket_name,replication_setting,counter=0)
+        urls=return_all_hosts_as_urls
+
+        change_replication_proc=Proc.new do
+            has_http_instance?
+            urls.each do |url|
+                http_client.headers = {'Content-Type'=> 'application/json'}
+                http_client.url     = "#{url}/riak/#{bucket_name}"
+                http_client.body    = json_encode({"props"=>replication_setting})
+                res=HTTPI.request(:put,http_client)
+            end
+        end
+
+        EmRiak.async ? Thread.new{ change_replication_proc.call } : change_replication_proc.call
+    end
+
+    def get_bucket_replication(bucket_name,results=[])
+        urls=return_all_hosts_as_urls
+
+        has_http_instance?
+        urls.each do |url|
+            http_client.headers = {'Content-Type'=> 'application/json'}
+            http_client.url     = "#{url}/riak/#{bucket_name}"
+            res=HTTPI.request(:get,http_client)
+            results << json_decode(res.body)
+        end
+        results
+    end
+
+    def return_all_hosts_as_urls(urls=[])
+        EmRiak.hosts.each do |host| 
+            data= host.class==Array ? host[1] : host
+            if host[1].class==Array
+                data.each{|url| urls << url }
+            else
+                urls << data
+            end
+        end
+        urls
+    end
+
+    def select_bucket(data)
+        return EmRiak.bucket if !data[:opts]
+        return data[:opts][:bucket] ? data[:opts][:bucket] : EmRiak.bucket
+    end
+
+    def generate_path(url,current_weight=0,host_list=[])
+        if EmRiak.weight
+            selected_weight=rand(EmRiak.weight[:sum])
+            EmRiak.weight[:list].each do |weight_num|
+                current_weight+=weight_num
+                if selected_weight<=current_weight
+                    host_list=EmRiak.hosts["weight#{weight_num}".to_sym]
+                    break
+                end
+            end
+            host=host_list[rand(host_list.count)]
+            new_url=host+url
+        else
+            host=EmRiak.hosts[rand(EmRiak.hosts.length)]
+            new_url=host+url
+        end
+        {:host=>host,:url=>new_url}
+    end
     def has_http_instance?
         if !EmRiak.http_client
             HTTPI.log=false
-            HTTPI.adapter=:net_http
+            HTTPI.adapter= EmRiak.adapter ? EmRiak.adapter : :net_http
             EmRiak.http_client=HTTPI::Request.new
         end
     end
+    def handle_em_hosts(emhosts)
+        EmRiak.em_enqueue_pool={}
+        EmRiak.em_enqueue_pool_max=EM_ENQUEUE_POOL_MAX if EmRiak.em_enqueue_pool_max.nil?
+        # We support 2 host methods. 
+        # For easier way, send the hosts as array. ex : ["127.0.0.1:8091","127.0.0.1:8092","127.0.0.1:8093"]
+        # For advance way, configurate the host weight, the higher weight would take heavier load.
+        # Fromat : {:weight10=>["127.0.0.1:8091"],:weight5=>["127.0.0.1:8092"],:weight1=>["127.0.0.1:8093"]}
+        if emhosts.class==Hash
+            EmRiak.weight={:list=>[],:sum=>0}
+            emhosts.each{|weight,hosts| 
+                weight_num=weight.to_s.split("weight")[1] 
+                (hosts).count.times{
+                    EmRiak.weight[:list] << weight_num.to_i
+                    EmRiak.weight[:sum]+=weight_num.to_i
+                }
+            }
+            EmRiak.weight[:list].sort!
+            # create em enqueue pool here
+            emhosts.each{|weight,hosts| hosts.each{|host| EmRiak.em_enqueue_pool[host]={:alives=>[],:working=>[]} } } 
+        else 
+            EmRiak.weight=nil
+            # create em enqueue pool here
+            emhosts.each{|host| EmRiak.em_enqueue_pool[host]={:alives=>[],:working=>[]} }            
+        end
+        EmRiak.hosts=emhosts
+    end
+
+    def release_enqueue_pool(host,alive_counter)
+        operating_pool=EmRiak.em_enqueue_pool[host][:alives]
+        current_counts=operating_pool.count
+        loop{ 
+            enqueue=operating_pool.pop
+            enqueue=nil
+            break if operating_pool.count<=3
+        } if current_counts>7 && current_counts>(alive_counter-5)
+    end 
 end

data/lib/em-riak/configurations.rb

@@ -3,6 +3,10 @@ RESERVED_MAPPER=[:bucket]
 SEARCH_SUPPORT=[:secondary_index,:full_text,:map_reduce]
 RETRY_TIMES=3
 EMMAX=2000
+EM_ENQUEUE_POOL_MAX=30
+EM_ENQUEUE_POOL_RELEASE_DELAY=30
+EM_ENQUEUE_POOL_REDO_DELAY=10
+EM_FAIL_RETRY_DELAY=100
 EM_REQUEST_TIMEOUT=50
 EM_INACTIVITY_TIMEOUT=0
 RIAKBUCKET="test_bucket"

data/lib/em-riak/httpi_hack.rb

@@ -0,0 +1,23 @@
+module HTTPI
+  module Adapter
+    class NetHTTP < Base
+      def request(method)
+        unless REQUEST_METHODS.include? method
+          raise NotSupportedError, "Net::HTTP does not support custom HTTP methods"
+        end
+
+        do_request(method) do |http, http_request|
+          http_request.body = @request.body
+          return_data=http.request http_request
+          http.finish
+          return_data
+        end
+      rescue OpenSSL::SSL::SSLError
+        raise SSLError
+      rescue Errno::ECONNREFUSED   # connection refused
+        $!.extend ConnectionError
+        raise
+      end
+    end
+  end
+end

data/lib/em-riak/map_reduce.rb

@@ -34,6 +34,9 @@ module EmRiak
             res=callback_handler(res,result_handler) if !callback_proc
 
             res
+        rescue Exception=>e 
+            puts "Map Reduce Error #{e} #{e.backtrace}"
+            []
         end
 
         private

data/lib/em-riak/version.rb

@@ -1,3 +1,3 @@
 module EmRiak
-    VERSION = "0.2.8"
+    VERSION = "0.2.91"
 end

data/lib/em-riak.rb

@@ -2,4 +2,5 @@ require 'em-riak/configurations'
 require 'em-riak/basic'
 require 'em-riak/secondary_index'
 require 'em-riak/map_reduce'
+require 'em-riak/httpi_hack'
 require 'em-riak/utils'

data/spec/unit/basic.rb

@@ -5,7 +5,7 @@ require 'em-http-request'
 require 'httpi'
 require 'yajl'
 require 'em-riak'
-TEST_RIAK_HOSTS=["http://127.0.0.1:8091","http://127.0.0.1:8092","http://127.0.0.1:8093","http://127.0.0.1:8094"]
+TEST_RIAK_HOSTS=["http://127.0.0.1:8091"]
 require File.expand_path('../../../lib/em-riak', __FILE__)
 gem 'minitest'
 require 'minitest/autorun'
@@ -50,21 +50,24 @@ class TestAsyncCRUD < MiniTest::Unit::TestCase
     end
     def test_async_find
         EM.run do
+            EmRiak.create("member5",@data)
             EmRiak.find("member5"){|result| 
                 assert_equal @data, result 
                 EM.stop
             }
         end
+        sleep(3)
     end
     def test_async_create
         EM.run do 
             EmRiak.create("member5",@data){|member|
-                assert_equal nil,member # Because Riak does not response data when create, so it's nil. Will response data with next release.
+                assert_equal @data,member[:body] # Because Riak does not response data when create, so it's nil. Will response data with next release.
                 member=EmRiak.find("member5")
                 assert_equal @data, member
                 EM.stop
             }
         end
+        sleep(3)
     end
 
     def test_async_destroy
@@ -76,5 +79,6 @@ class TestAsyncCRUD < MiniTest::Unit::TestCase
                 EM.stop
             }
         end
+        sleep(3)
     end
 end

data/spec/unit/core.rb

@@ -5,7 +5,7 @@ require 'em-http-request'
 require 'httpi'
 require 'yajl'
 require 'em-riak'
-TEST_RIAK_HOSTS=["http://127.0.0.1:8091","http://127.0.0.1:8092","http://127.0.0.1:8093","http://127.0.0.1:8094"]
+TEST_RIAK_HOSTS=["http://127.0.0.1:8091"]
 require File.expand_path('../../../lib/em-riak', __FILE__)
 gem 'minitest'
 require 'minitest/autorun'
@@ -17,8 +17,11 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
         @data={:name=>"5",:riak_key=>"member5"}
     end
     
-    def test_hosts_weight # Will be available in next release
-        assert_equal true,true
+    def test_hosts_weight
+        EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>{:weight10=>["http://127.0.0.1:8091"]}})
+        response=EmRiak.send :generate_path, "/riak/test-member/member5"
+        assert_equal "http://127.0.0.1:8091",response[:host]
+        assert_equal "http://127.0.0.1:8091/riak/test-member/member5",response[:url]
     end
 
     def test_independent_bucket
@@ -32,7 +35,7 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
         assert_equal @data,@member
     end
 
-    def test_replication(results=[],origin={:name=>"MyJobIsToTestReplication",:duty=>"Heavyyyyyyy"})
+    def test_data_consistent(results=[],origin={:name=>"MyJobIsToTestReplication",:duty=>"Heavyyyyyyy"})
         EmRiak.create("member99",{:replication=>{:w=>4}},origin)
         origin.merge!({:riak_key=>"member99"})
         sleep(1.5) # Wait for data sync...
@@ -74,6 +77,41 @@ class TestEmRiakCoreFunctions < MiniTest::Unit::TestCase
         should_be_data={:name=>"5",:riak_key=>"member5",:head=>{"x-riak-index-likes_bin"=>"cool, ruby, social, 天天開心, 寫程式好Happy","returnbody"=>"true"}}
         assert_equal should_be_data,data
     end
+
+    def test_em_enqueue_pool
+        EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>{:weight10=>["http://127.0.0.1:8091"],:weight5=>["http://127.0.0.1:8092"]}})
+        enqueue_pools={"http://127.0.0.1:8091"=>{:alives=>[],:working=>[]},"http://127.0.0.1:8092"=>{:alives=>[],:working=>[]}}
+        assert_equal enqueue_pools, EmRiak.em_enqueue_pool
+
+        EmRiak::Connection.new({:bucket=>"test-member",:async=>false,:hosts=>["http://127.0.0.1:8091","http://127.0.0.1:8092"]})
+        assert_equal enqueue_pools, EmRiak.em_enqueue_pool
+
+        counter=0
+        EM.run do
+            (1..100).each{|member_id|
+                EmRiak.create("member#{member_id}",{:id=>member_id}){ 
+                    counter+=1
+                    ["http://127.0.0.1:8091","http://127.0.0.1:8092"].each do |host|
+                        #puts "#{EmRiak.em_enqueue_pool[host][:working].count} -  #{EmRiak.em_enqueue_pool[host][:alives].count}"
+                        assert_operator EmRiak.em_enqueue_pool[host][:working].count , :<, EmRiak.em_enqueue_pool_max+1
+                    end
+                    puts "test em_enqueue pool counter #{counter} "
+                    EM.stop if counter==100
+                }
+            }
+        end
+    end
+
+    def test_bucket_replication
+        replication={"n_val"=>2,"r"=>4,"w"=>4}
+        EmRiak.replication={"n_val"=>2,"r"=>4,"w"=>4}
+        assert_equal replication, EmRiak.replication
+
+        props=EmRiak.send :get_bucket_replication, "test-member"
+        props.each{ |prop| replication.each{|key,value| assert_equal value, prop["props"][key] }}
+
+        EmRiak.replication={"n_val"=>3,"r"=>"quorum","w"=>"quorum"}
+    end
 end
 
 class TestEmRiakStorageObjcetFunctionality < MiniTest::Unit::TestCase
@@ -106,4 +144,20 @@ class TestEmRiakStorageObjcetFunctionality < MiniTest::Unit::TestCase
         assert_equal nil,response
         assert_equal nil,member
     end
+end
+
+class TestEmRiakVarietyDataType < MiniTest::Unit::TestCase
+    def setup
+        EmRiak::Connection.new({:bucket=>"test-datatype",:async=>false,:hosts=>TEST_RIAK_HOSTS})
+        @data_types_mapper={:Hash=>{:name=>"hihi"},:String=>"hihihi"}
+        @match_data={:Hash=>{:name=>"hihi",:riak_key=>"test-data"},:String=>"hihihi"}
+    end
+
+    def test_data_types
+        @data_types_mapper.each do |data_type,data_body| 
+            EmRiak.create("test-data",data_body)
+            data=EmRiak.find("test-data")
+            assert_equal @match_data[data_type],data
+        end
+    end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: em-riak
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.91
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-28 00:00:00.000000000Z
+date: 2013-03-08 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: escape_utils
-  requirement: &70274801323780 !ruby/object:Gem::Requirement
+  requirement: &70281193511080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274801323780
+  version_requirements: *70281193511080
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70274801322140 !ruby/object:Gem::Requirement
+  requirement: &70281193510640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274801322140
+  version_requirements: *70281193510640
 - !ruby/object:Gem::Dependency
   name: httpi
-  requirement: &70274801321400 !ruby/object:Gem::Requirement
+  requirement: &70281193510220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274801321400
+  version_requirements: *70281193510220
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &70274801320120 !ruby/object:Gem::Requirement
+  requirement: &70281193509800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274801320120
+  version_requirements: *70281193509800
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &70274801319460 !ruby/object:Gem::Requirement
+  requirement: &70281193509380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274801319460
+  version_requirements: *70281193509380
 description: An extremely fast and convenient riak client for eventmachine.
 email: von@vonstark.co
 executables: []
@@ -83,6 +83,7 @@ files:
 - lib/em-riak/eventmachine.rb
 - lib/em-riak/full_text_search.rb
 - lib/em-riak/grapher.rb
+- lib/em-riak/httpi_hack.rb
 - lib/em-riak/luwak.rb
 - lib/em-riak/map_reduce.rb
 - lib/em-riak/model.rb