rhoconnect 3.3.6 → 3.4.2

data/lib/rhoconnect/store.rb

@@ -1,4 +1,6 @@
 require 'digest/sha1'
+require 'set'
+require 'connection_pool'
 
 module Rhoconnect
   
@@ -18,7 +20,7 @@ module Rhoconnect
       def create(server=nil)
         @@db ||= _get_redis(server)
         raise "Error connecting to Redis store." unless @@db and 
-          (@@db.is_a?(Redis) or @@db.is_a?(Redis::Client))
+          (@@db.is_a?(Redis) or @@db.is_a?(Redis::Client) or @@db.is_a?(ConnectionPool::Wrapper))
       end
       
       def start_transaction
@@ -53,7 +55,7 @@ module Rhoconnect
       end
       
       def put_object(dockey, key, data={})
-        _put_object(dockey, key, data)
+        _put_objects(dockey, {key => data})
       end
   
       # Adds set with given data, replaces existing set
@@ -64,12 +66,7 @@ module Rhoconnect
           flash_data(dockey) unless append
           # Inserts a hash or array
           if data.is_a?Hash
-            @@db.pipelined do
-              data.each do |key,value|
-                raise ArgumentError, "Invalid value object: #{value.inspect}. Hash is expected." unless value.is_a?(Hash)
-                _put_object(dockey, key, value)   
-              end
-            end
+            _put_objects(dockey, data)
           else
             put_list(dockey,data,append)
           end
@@ -81,8 +78,8 @@ module Rhoconnect
         if dockey and data
           flash_data(dockey) unless append
           @@db.pipelined do
-            data.each do |value|
-              @@db.rpush(dockey, value)
+            data.each do |element|
+              @@db.rpush(dockey, element)
             end
           end
         end
@@ -95,29 +92,45 @@ module Rhoconnect
         return 0 unless dockey and data
         
         new_object_count = 0
-        objs = get_objects(dockey, data.keys)
+        objs = get_objects(dockey, data.keys) || {}
+        
+        collected_adds = {}
+        collected_rems = {}
+        my_bucket = nil
         @@db.pipelined do
-          data.each do |key,value|
+          data.each do |key,obj|
             is_create = objs[key].nil?
             new_object_count += 1 if is_create
             obj_bucket = _add_bucket_index(dockey, "#{_create_obj_index(key)}")
-            value.each do |attrib,value|
-              next if _is_reserved?(attrib, value)
-              
-              new_element = setelement(key,attrib,value)
-              element_exists = is_create ? false : objs[key].has_key?(attrib)
-              if element_exists
-                existing_element = setelement(key,attrib,objs[key][attrib])
-                if existing_element != new_element
-                  @@db.srem(obj_bucket, existing_element)
-                  @@db.sadd(obj_bucket, new_element)
-                end
-              else
-                @@db.sadd(obj_bucket, new_element)
-              end
+            
+            # collect SREM (if object exists in DB)
+            unless is_create
+              old_element = set_obj_element(key,objs[key])
+              collected_rems[obj_bucket] ||= []
+              collected_rems[obj_bucket] << old_element
             end
+            # update the object and collect SADD
+            objs[key] ||= {}
+            objs[key].merge!(obj)
+            
+            new_element = set_obj_element(key,objs[key])
+            collected_adds[obj_bucket] ||= []
+            collected_adds[obj_bucket] << new_element
+          end
+          # process all SADD and SREM commands as one
+          # SREM must go first
+          collected_rems.each do |bucket, bucket_data|
+            @@db.srem(bucket, bucket_data)
+          end
+          collected_adds.each do |bucket, bucket_data|
+            @@db.sadd(bucket, bucket_data)
           end
         end
+        
+        
+        #data1 = @@db.smembers(my_bucket)
+        #puts "data1 is #{data1.inspect}"
+        
         new_object_count
       end
       
@@ -125,28 +138,14 @@ module Rhoconnect
       def delete_objects(dockey,data=[])
         return 0 unless dockey and data
         
-        deleted_object_count = 0
         objs = get_objects(dockey, data)
-        @@db.pipelined do
-          data.each do |id|
-            if _delete_object(dockey, id, objs[id])
-              deleted_object_count += 1
-            end
-          end
-        end
-        _cleanup_buckets(dockey, objs.keys)
-        deleted_object_count
+        _delete_objects(dockey, objs)
       end
       
       # Deletes data from a given doctype,source,user
       def delete_data(dockey,data={})
         if dockey and data
-          @@db.pipelined do
-            data.each do |key,value|
-              _delete_object(dockey, key, value)
-            end
-          end
-          _cleanup_buckets(dockey, data.keys)
+          _delete_objects(dockey, data)
         end
         true
       end
@@ -180,15 +179,12 @@ module Rhoconnect
       end
       
       def get_object(dockey, key)
-        _get_object(dockey, key)
+        res = _get_objects(dockey, [key])
+        (res and res.size > 0) ? res.values[0] : nil
       end
       
       def get_objects(dockey, keys)
-        res = {}
-        keys.each do |key|
-          res[key] = _get_object(dockey, key)
-        end if keys
-        res
+        _get_objects(dockey, keys)
       end
   
       # Retrieves set for given dockey,source,user
@@ -197,14 +193,18 @@ module Rhoconnect
         if dockey
           if type == Hash
             buckets = _get_buckets(dockey)
-            buckets.each do |bucket|
-              members = @@db.smembers(bucket)
-              members.each do |element|
-                key,attrib,value = getelement(element)
-                res[key] = {} unless res[key]
-                res[key].merge!({attrib => value})
-              end if members
-            end if buckets
+            members = @@db.pipelined do
+              buckets.each do |bucket|
+                @@db.smembers(bucket)
+              end if buckets
+            end
+            members.each do |elements|
+              elements.each do |element|
+                key,obj = get_obj_element(element)
+                res[key] = obj
+                #res[key].merge!({attrib => value})
+              end if elements
+            end if members
           else
             res = get_list(dockey)
           end
@@ -221,35 +221,116 @@ module Rhoconnect
       end
   
       # Retrieves diff data hash between two sets
+      # each entry is in the form of DIFF_OBJ_ELEMENT => [OBJ_KEY, OBJ_DATA_PAIRS]
       def get_diff_data(src_dockey,dst_dockey,p_size=nil)
         res = {}
         if src_dockey and dst_dockey
           # obtain combined indices
           indices = @@db.hgetall("#{dst_dockey}:indices")
-          indices.merge!(@@db.hgetall("#{dst_dockey}:indices"))
           indices.keys.each do |index|
             dst_bucket_name = "#{dst_dockey}:#{index}"
             src_bucket_name = "#{src_dockey}:#{index}"
-            @@db.sdiff(dst_bucket_name,src_bucket_name).each do |element|
-              key,attrib,value = getelement(element)
-              res[key] = {} unless res[key]
-              res[key].merge!({attrib => value})
+            diff_elements =  @@db.sdiff(dst_bucket_name,src_bucket_name)
+            diff_elements.each do |element|
+              keypairs = get_obj_key_and_pairs(element)
+              next unless keypairs
+              res[element] = keypairs
+              return res if p_size and (res.size >= p_size)
             end
-            break if p_size and (res.size >= p_size)
           end
         end
-        if p_size
-          diff = {}
-          page_size = p_size
-          res.each do |key,item|
-            diff[key] = item
-            page_size -= 1
-            break if page_size <= 0         
+        res
+      end
+      
+      def get_inserts_deletes(inserts_elements_map, deletes_elements_map)
+        inserts_obj_hash = {}
+        inserts_elements_map.each do |element,keypairs|
+          key,obj_pairs = keypairs[0],keypairs[1]
+          next unless (key and obj_pairs)
+          inserts_obj_hash[key] = Set.new(obj_pairs)
+        end
+        
+        deletes_obj_hash = {}
+        deletes_elements_map.each do |element,keypairs|
+          key,obj_pairs = keypairs[0],keypairs[1]
+          next unless (key and obj_pairs)
+          deletes_obj_hash[key] = Set.new(obj_pairs)
+        end
+        # modified attributes
+        inserts = {}
+        deletes = {}
+        
+        inserts_obj_hash.each do |key, obj_set|
+          deletes_pairs = nil
+          inserts_pairs = nil
+          if deletes_obj_hash.has_key?(key)
+            deletes_pairs = deletes_obj_hash[key].dup.subtract(obj_set).to_a
+            inserts_pairs = obj_set.dup.subtract(deletes_obj_hash[key]).to_a
+            # remove the key from the deletes set - we already processed it
+            deletes_obj_hash.delete(key)
+          else
+            # if object is not in the deletes set - then, it's all inserts
+            inserts_pairs = obj_set.to_a
+          end
+          # split resulting pairs
+          if inserts_pairs and inserts_pairs.size > 0
+            inserts[key] = split_obj_pairs(inserts_pairs)
+          end
+          if deletes_pairs and deletes_pairs.size > 0
+            deletes[key] = split_obj_pairs(deletes_pairs)
           end
-          diff
-        else  
-          res
         end
+        # after we analyzed the inserts__obj_hash
+        # => deletes_obj_hash should contain only the unmatched deletes
+        deletes_obj_hash.each do |key, obj_set|
+          if obj_set.size > 0
+            deletes[key] = split_obj_pairs(obj_set.to_a)
+          end
+        end
+        
+        [inserts, deletes]   
+      end
+      
+      def update_elements(dockey, inserts_elements_map, deletes_elements_map)
+        indices_to_cleanup = Set.new
+        @@db.pipelined do
+          collected_adds = {}
+          collected_rems = {}
+          
+          inserts_elements_map.each do |element,keypairs|
+            key = keypairs[0]
+            next if not key or not element or element.size == 0
+          
+            obj_bucket_index = _create_obj_index(key)
+            bucket_name = "#{dockey}:#{obj_bucket_index}"
+            _add_bucket_index(dockey, obj_bucket_index)
+            
+            collected_adds[bucket_name] ||= []
+            collected_adds[bucket_name] << element
+          end
+          
+          deletes_elements_map.each do |element,keypairs|
+            key = keypairs[0]
+            next if not key or not element or element.size == 0
+          
+            obj_bucket_index = _create_obj_index(key)
+            bucket_name = "#{dockey}:#{obj_bucket_index}"
+            indices_to_cleanup << bucket_name
+            
+            collected_rems[bucket_name] ||= []
+            collected_rems[bucket_name] << element
+          end
+          
+          # now, perform SREM first, then SADD
+          collected_rems.each do |bucket, bucket_data|
+            @@db.srem(bucket, bucket_data)
+          end
+          collected_adds.each do |bucket,bucket_data|
+            @@db.sadd(bucket, bucket_data)
+          end
+        end 
+        # now, cleanup buckets if necessary
+        _cleanup_buckets(dockey, indices_to_cleanup.to_a)
       end
 
       # Deletes all keys matching a given mask
@@ -367,7 +448,7 @@ module Rhoconnect
         data.each do |key,hash_value|
           unique_record_key = setelement(current_score,assoc_key, key)
           Store.db.zadd(dockey, current_score, unique_record_key)
-          Store.put_data("#{dockey}:#{unique_record_key}",{key => hash_value})
+         	Store.put_data("#{dockey}:#{unique_record_key}",{key => hash_value})
         end
         true
       end
@@ -403,6 +484,62 @@ module Rhoconnect
         Store.db.zremrangebyrank(dockey, 0, -1)
       end
       
+      # these methods should be used for transactional, multi-source CUD queue requests
+      def put_multi_zdata(dockey,assoc_key,sources=[],data={},append=false)
+        return true unless (dockey and assoc_key and sources and data)
+        flush_multi_zdata(dockey) unless append
+        current_score = 0
+        current_score_data = Store.db.zrevrange(dockey,0,0,:with_scores => true)
+        current_score = current_score_data[-1][1].to_i if current_score_data and current_score_data[-1]
+        current_score += 1
+        Store.put_data("#{dockey}:#{current_score.to_i}:#{assoc_key}:sources", sources)
+        data.each do |source_key,source_hashes|
+          source_hashes.each do |operation,obj_hashes|
+            unique_zrecord_key = setelement("#{current_score.to_i},#{assoc_key}",source_key,operation)
+            Store.db.zadd(dockey, current_score.to_i, unique_zrecord_key)
+            Store.put_data("#{dockey}:#{unique_zrecord_key}", obj_hashes)
+         	end
+        end
+        true
+      end
+      
+      def get_multi_zdata(dockey)
+        data = Store.db.zrange(dockey, 0, -1)
+        ret = []
+        sources = []
+        keys = []
+        unless data.nil?
+          scores = []
+          data.each do |zsetkey|
+            scored_assoc_key,source_key,operation = getelement(zsetkey)
+            score,assoc_key = scored_assoc_key.split(',')
+            obj_hash = Store.get_data "#{dockey}:#{zsetkey}"
+            if scores[-1] != score.to_i
+              sources << Store.get_data("#{dockey}:#{score}:#{assoc_key}:sources", Array)
+              ret << {source_key => {operation => obj_hash}}
+              scores << score.to_i
+              keys << assoc_key
+            else
+              ret[-1][source_key] ||= {}
+              ret[-1][source_key].merge!({operation => obj_hash})
+            end
+          end
+        end
+        [ret, sources, keys]
+      end
+      
+      # Deletes all keys and their hashes from the Redis DB
+      def flush_multi_zdata(dockey)
+        data = Store.db.zrange(dockey, 0, -1)
+        data.each do |zsetkey|
+          scored_assoc_key,source_key,operation = getelement(zsetkey)
+          score,assoc_key = scored_assoc_key.split(',')
+          _delete_doc("#{dockey}:#{zsetkey}")
+          _delete_doc("#{dockey}:#{score}:#{assoc_key}:sources")
+        end
+        Store.db.zremrangebyrank(dockey, 0, -1)
+      end
+      
       def exists?(key)
         @@db.exists(key) || @@db.exists("#{key}:indices")
       end
@@ -411,20 +548,46 @@ module Rhoconnect
       alias_method :set_data, :put_data
       
       private
+if RUBY_VERSION =~ /1.9/
       def _get_redis(server=nil)
         url = ENV[REDIS_URL] || ENV[REDISTOGO_URL] || nil
         if url
-          Redis.connect(:url => url)
+          ConnectionPool::Wrapper.new(:size => Rhoconnect.connection_pool_size,
+                                      :timeout => Rhoconnect.connection_pool_timeout) do
+            Redis.connect(:url => url, :timeout => Rhoconnect.redis_timeout, :thread_safe => true)
+          end
         elsif server and server.is_a?(String)
           host,port,db,password = server.split(':')
-          Redis.new(:thread_safe => true, :host => host,
-            :port => port, :db => db, :password => password)
-        elsif server and server.is_a?(Redis)
+          ConnectionPool::Wrapper.new(:size => Rhoconnect.connection_pool_size,
+                                      :timeout => Rhoconnect.connection_pool_timeout) do
+            Redis.connect(:thread_safe => true, :host => host,
+              :port => port, :db => db, :password => password, :timeout => Rhoconnect.redis_timeout)
+          end
+        elsif server and (server.is_a?(Redis) or server.is_a?(ConnectionPool::Wrapper))
           server
         else
-          Redis.new(:thread_safe => true)
+          ConnectionPool::Wrapper.new(:size => 5, :timeout => 30) do
+            Redis.connect(:timeout => 30, :thread_safe => true)
+          end
         end
       end
+else # Ruby 1.8 does not support Connnection Pools      
+      def _get_redis(server=nil)
+        url = ENV[REDIS_URL] || ENV[REDISTOGO_URL] || nil
+        if url
+          Redis.connect(:url => url, :timeout => Rhoconnect.redis_timeout, :thread_safe => true)
+        elsif server and server.is_a?(String)
+          host,port,db,password = server.split(':')
+          Redis.connect(:thread_safe => true, :host => host,
+              :port => port, :db => db, :password => password, :timeout => Rhoconnect.redis_timeout)
+        elsif server and (server.is_a?(Redis) or server.is_a?(ConnectionPool::Wrapper))
+          server
+        else
+          Redis.connect(:timeout => 30, :thread_safe => true)
+        end
+      end
+end # end of if RUBY_VERSION
+
       
       def _lock_key(dockey)
         "lock:#{dockey}"
@@ -476,57 +639,142 @@ module Rhoconnect
         @@db.hvals("#{dockey}:indices")
       end
       
-      def _cleanup_buckets(dockey, keys)
-        keys.each do |key|
-          obj_index = _create_obj_index(key)
-          bucket_name = "#{dockey}:#{obj_index}"
-          _remove_bucket_index(dockey, obj_index) unless @@db.exists(bucket_name)
-        end if keys
+      def _cleanup_buckets(dockey, indices_to_cleanup)
+        indices_to_cleanup.each do |index|
+          bucket_name = "#{dockey}:#{index}"
+          _remove_bucket_index(dockey, index) unless @@db.exists(bucket_name)
+        end if indices_to_cleanup
       end
       
-      def _put_object(dockey, key, obj={})
-        return if obj.empty? or not dockey or not key
+      def _put_objects(dockey, data={})
+        return if data.empty? or not dockey
         
-        obj_bucket_index = _create_obj_index(key)
-        bucket_name = "#{dockey}:#{obj_bucket_index}"
-        _add_bucket_index(dockey, obj_bucket_index)
+        collected_adds = {}
         @@db.pipelined do
-          obj.each do |attrib, value|
-            unless _is_reserved?(attrib,value)
-              @@db.sadd(bucket_name,setelement(key,attrib,value))
-            end
+          data.each do |key,obj|
+            raise ArgumentError, "Invalid value object: #{obj.inspect}. Hash is expected." unless obj.is_a?(Hash)
+            next if obj.empty? or not key
+              
+            obj_bucket_index = _create_obj_index(key)
+            bucket_name = "#{dockey}:#{obj_bucket_index}"
+            _add_bucket_index(dockey, obj_bucket_index)
+            collected_adds[bucket_name] ||= []
+            collected_adds[bucket_name] << set_obj_element(key, obj)
+            
+          end
+          # all SADD operations on a bucket key
+          # are combined into one - it proves to perform faster
+          collected_adds.each do |bucket,bucket_data|
+            @@db.sadd(bucket, bucket_data)
           end
         end
       end
       
-      def _delete_object(dockey, key, obj)
-        return false if not key or not dockey or not obj
+      def _delete_objects(dockey, data={})
+        return 0 if data.empty? or not dockey
         
-        obj_bucket_index = _create_obj_index(key)
-        bucket_name = "#{dockey}:#{obj_bucket_index}"
+        deleted_object_count = 0
+        indices_to_cleanup = Set.new
+        collected_rems = {}
         @@db.pipelined do
-          obj.each do |attrib,value|
-            @@db.srem(bucket_name, setelement(key,attrib,value))
+          data.each do |key, obj|
+            next if obj.empty? or not key
+            obj_bucket_index = _create_obj_index(key)
+            bucket_name = "#{dockey}:#{obj_bucket_index}"
+            indices_to_cleanup << obj_bucket_index
+            
+            collected_rems[bucket_name] ||= []
+            collected_rems[bucket_name] << set_obj_element(key, obj)
+            deleted_object_count += 1
+          end
+          
+          # all SREM operations on a bucket
+          # are combined into one
+          collected_rems.each do |bucket,bucket_data|
+            @@db.srem(bucket, bucket_data)
           end
         end
-        true
+        _cleanup_buckets(dockey, indices_to_cleanup.to_a)
+        deleted_object_count
       end
       
-      def _get_object(dockey, key)
+      def _get_objects(dockey, keys)
         return nil unless dockey
         res = nil
-        obj_bucket_index = _create_obj_index(key)
-        bucket_name = "#{dockey}:#{obj_bucket_index}"
-        key_str_size = "#{key}".size
-        members = @@db.smembers(bucket_name)
-        members.each do |element|
-          next unless element[0..key_str_size - 1] == key
-          obj_id,attrib,value = getelement(element)
-          res ||= {}
-          res[attrib] = value
+        keys_map = Set.new
+        buckets = Set.new
+        keys.each do |key|
+          obj_bucket_index = _create_obj_index(key)
+          bucket_name = "#{dockey}:#{obj_bucket_index}"
+          keys_map << key
+          buckets << bucket_name
+        end
+        members = @@db.pipelined do
+          buckets.to_a.each do |bucket_name|
+            @@db.smembers(bucket_name)
+          end
+        end
+        members.each do |bucket_data|
+          bucket_data.each do |element|
+            key,pairs = get_obj_key_and_pairs(element)
+            next unless keys_map.include?(key)
+            obj = split_obj_pairs(pairs)
+            next if obj.empty?
+            res ||= {}
+            res[key] = obj
+          end if bucket_data
         end if members
         res
       end
+      
+      # operations on object elements      
+      def get_obj_element(elem)
+        key,pairs = get_obj_key_and_pairs(elem)
+        return unless (key and pairs)
+        [key,split_obj_pairs(pairs)]
+      end
+      def get_obj_key_and_pairs(elem)
+        pairs = elem.split(":^rho&:")
+        return unless pairs
+        [pairs[0], pairs[1..-1]]
+      end
+      def split_obj_pairs(pairs)
+        obj = {}
+        pairs.each do |pair|
+          attrib,value = pair.split(':',2)
+          obj[attrib] = value
+        end
+        obj
+      end
+
+# Set Obj Element MUST ensure the order of attribs
+# In 1.8.7 Hash keys are not-sorted - therefore
+# to ensure same order - we sort them before storing      
+if RUBY_VERSION =~ /1.8/
+      def set_obj_element(key, obj)
+        return unless (key and key.size > 0 and obj and obj.size > 0)
+        elem = "#{key}"
+        obj.sort.each do |attrib, value|
+          unless _is_reserved?(attrib,value)
+            elem += ":^rho&:#{attrib}:#{value}"
+          end
+        end
+        elem
+      end
+# in Ruby 1.9.x Hash keys are sorted (always in the same order), so 
+# we do not need to do redundant sorting
+else
+    def set_obj_element(key, obj)
+      return unless (key and key.size > 0 and obj and obj.size > 0)
+      elem = "#{key}"
+      obj.each do |attrib, value|
+        unless _is_reserved?(attrib,value)
+          elem += ":^rho&:#{attrib}:#{value}"
+        end
+      end
+      elem
+    end
+end # if Ruby 1.8
     end
   end
 end