rhoconnect 3.2.1 → 3.3.1.beta2

data/lib/rhoconnect/store.rb

@@ -1,3 +1,5 @@
+require 'digest/sha1'
+
 module Rhoconnect
   
   class StoreLockException < RuntimeError; end
@@ -18,6 +20,41 @@ module Rhoconnect
         raise "Error connecting to Redis store." unless @@db and 
           (@@db.is_a?(Redis) or @@db.is_a?(Redis::Client))
       end
+      
+      def start_transaction
+        @@db.multi
+      end
+      
+      def execute_transaction
+        @@db.exec
+      end
+      
+      def doc_type(dockey)
+        @@db.type(dockey) if dockey
+      end
+      
+      def set_db_doc(dockey, data, append=false)
+        if data.is_a?(String)
+          put_value(dockey, data)
+        else
+          put_data(dockey, data, append)
+        end
+      end
+      
+      def get_db_doc(dockey)
+        doctype = Store.doc_type(dockey)
+        if doctype == 'string'
+          Store.get_value(dockey)
+        elsif doctype == 'list'
+          Store.get_data(dockey, Array).to_json
+        else
+          Store.get_data(dockey).to_json
+        end
+      end
+      
+      def put_object(dockey, key, data={})
+        _put_object(dockey, key, data)
+      end
   
       # Adds set with given data, replaces existing set
       # if it exists or appends data to the existing set
@@ -26,22 +63,26 @@ module Rhoconnect
         if dockey and data
           flash_data(dockey) unless append
           # Inserts a hash or array
-          if data.is_a?(Hash)
+          if data.is_a?Hash
             @@db.pipelined do
               data.each do |key,value|
-                raise ArgumentError, "Invalid value object: #{value.inspect}. Hash is expected." unless value.is_a?(Hash)   
-                value.each do |attrib,value|
-                  unless _is_reserved?(attrib,value)
-                    @@db.sadd(dockey,setelement(key,attrib,value))
-                  end
-                end
+                raise ArgumentError, "Invalid value object: #{value.inspect}. Hash is expected." unless value.is_a?(Hash)
+                _put_object(dockey, key, value)   
               end
             end
           else
-            @@db.pipelined do
-              data.each do |value|
-                @@db.sadd(dockey,value)
-              end
+            put_list(dockey,data,append)
+          end
+        end
+        true
+      end
+      
+      def put_list(dockey, data=[], append=false)
+        if dockey and data
+          flash_data(dockey) unless append
+          @@db.pipelined do
+            data.each do |value|
+              @@db.rpush(dockey, value)
             end
           end
         end
@@ -54,24 +95,25 @@ module Rhoconnect
         return 0 unless dockey and data
         
         new_object_count = 0
-        doc = get_data(dockey)
+        objs = get_objects(dockey, data.keys)
         @@db.pipelined do
           data.each do |key,value|
-            is_create = doc[key].nil?
+            is_create = objs[key].nil?
             new_object_count += 1 if is_create
+            obj_bucket = _add_bucket_index(dockey, "#{_create_obj_index(key)}")
             value.each do |attrib,value|
               next if _is_reserved?(attrib, value)
               
               new_element = setelement(key,attrib,value)
-              element_exists = is_create ? false : doc[key].has_key?(attrib)
+              element_exists = is_create ? false : objs[key].has_key?(attrib)
               if element_exists
-                existing_element = setelement(key,attrib,doc[key][attrib])
+                existing_element = setelement(key,attrib,objs[key][attrib])
                 if existing_element != new_element
-                  @@db.srem(dockey, existing_element)
-                  @@db.sadd(dockey, new_element)
+                  @@db.srem(obj_bucket, existing_element)
+                  @@db.sadd(obj_bucket, new_element)
                 end
               else
-                @@db.sadd(dockey, new_element)
+                @@db.sadd(obj_bucket, new_element)
               end
             end
           end
@@ -84,20 +126,30 @@ module Rhoconnect
         return 0 unless dockey and data
         
         deleted_object_count = 0
-        doc = get_data(dockey)
+        objs = get_objects(dockey, data)
         @@db.pipelined do
           data.each do |id|
-            if doc[id]
-              doc[id].each do |name,value|
-                @@db.srem(dockey, setelement(id,name,value))
-              end
+            if _delete_object(dockey, id, objs[id])
               deleted_object_count += 1
             end
-            doc.delete(id)
           end
         end
+        _cleanup_buckets(dockey, objs.keys)
         deleted_object_count
       end
+      
+      # Deletes data from a given doctype,source,user
+      def delete_data(dockey,data={})
+        if dockey and data
+          @@db.pipelined do
+            data.each do |key,value|
+              _delete_object(dockey, key, value)
+            end
+          end
+          _cleanup_buckets(dockey, data.keys)
+        end
+        true
+      end
     
       # Adds a simple key/value pair
       def put_value(dockey,value)
@@ -122,33 +174,68 @@ module Rhoconnect
       def decr(dockey)
         @@db.decr(dockey)
       end
+      
+      def update_count(dockey, count)
+        Store.db.incrby(dockey, count)
+      end
+      
+      def get_object(dockey, key)
+        _get_object(dockey, key)
+      end
+      
+      def get_objects(dockey, keys)
+        res = {}
+        keys.each do |key|
+          res[key] = _get_object(dockey, key)
+        end if keys
+        res
+      end
   
       # Retrieves set for given dockey,source,user
       def get_data(dockey,type=Hash)
         res = type == Hash ? {} : []
         if dockey
-          members = @@db.smembers(dockey)
-          members.each do |element|
-            if type == Hash
-              key,attrib,value = getelement(element)
-              res[key] = {} unless res[key]
-              res[key].merge!({attrib => value})
-            else
-              res << element
-            end
-          end if members
-          res
+          if type == Hash
+            buckets = _get_buckets(dockey)
+            buckets.each do |bucket|
+              members = @@db.smembers(bucket)
+              members.each do |element|
+                key,attrib,value = getelement(element)
+                res[key] = {} unless res[key]
+                res[key].merge!({attrib => value})
+              end if members
+            end if buckets
+          else
+            res = get_list(dockey)
+          end
         end
+        res
+      end
+      
+      def get_list(dockey)
+        res = []
+        if dockey
+          res = @@db.lrange(dockey, 0, -1)
+        end
+        res
       end
   
       # Retrieves diff data hash between two sets
       def get_diff_data(src_dockey,dst_dockey,p_size=nil)
         res = {}
         if src_dockey and dst_dockey
-          @@db.sdiff(dst_dockey,src_dockey).each do |element|
-            key,attrib,value = getelement(element)
-            res[key] = {} unless res[key]
-            res[key].merge!({attrib => value})
+          # obtain combined indices
+          indices = @@db.hgetall("#{dst_dockey}:indices")
+          indices.merge!(@@db.hgetall("#{dst_dockey}:indices"))
+          indices.keys.each do |index|
+            dst_bucket_name = "#{dst_dockey}:#{index}"
+            src_bucket_name = "#{src_dockey}:#{index}"
+            @@db.sdiff(dst_bucket_name,src_bucket_name).each do |element|
+              key,attrib,value = getelement(element)
+              res[key] = {} unless res[key]
+              res[key].merge!({attrib => value})
+            end
+            break if p_size and (res.size >= p_size)
           end
         end
         if p_size
@@ -159,56 +246,28 @@ module Rhoconnect
             page_size -= 1
             break if page_size <= 0         
           end
-          [diff,res.size]
+          diff
         else  
-          [res,res.size]
+          res
         end
       end
 
-      # Deletes data from a given doctype,source,user
-      def delete_data(dockey,data={})
-        if dockey and data
-          @@db.pipelined do
-            data.each do |key,value|
-              value.each do |attrib,val|
-                @@db.srem(dockey,setelement(key,attrib,val))
-              end
-            end
-          end
-        end
-        true
-      end
-    
       # Deletes all keys matching a given mask
       def flash_data(keymask)
-        if keymask[/[*\[\]?]/]
+        if keymask.to_s[/[*\[\]?]/]
           # If the keymask contains any pattern matching characters
           # Use keys command to find all keys matching pattern (this is extremely expensive)
           # Then delete matches
           @@db.keys(keymask).each do |key|
-            @@db.del(key)
+            _delete_doc(key)
           end
         else
           # The keymask doesn't contain pattern matching characters
           # A delete call is all that is needed
-          @@db.del(keymask)
+          _delete_doc(keymask)
         end
       end
       
-      def exists?(key)
-        @@db.exists(key)
-      end
-    
-      # Returns array of keys matching a given keymask
-      def get_keys(keymask)
-        @@db.keys(keymask)
-      end
-    
-      # Returns true if given item is a member of the given set
-      def ismember?(setkey,item)
-        @@db.sismember(setkey,item)
-      end
-      
       # Lock a given key and release when provided block is finished
       def lock(dockey,timeout=0,raise_on_expire=false)
         m_lock = get_lock(dockey,timeout,raise_on_expire)
@@ -269,12 +328,33 @@ module Rhoconnect
       
       # Create a copy of srckey in dstkey
       def clone(srckey,dstkey)
-        @@db.sdiffstore(dstkey,srckey,'')
+        buckets = _get_bucket_indices(srckey)
+        if buckets.size
+          @@db.pipelined do
+            buckets.each do |bucket_index|
+              _add_bucket_index(dstkey, bucket_index)
+              @@db.sdiffstore("#{dstkey}:#{bucket_index}", "#{srckey}:#{bucket_index}", '')
+            end
+          end
+        else
+          @@db.sdiffstore(dstkey,srckey,'')
+        end
       end
       
       # Rename srckey to dstkey
       def rename(srckey,dstkey)
-        @@db.rename(srckey,dstkey) if @@db.exists(srckey)
+        buckets = _get_bucket_indices(srckey)
+        if buckets.size
+          @@db.pipelined do
+            @@db.del("#{srckey}:indices")
+            buckets.each do |bucket_index|
+              _add_bucket_index(dstkey, bucket_index)
+              @@db.rename("#{srckey}:#{bucket_index}", "#{dstkey}:#{bucket_index}")
+            end
+          end
+        else
+          @@db.rename(srckey,dstkey) if @@db.exists(srckey)
+        end
       end
       
       def put_zdata(dockey,assoc_key,data={},append=false)
@@ -327,6 +407,10 @@ module Rhoconnect
         Store.db.zremrangebyrank(dockey, 0, -1)
       end
       
+      def exists?(key)
+        @@db.exists(key) || @@db.exists("#{key}:indices")
+      end
+      
       alias_method :set_value, :put_value
       alias_method :set_data, :put_data
       
@@ -353,6 +437,100 @@ module Rhoconnect
       def _is_reserved?(attrib,value) #:nodoc:
         RESERVED_ATTRIB_NAMES.include? attrib
       end
+      
+      # operations with docs that are split into buckets
+      def _delete_doc(dockey)
+        # check if this doc has buckets
+        if(@@db.exists("#{dockey}:indices"))
+          buckets_list = _get_buckets(dockey)
+          # delete all buckets
+          @@db.pipelined do
+            @@db.del("#{dockey}:indices")
+            buckets_list.each do |bucket|
+              @@db.del(bucket)
+            end
+          end
+        end
+        
+        # delete main doc 
+        @@db.del(dockey)
+      end
+      
+      # create object's bucket index
+      # using SHA1 hashing
+      def _create_obj_index(key)
+        Digest::SHA1.hexdigest(key)[0..1]
+      end
+      
+      def _add_bucket_index(dockey, bucket_index)
+        bucket_name = "#{dockey}:#{bucket_index}"
+        @@db.hsetnx("#{dockey}:indices", bucket_index, bucket_name)
+        bucket_name
+      end
+      
+      def _remove_bucket_index(dockey, bucket_index)
+        @@db.hdel("#{dockey}:indices", bucket_index)
+      end
+      
+      def _get_bucket_indices(dockey)
+        @@db.hkeys("#{dockey}:indices")
+      end
+      
+      def _get_buckets(dockey)
+        @@db.hvals("#{dockey}:indices")
+      end
+      
+      def _cleanup_buckets(dockey, keys)
+        keys.each do |key|
+          obj_index = _create_obj_index(key)
+          bucket_name = "#{dockey}:#{obj_index}"
+          _remove_bucket_index(dockey, obj_index) unless @@db.exists(bucket_name)
+        end if keys
+      end
+      
+      def _put_object(dockey, key, obj={})
+        return if obj.empty? or not dockey or not key
+        
+        obj_bucket_index = _create_obj_index(key)
+        bucket_name = "#{dockey}:#{obj_bucket_index}"
+        _add_bucket_index(dockey, obj_bucket_index)
+        @@db.pipelined do
+          obj.each do |attrib, value|
+            unless _is_reserved?(attrib,value)
+              @@db.sadd(bucket_name,setelement(key,attrib,value))
+            end
+          end
+        end
+      end
+      
+      def _delete_object(dockey, key, obj)
+        return false if not key or not dockey or not obj
+        
+        obj_bucket_index = _create_obj_index(key)
+        bucket_name = "#{dockey}:#{obj_bucket_index}"
+        @@db.pipelined do
+          obj.each do |attrib,value|
+            @@db.srem(bucket_name, setelement(key,attrib,value))
+          end
+        end
+        true
+      end
+      
+      def _get_object(dockey, key)
+        return nil unless dockey
+        res = nil
+        obj_bucket_index = _create_obj_index(key)
+        bucket_name = "#{dockey}:#{obj_bucket_index}"
+        key_str_size = "#{key}".size
+        members = @@db.smembers(bucket_name)
+        members.each do |element|
+          next unless element[0..key_str_size - 1] == key
+          obj_id,attrib,value = getelement(element)
+          res ||= {}
+          res[attrib] = value
+        end if members
+        res
+      end
     end
   end
-end  
+end 

data/lib/rhoconnect/tasks.rb

@@ -120,7 +120,8 @@ namespace :rhoconnect do
       puts "\nNew password can't be empty."
     elsif new_pass == new_pass_confirm
       puts ""
-      RestClient.post("#{$url}/api/user/update_user", {:app_name => $appname, :api_token => $token,
+      admin_user = 'rhoadmin'
+      RestClient.put("#{$url}/rc/#{Rhoconnect::API_VERSION}/users/#{admin_user}", {:app_name => $appname, :api_token => $token,
         :attributes => {:new_password => new_pass}})
     else
       puts "\nNew password and confirmation must match."
@@ -131,8 +132,7 @@ namespace :rhoconnect do
   task :reset_refresh => :config do
     user = ask "user: "
     source_name = ask "source name: "
-    RestClient.post("#{$url}/api/source/set_refresh_time", {:api_token => $token,
-      :user_name => user, :source_name => source_name})
+    RestClient.put("#{$url}/rc/#{Rhoconnect::API_VERSION}/readstate/users/#{user}/sources/#{source_name}", {:api_token => $token})
   end
   
   begin