rhoconnect 3.0.6 → 3.1.0.beta1

data/lib/rhoconnect/predefined_adapters/bench_adapter.rb

@@ -0,0 +1,61 @@
+require 'json'
+require 'rest_client'
+require 'uri'
+
+module Rhoconnect
+  # register the adapter as pre-defined
+  Rhoconnect.register_predefined_source('RhoInternalBenchmarkAdapter')
+  
+  class RhoInternalBenchmarkAdapter < Rhoconnect::SourceAdapter
+     def initialize(source) 
+       super(source)
+     end
+
+     def login
+       true
+     end
+
+    def query(params=nil)
+      Rhoconnect::Store.lock(lock_name,1) do
+        @result = Rhoconnect::Store.get_data(db_name)
+      end
+      @result
+    end
+
+    def create(create_hash)
+      id = create_hash['mock_id']
+      Rhoconnect::Store.lock(lock_name,1) do
+        Rhoconnect::Store.put_data(db_name,{id=>create_hash},true) if id
+      end
+      id
+    end
+
+    def update(update_hash)
+      id = update_hash.delete('id')
+      return unless id
+      Rhoconnect::Store.lock(lock_name,1) do
+        data = Rhoconnect::Store.get_data(db_name)
+        return unless data and data[id]
+        update_hash.each do |attrib,value|
+          data[id][attrib] = value
+        end
+        Rhoconnect::Store.put_data(db_name,data)
+      end
+    end
+
+    def delete(delete_hash)
+      id = delete_hash.delete('id')
+      Rhoconnect::Store.lock(lock_name,1) do
+        Rhoconnect::Store.delete_data(db_name,{id=>delete_hash}) if id
+      end
+    end
+
+    def db_name
+      "test_db_storage:#{@source.app_id}:#{@source.user_id}"
+    end
+
+    def lock_name
+      "lock:#{db_name}"
+    end
+  end
+end

data/lib/rhoconnect/source.rb

@@ -127,6 +127,11 @@ module Rhoconnect
           
     def self.load(obj_id,params)
       validate_attributes(params)
+      
+      # if source is pre-defined 
+      # create it dynamically here
+      Rhoconnect.create_predefined_source(obj_id,params)
+      
       model_hash = @@model_data[obj_id.to_sym]
       obj = new(model_hash) if model_hash
       if obj

data/lib/rhoconnect/source_adapter.rb

@@ -27,9 +27,13 @@ module Rhoconnect
             source.name = source.name.dup if source.name.frozen?
             source.name.strip!
           end
+          # certain adapters are pre-defined and reserved
+          # load them and instantiate
+          if Rhoconnect.predefined_sources.has_key?(source.name)
+            adapter=(Object.const_get(source.name)).new(source)
+          elsif Object.const_defined?(source.name) && Object.const_get(source.name).to_s.split("::").first != 'Rhoconnect'
           # fix until source adpaters are phased out, checking for Rhoconnect namespace 
           # so that backend models with same name as Rhoconnect models are instantiated correctly
-          if Object.const_defined?(source.name) && Object.const_get(source.name).to_s.split("::").first != 'Rhoconnect'
             require under_score(source.name)
             adapter=(Object.const_get(source.name)).new(source)
           else
@@ -95,6 +99,11 @@ module Rhoconnect
       Rhoconnect.expire_bulk_data(current_user.login,partition)
     end
   
+    # do pre-processing before CUD operation
+    def validate(operation,operations_hashes,client_ids)
+      {}
+    end
+  
     def create(create_hash); end
 
     def update(update_hash); end

data/lib/rhoconnect/source_sync.rb

@@ -10,16 +10,16 @@ module Rhoconnect
     end
     
     # CUD Operations
-    def create(client_id)
-      _measure_and_process_cud('create',client_id)
+    def create
+      _measure_and_process_cud('create')
     end
     
-    def update(client_id)
-      _measure_and_process_cud('update',client_id)
+    def update
+      _measure_and_process_cud('update')
     end
     
-    def delete(client_id)
-      _measure_and_process_cud('delete',client_id)
+    def delete
+      _measure_and_process_cud('delete')
     end
     
     # Pass through CUD to adapter, no data stored
@@ -64,25 +64,25 @@ module Rhoconnect
       res
     end
 
-    def process_cud(client_id)
+    def process_cud
       if @source.cud_queue or @source.queue
-        async(:cud,@source.cud_queue || @source.queue,client_id)
+        async(:cud,@source.cud_queue || @source.queue)
       else
-        do_cud(client_id)
+        do_cud
       end   
     end
     
-    def do_cud(client_id)
+    def do_cud
       return if _auth_op('login') == false
-      self.create(client_id)
-      self.update(client_id)
-      self.delete(client_id)
+      self.create
+      self.update
+      self.delete
       _auth_op('logoff')
     end
     
     def process_query(params=nil)
       if @source.query_queue or @source.queue
-        async(:query,@source.query_queue || @source.queue,nil,params)
+        async(:query,@source.query_queue || @source.queue,params)
       else
         do_query(params)
       end   
@@ -105,10 +105,10 @@ module Rhoconnect
     end
     
     # Enqueue a job for the source based on job type
-    def async(job_type,queue_name,client_id=nil,params=nil)
+    def async(job_type,queue_name,params=nil)
       SourceJob.queue = queue_name
       Resque.enqueue(SourceJob,job_type,@source.id,
-        @source.app_id,@source.user_id,client_id,params)
+        @source.app_id,@source.user_id,params)
     end
     
     def push_objects(objects,timeout=10,raise_on_expire=false,rebuild_md=true)
@@ -180,108 +180,181 @@ module Rhoconnect
       true
     end
     
-    def _process_create(client,key,value,links,creates,deletes)
-      # Perform operation
+    def _process_create(modified_recs,key,value)
       link = @adapter.create value
       # Store object-id link for the client
       # If we have a link, store object in client document
       # Otherwise, store object for delete on client
-      if link
-        links ||= {}
-        links[key] = { 'l' => link.to_s }
-        creates ||= {}
-        creates[link.to_s] = value
-      else
-        deletes ||= {}
-        deletes[key] = value
+      modified_recs.each do |modified_rec|
+        if link
+          modified_rec[:links] ||= {}
+          modified_rec[:links][modified_rec[:key]] = { 'l' => link.to_s }
+          modified_rec[:creates] ||= {}
+          modified_rec[:creates][link.to_s] = value
+        else
+          modified_rec[:deletes] ||= {}
+          modified_rec[:deletes][modified_rec[:key]] = value
+        end
       end
     end
     
-    def _process_update(client,key,value)
-      begin
-        # Add id to object hash to forward to backend call
-        value['id'] = key
-        # Perform operation
-        @adapter.update value
-      rescue Exception => e
-        # TODO: This will be slow!
-        cd = client.get_data(:cd)
-        client.put_data(:update_rollback,{key => cd[key]},true) if cd[key]
-        raise e
-      end
+    def _process_update(modified_recs,key,value)
+      # Add id to object hash to forward to backend call
+      value['id'] = key
+      # Perform operation
+      @adapter.update value
     end
     
-    def _process_delete(client,key,value,dels)
+    def _process_delete(modified_recs,key,value)
       value['id'] = key
-      # Perform operation
       @adapter.delete value
-      dels ||= {}
-      dels[key] = value
+      # Perform operation
+      modified_recs.each do |modified_rec|
+        modified_rec[:dels] ||= {}
+        modified_rec[:dels][modified_rec[:key]] = value
+      end
     end
     
-    def _measure_and_process_cud(operation,client_id)
+    def _measure_and_process_cud(operation)
       Stats::Record.update("source:#{operation}:#{@source.name}") do
-        _process_cud(operation,client_id)
+        _process_cud(operation)
       end
     end
     
-    def _process_cud(operation,client_id)
+    def _process_cud(operation)
+      # take everything from the queue and erase it
+      # so that no other process will be able to process it again
+      operation_hashes = []
+      client_ids = []
+      @source.lock(operation) do |s|
+        operation_hashes,client_ids = s.get_zdata(operation)
+        s.flush_zdata(operation)
+      end
+      invalid_meta = @adapter.validate(operation,operation_hashes,client_ids)
+      
       errors,links,deletes,creates,dels = {},{},{},{},{}
-      client = Client.load(client_id,{:source_name => @source.name})
-      modified = client.get_data(operation)
-      # Process operation queue, one object at a time
-      modified.each do |key,value|
-        begin
-          # Remove object from queue
-          modified.delete(key)
-          # Call on source adapter to process individual object
-          case operation
-          when 'create'
-            _process_create(client,key,value,links,creates,deletes)
-          when 'update'
-            _process_update(client,key,value)
-          when 'delete'
-            _process_delete(client,key,value,dels)
+      operation_hashes.each_with_index do |client_operation_data,index|
+        client_id = client_ids[index]
+        
+        current_invalid_meta = invalid_meta[index] || {}
+        client_operation_data.each do |key, value|
+          begin
+            continue_loop = true
+            modified_recs = [{:client_id => client_id, :key => key, :value => value }]
+            record_invalid_meta = current_invalid_meta[key] || {}
+            # Remove object from queue
+            client_operation_data.delete(key)
+            
+            # skip the rec - if it is a duplicate of some other record
+            next if record_invalid_meta[:duplicate_of]
+            
+            # prepare duplicate docs
+            duplicates = record_invalid_meta[:duplicates] || {}
+            duplicates.each do |duplicate|
+              modified_recs << duplicate
+            end
+            
+            # raise conflict error if record is marked with one
+            raise SourceAdapterObjectConflictError.new(record_invalid_meta[:error]) if record_invalid_meta[:error]
+            
+            # Call on source adapter to process individual object
+            case operation
+            when 'create'
+              _process_create(modified_recs,key,value)
+            when 'update'
+              _process_update(modified_recs,key,value)
+            when 'delete'
+              _process_delete(modified_recs,key,value)
+            end
+          rescue Exception => e
+            log "SourceAdapter raised #{operation} exception: #{e}"
+            log e.backtrace.join("\n")
+            continue_loop = false
+            modified_recs.each do |modified_rec|
+              modified_rec[:errors] ||= {}
+              modified_rec[:errors][modified_rec[:key]] = modified_rec[:value]
+              modified_rec[:errors]["#{modified_rec[:key]}-error"] = {'message'=>e.message}
+            end
           end
-        rescue Exception => e
-          log "SourceAdapter raised #{operation} exception: #{e}"
-          log e.backtrace.join("\n")
-          errors ||= {}
-          errors[key] = value
-          errors["#{key}-error"] = {'message'=>e.message}
-          break
+          
+          { :errors => errors, 
+            :links => links, 
+            :deletes => deletes, 
+            :creates => creates, 
+            :dels => dels }.each do |doc_name, doc|
+            modified_recs.each do |modified_rec|
+              doc[modified_rec[:client_id]] ||= {}
+              next unless modified_rec[doc_name]
+              doc[modified_rec[:client_id]].merge!(modified_rec[doc_name])
+            end
+          end
+          break unless continue_loop
+        end
+      
+        # Record rest of queue (if something in the middle failed)
+        if not client_operation_data.empty?
+          @source.put_zdata(operation,client_id,client_operation_data,true)
         end
       end
-      # Record operation results
+      
+      # now, process all the modified docs
+      processed_clients = {}
+      client_ids.each do |client_id|
+        processed_clients[client_id] = Client.load(client_id,{:source_name => @source.name}) unless processed_clients[client_id]
+      end
       { "delete_page" => deletes,
         "#{operation}_links" => links,
-        "#{operation}_errors" => errors }.each do |doctype,value|
-        client.put_data(doctype,value,true) unless value.empty?
+        "#{operation}_errors" => errors }.each do |doctype,client_docs|
+          client_docs.each do |client_id,data|
+            client = processed_clients[client_id]
+            client.put_data(doctype,data,true) unless data.empty?
+          end
       end
-      unless operation != 'create' and creates.empty?
-        client.put_data(:cd,creates,true)
-        client.update_count(:cd_size,creates.size)
-        @source.lock(:md) do |s| 
-          s.put_data(:md,creates,true)
-          s.update_count(:md_size,creates.size)
+        
+      if operation == 'create'
+        total_creates = {}
+        creates.each do |client_id,client_doc|
+          next if client_doc.empty?
+          client = processed_clients[client_id]
+          client.put_data(:cd,client_doc,true)
+          client.update_count(:cd_size,client_doc.size)
+          total_creates.merge!(client_doc)
+        end
+        unless total_creates.empty?
+          @source.lock(:md) do |s| 
+            s.put_data(:md,total_creates,true)
+            s.update_count(:md_size,total_creates.size)
+          end
         end
       end
+        
       if operation == 'delete'
         # Clean up deleted objects from master document and corresponding client document
-        client.delete_data(:cd,dels)
-        client.update_count(:cd_size,-dels.size)
-        @source.lock(:md) do |s| 
-          s.delete_data(:md,dels)
-          s.update_count(:md_size,-dels.size)
+        total_dels = {}
+        dels.each do |client_id,client_doc|
+          next if client_doc.empty?
+          client = processed_clients[client_id]
+          client.delete_data(:cd,client_doc)
+          client.update_count(:cd_size,-client_doc.size)
+          total_dels.merge!(client_doc)
+        end
+        unless total_dels.empty?
+          @source.lock(:md) do |s| 
+            s.delete_data(:md,total_dels)
+            s.update_count(:md_size,-total_dels.size)
+          end
         end
       end
-      # Record rest of queue (if something in the middle failed)
-      if modified.empty?
-        client.flash_data(operation)
-      else
-        client.put_data(operation,modified)
-      end
-      modified.size
+      if operation == 'update'
+        errors.each do |client_id, error_doc|
+          next if error_doc.empty?
+          client = processed_clients[client_id]
+          cd = client.get_data(:cd)
+          error_doc.each do |key, value|
+            client.put_data(:update_rollback,{key => cd[key]},true) if cd[key]
+          end
+        end
+      end   
     end
     
     # Metadata Operation; source adapter returns json

data/lib/rhoconnect/store.rb

@@ -277,6 +277,54 @@ module Rhoconnect
         @@db.rename(srckey,dstkey) if @@db.exists(srckey)
       end
       
+      def put_zdata(dockey,assoc_key,data={},append=false)
+        return true unless (dockey and assoc_key and data)
+        flush_zdata(dockey) unless append
+        current_score = 0
+        current_score_data = Store.db.zrevrange(dockey,0,0,:with_scores => true)
+        current_score = current_score_data[-1].to_i if current_score_data
+        current_score += 1
+        Store.db.pipelined do
+          data.each do |key,hash_value|
+            unique_record_key = setelement(current_score,assoc_key, key)
+            Store.db.zadd(dockey, current_score, unique_record_key)
+            Store.put_data(unique_record_key,{key => hash_value})
+          end
+        end
+        true
+      end
+      
+      # Retrieves set for given dockey,associated key (client_id), obj_hashes
+      def get_zdata(dockey)
+        data = Store.db.zrange(dockey, 0, -1)
+        ret = []
+        keys = []
+        scores = []
+        data.each do |zsetkey|
+          obj_hash = Store.get_data zsetkey
+          score,key,objkey = getelement(zsetkey)
+          if scores[-1] != score
+            ret << obj_hash
+            keys << key
+            scores << score
+          else
+            ret[-1].merge!(obj_hash)
+          end
+        end
+        [ret, keys]
+      end
+      
+      # Deletes all keys and their hashes from the Redis DB
+      def flush_zdata(dockey)
+        data = Store.db.zrange(dockey, 0, -1)
+        Store.db.pipelined do
+          data.each do |hash_key|
+            Store.db.del(hash_key)
+          end
+        end
+        Store.db.zremrangebyrank(dockey, 0, -1)
+      end
+      
       alias_method :set_value, :put_value
       alias_method :set_data, :put_data
       

data/lib/rhoconnect/test_methods.rb

@@ -132,8 +132,8 @@ module Rhoconnect
       if @s.is_pass_through?
         @ss.pass_through_cud({'create'=> {'temp-id' => record}},nil)
       else
-        @c.put_data(:create,{'temp-id' => record})
-        @ss.process_cud(@c.id)
+        @s.put_zdata(:create,@c.id,{'temp-id' => record},true)
+        @ss.process_cud
         links = @c.get_data(:create_links)['temp-id']
         links ? links['l'] : nil
       end
@@ -161,8 +161,8 @@ module Rhoconnect
       if @s.is_pass_through?
         @ss.pass_through_cud({'update'=> record },nil)
       else
-        @c.put_data(:update,record)
-        @ss.process_cud(@c.id)
+        @s.put_zdata(:update,@c.id,record,true)
+        @ss.process_cud
       end
     end
     
@@ -194,8 +194,8 @@ module Rhoconnect
       if @s.is_pass_through?
         @ss.pass_through_cud({'delete'=> record },nil)
       else
-        @c.put_data(:delete,record)
-        @ss.process_cud(@c.id)
+        @s.put_zdata(:delete,@c.id,record,true)
+        @ss.process_cud
       end
     end
     

data/lib/rhoconnect/version.rb

@@ -1,3 +1,3 @@
 module Rhoconnect
-  VERSION = '3.0.6'
+  VERSION = '3.1.0.beta1'
 end

data/lib/rhoconnect.rb

@@ -46,7 +46,7 @@ module Rhoconnect
     attr_accessor :base_directory, :app_directory, :data_directory, 
       :vendor_directory, :blackberry_bulk_sync, :redis, :environment,
       :log_disabled, :license, :bulk_sync_poll_interval, :stats, :appserver, :api_token,
-      :raise_on_expired_lock, :lock_duration, :cookie_expire
+      :raise_on_expired_lock, :lock_duration, :cookie_expire, :predefined_sources
   end
   
   ### Begin Rhoconnect setup methods  
@@ -70,6 +70,7 @@ module Rhoconnect
     Rhoconnect.lock_duration = get_setting(config,environment,:lock_duration)
     Rhoconnect.environment = environment
     Rhoconnect.cookie_expire = get_setting(config,environment,:cookie_expire)  || 31536000
+    Rhoconnect.predefined_sources = {}
     yield self if block_given?
     Store.create(Rhoconnect.redis)
     Resque.redis = Store.db
@@ -107,6 +108,10 @@ module Rhoconnect
         # load ruby file for source adapter to re-load class
         load under_score(source_name+'.rb')
       end
+      
+      # load all pre-defined adapters files
+      Dir[File.join(File.dirname(__FILE__),'rhoconnect','predefined_adapters','*.rb')].each { |adapter| load adapter }
+      
       # Create associations for all sources
       Source.update_associations(app.sources)
     end
@@ -149,7 +154,25 @@ module Rhoconnect
     source_config
   end
 
-  ### End Rhoconnect setup methods  
+  ### End Rhoconnect setup methods
+  def register_predefined_source(source_name)
+    return if Rhoconnect.predefined_sources.has_key?(source_name)
+    
+    Rhoconnect.predefined_sources[source_name] = {:source_loaded => false}
+  end
+  
+  def create_predefined_source(source_name,params)
+    source_data = Rhoconnect.predefined_sources[source_name]
+    return unless source_data
+    if source_data[:source_loaded] != true
+      source_config = Rhoconnect.source_config(source_name)
+      source_config[:name] = source_name
+      Source.create(source_config,params)
+      app = App.load(Rhoconnect::APP_NAME)
+      app.sources << source_name
+      source_data[:source_loaded] = true
+    end
+  end
   
   def check_default_secret!(secret)
     if secret == '<changeme>'                        

data/spec/apps/rhotestapp/sources/sample_adapter.rb

@@ -19,6 +19,35 @@ class SampleAdapter < SourceAdapter
   def sync
     super
   end
+  
+  def validate(operation,operation_hashes,client_ids)
+    invalid_meta = {}
+    used_ids = {}
+    operation_hashes.each_with_index do |operation_hash,index|
+      client_id = client_ids[index]
+      operation_hash.each do |key,value|
+        if value['force_duplicate_error'] == '1'
+          invalid_meta[index] ||= {}
+          invalid_meta[index][key] ||= {}
+          invalid_meta[index][key][:error] = "Error during #{operation}: object confict detected"
+        end
+        if value['duplicate_of_cid']
+          invalid_meta[index] ||= {}
+          invalid_meta[index][key] ||= {}
+          invalid_meta[index][key][:duplicate_of] = true
+          master_client_id = value['duplicate_of_cid']
+          master_key = value['duplicate_of_key']
+          master_index = value['duplicate_of_queue_index'].to_i
+          
+          invalid_meta[master_index] ||= {}
+          invalid_meta[master_index][master_key] ||= {}
+          invalid_meta[master_index][master_key][:duplicates] ||= []
+          invalid_meta[master_index][master_key][:duplicates] <<  {:client_id => client_id, :key => key, :value => value}
+        end
+      end
+    end
+    invalid_meta
+  end
  
   def create(create_hash)
     Store.put_data('test_create_storage',{create_hash['_id']=>create_hash},true)

data/spec/jobs/source_job_spec.rb

@@ -9,7 +9,7 @@ describe "SourceJob" do
     
     it "should perform process_query" do
       set_state('test_db_storage' => @data)  
-      SourceJob.perform('query',@s.id,@s.app_id,@s.user_id,nil,nil)
+      SourceJob.perform('query',@s.id,@s.app_id,@s.user_id,nil)
       verify_result(@s.docname(:md) => @data,
         @s.docname(:md_size) => @data.size.to_s)
     end
@@ -17,13 +17,13 @@ describe "SourceJob" do
     it "should perform process_cud" do
       expected = {'backend_id'=>@product1}
       @product1['link'] = 'abc'
-      set_state(@c.docname(:create) => {'1'=>@product1})
-      SourceJob.perform('cud',@s.id,@s.app_id,@s.user_id,@c.id,nil)
+      set_zstate({@s.docname(:create) => {'1'=>@product1}}, @c.id, true)
+      SourceJob.perform('cud',@s.id,@s.app_id,@s.user_id,nil)
+      verify_zresult(@s.docname(:create) => [])
       verify_result(@s.docname(:md) => expected,
         @s.docname(:md_size) => expected.size.to_s,
         @c.docname(:cd) => expected,
-        @c.docname(:cd_size) => expected.size.to_s,
-        @c.docname(:create) => {})
+        @c.docname(:cd_size) => expected.size.to_s)
     end
   end  
 end

data/spec/source_adapter_spec.rb

@@ -33,6 +33,16 @@ describe "SourceAdapter" do
       @sa.class.name.should == @s.name
     end
     
+    it "should create all existing pre-defined adapters" do
+      Rhoconnect.predefined_sources.keys.sort.should == ['RhoInternalBenchmarkAdapter'].sort
+      Rhoconnect.predefined_sources.each do |adapter_name, filename|
+        pas = Source.load(adapter_name, @s_params)
+        pas.should_not == nil
+        pa = SourceAdapter.create(pas)
+        pa.class.name.should == "Rhoconnect::#{adapter_name}"
+      end
+    end
+    
     it "should create DynamicAdapter" do
       @sa1 = SourceAdapter.create(@s2)
       @sa1.class.name.should == 'Rhoconnect::DynamicAdapter'