rhoconnect 3.4.5 → 4.0.0.beta.10

data/lib/rhoconnect/handler/changes/execute_methods.rb

@@ -0,0 +1,88 @@
+# this is the actual pseudo-middleware handler for Create/Update/Delete
+module Rhoconnect
+  module Handler
+    module Changes
+      module ExecuteMethods
+        # combined CUD filter (aka 'queue_updates')
+        def execute_cud_handler(route_handler)
+          _extract_cud_params
+          _run_cud_handler(route_handler)
+        end
+
+        # individual filters
+        def execute_create_handler(route_handler)
+          _extract_cud_params
+          _run_cud_handler(route_handler, 'create')
+        end
+
+        def execute_update_handler(route_handler)
+          _extract_cud_params
+          # merge specific 'update' params
+          begin
+            if params["update"]
+              cud = params["update"]
+              params.delete("update")
+              params.merge!("update" => {params[:id] => cud})
+            end     
+          rescue JSON::ParserError => jpe
+            log jpe.message + jpe.backtrace.join("\n")
+            throw :halt, [500, "Server error while processing client data"]
+          rescue Exception => e
+            log e.message + e.backtrace.join("\n")
+            throw :halt, [500, "Internal server error"]
+          end
+
+          _run_cud_handler(route_handler, 'update')
+        end
+
+        def execute_delete_handler(route_handler)
+          begin
+            obj = current_client.get_object(:cd, params[:id])
+            params.merge!("delete" => { params[:id] => obj } )     
+          rescue JSON::ParserError => jpe
+            log jpe.message + jpe.backtrace.join("\n")
+            throw :halt, [500, "Server error while processing client data"]
+          rescue Exception => e
+            log e.message + e.backtrace.join("\n")
+            throw :halt, [500, "Internal server error"]
+          end
+
+          _run_cud_handler(route_handler, 'delete')
+        end
+
+        # encapsulate common code
+        def _run_cud_handler(route_handler, operations = ['create', 'update', 'delete'])
+          @handler = nil
+          if operations.is_a?String
+            operations = [operations]
+          end
+          if not current_source.is_pass_through?
+            @handler = Rhoconnect::Handler::Changes::Runner.new(operations, @model, current_client, route_handler, params)
+          else
+            @handler = Rhoconnect::Handler::Changes::PassThroughRunner.new(operations, @model, current_client, route_handler, params)
+          end
+          catch_all do
+            @handler.run
+          end
+          status 200
+        end
+
+        def _extract_cud_params
+          begin
+            if params["cud"]
+              cud = JSON.parse(params["cud"])
+              params.delete("cud")
+              params.merge!(cud)
+            end     
+          rescue JSON::ParserError => jpe
+            log jpe.message + jpe.backtrace.join("\n")
+            throw :halt, [500, "Server error while processing client data"]
+          rescue Exception => e
+            log e.message + e.backtrace.join("\n")
+            throw :halt, [500, "Internal server error"]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/changes/pass_through_runner.rb

@@ -0,0 +1,11 @@
+module Rhoconnect
+  module Handler
+  	module Changes
+  	  class PassThroughRunner < Rhoconnect::Handler::Changes::Runner
+    		def run
+    		  @engine.do_pass_through_cud
+    		end
+  	  end
+    end
+  end
+end

data/lib/rhoconnect/handler/changes/runner.rb

@@ -0,0 +1,53 @@
+module Rhoconnect
+  module Handler
+    module Changes
+      class Runner
+      	attr_accessor :source,:client,:engine,:model,:params,:operations
+
+        def initialize(operations,model,client,route_handler, params = {})
+          raise ArgumentError.new(UNKNOWN_CLIENT) unless client
+          raise ArgumentError.new(UNKNOWN_SOURCE) unless (model and model.source)
+          raise ArgumentError.new('Invalid app for source') unless model.source.app
+
+     	    @source,@client = model.source,client
+          @model = model
+     	    @client.last_sync = Time.now if @client
+     	    @params = params
+          @operations = operations
+     	    @engine = Rhoconnect::Handler::Changes::Engine.new(@operations, @model, route_handler, @params)
+        end
+
+        def run
+          _process_blobs(params)
+          processed = 0
+          operations.each do |op|
+            key,value = op,params[op]
+            processed += _receive_cud(key,value) if value
+          end
+          @engine.do_cud
+        end
+
+        private
+        def _receive_cud(operation,params)
+          return 0 if not operations.include?(operation)
+          @source.push_queue(operation,@client.id,params,true)
+          return 1
+        end
+        
+        def _process_blobs(params)
+          unless params[:blob_fields].nil?
+            [:create,:update].each do |utype|
+              objects = params[utype] || {}
+              objects.each do |id,obj|
+                params[:blob_fields].each do |field|
+                  blob = params["#{field}-#{id}"]
+                  obj[field] = @model.store_blob(obj,field,blob)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/changes.rb

@@ -0,0 +1,31 @@
+require 'rhoconnect/handler/helpers.rb'
+require 'rhoconnect/handler/changes/execute_methods.rb'
+require 'rhoconnect/handler/changes/runner.rb'
+require 'rhoconnect/handler/changes/pass_through_runner.rb'
+require 'rhoconnect/handler/changes/engine.rb'
+
+module Rhoconnect
+  module Handler
+  	module Changes
+  	  def self.registered(app)
+  	  	# CUD
+	    app.post "/", :rc_handler => :cud, :login_required => true, :admin_required => false,
+                      :source_required => true, :client_required => true, 
+	                  :deprecated_route => {:verb => :post, :url => ['/api/application', '/application', '/api/application/queue_updates']} do
+	      operation = params[:operation]
+	      @model.send operation.to_sym, params["#{operation}_object".to_sym]
+	    end
+
+	    app.put "/:id", :rc_handler => :update, :login_required => true, :admin_required => false,
+                        :source_required => true, :client_required => true do
+	      @model.update(params[:update_object])
+	    end
+
+	    app.delete "/:id", :rc_handler => :delete, :login_required => true, :admin_required => false,
+                           :source_required => true, :client_required => true do
+	      @model.delete(params[:delete_object])
+	    end
+  	  end
+  	end
+  end
+end

data/lib/rhoconnect/handler/helpers/auth_method.rb

@@ -0,0 +1,29 @@
+module Rhoconnect
+  module Handler
+    module Helpers
+      module AuthMethod
+        def auth_method(operation,client_id=-1)
+          edockey = nil
+          docobj = nil
+          if client_id == -1 
+            edockey = :errors
+            docobj = @source
+          else
+            edockey = :search_errors
+            docobj = Client.load(client_id,{:source_name => @source.name})
+          end 
+          begin
+            docobj.flush_data(edockey) if operation == 'login'
+            @model.send operation
+          rescue Exception => e
+            log "Model raised #{operation} exception: #{e}"
+            log e.backtrace.join("\n")
+            docobj.put_data(edockey,{"#{operation}-error"=>{'message'=>e.message}},true)
+            return false
+          end
+          true
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/helpers/binding.rb

@@ -0,0 +1,18 @@
+module Rhoconnect
+  module Handler
+    module Helpers
+      module Binding
+    		def bind_handler(method_name, method_proc)
+    		  # do nothing if already bound
+    		  return method_proc if method_proc.is_a?Method
+
+    		  self.class.send :define_method, method_name, method_proc
+    		  method = self.class.instance_method(method_name)
+    		  self.class.send :remove_method, method_name
+    		  # bind it to self
+    		  method.bind(self)
+    		end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/helpers/bulk_data.rb

@@ -0,0 +1,53 @@
+module Rhoconnect
+  module Handler
+    module Helpers
+      module BulkData
+        def do_bulk_data(partition, client, sources = nil)
+          raise ArgumentError.new(UNKNOWN_CLIENT) unless client
+          name = Rhoconnect::BulkData.get_name(partition,client.user_id)
+          data = Rhoconnect::BulkData.load(name)
+
+          partition_sources = client.app.partition_sources(partition,client.user_id)	
+          sources ||= partition_sources
+          return {:result => :nop} if sources.length <= 0
+
+          do_bd_sync = data.nil?
+          do_bd_sync = (data.completed? and 
+              (data.refresh_time <= Time.now.to_i or !data.dbfiles_exist?)) unless do_bd_sync
+
+          if do_bd_sync  
+            data.delete if data
+            data = Rhoconnect::BulkData.create(:name => name,
+              :app_id => client.app_id,
+              :user_id => client.user_id,
+              :partition_sources => partition_sources,
+              :sources => sources,
+              :refresh_time => Time.now.to_i + Rhoconnect.bulk_sync_poll_interval)
+            Rhoconnect::BulkData.enqueue("data_name" => name)
+          end
+          
+          if data and data.completed? and data.dbfiles_exist?
+            client.update_clientdoc(sources)
+            sources.each do |src|
+              s = Source.load(src, {:user_id => client.user_id, :app_id => client.app_id})
+              errors = {}
+              s.lock(:errors) do
+                errors = s.get_data(:errors)
+              end
+              unless errors.empty?
+                # FIXME: :result => :bulk_sync_error, :errors => "#{errors}"
+                log "Bulk sync errors are found in #{src}: #{errors}"
+                # Delete all related bulk files
+                data.delete_files
+                return {:result => :url, :url => ''}
+              end
+            end
+            {:result => :url, :url => data.url}
+          elsif data
+            {:result => :wait}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/helpers/source_job.rb

@@ -0,0 +1,14 @@
+module Rhoconnect
+  module Handler
+  	module Helpers
+  	  module SourceJob
+        # Enqueue a job for the source based on job type
+      	def async(job_type,queue_name,params=nil)
+        	  Rhoconnect::SourceJob.queue = queue_name
+            Resque.enqueue(Rhoconnect::SourceJob,job_type,@source.id,
+                           @source.app_id,@source.user_id,params)
+      	end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/helpers.rb

@@ -0,0 +1,4 @@
+require 'rhoconnect/handler/helpers/auth_method.rb'
+require 'rhoconnect/handler/helpers/binding.rb'
+require 'rhoconnect/handler/helpers/source_job.rb'
+require 'rhoconnect/handler/helpers/bulk_data.rb'

data/lib/rhoconnect/handler/plugin_callbacks/execute_methods.rb

@@ -0,0 +1,99 @@
+module Rhoconnect
+  module Handler
+  	module PluginCallbacks
+  	  module ExecuteMethods
+  	  	def execute_push_objects_handler(route_handler)
+  	  	  source = _load_source
+          @model = Rhoconnect::Model::Base.create(source)
+          @handler = Rhoconnect::Handler::PluginCallbacks::Runner.new(route_handler, @model, params)
+          @handler.run
+          'done'
+  	  	end
+  	  	# same functionality in push_deletes - only differs in controller's route implementation
+  	  	alias_method :execute_push_deletes_handler, :execute_push_objects_handler
+
+  	  	# the following methods are not exposed as handlers - instead they're used directly
+  	  	# TODO: potentially deprecate them - push_* are fast enough already
+  	  	def do_fast_insert
+          source = _load_source
+          timeout = params[:timeout] || 10
+          raise_on_expire = params[:raise_on_expire] || false
+          new_objs = params[:data]
+      	  source.lock(:md,timeout,raise_on_expire) do |s|
+        	diff_count = new_objs.size
+        	source.put_data(:md, new_objs, true)
+        	source.update_count(:md_size,diff_count)
+      	  end
+      	  source.announce_changes
+          'done'
+        end
+
+        def do_fast_update
+          source = _load_source
+          timeout = params[:timeout] || 10
+          raise_on_expire = params[:raise_on_expire] || false
+          remove_hash = params[:delete_data]
+          new_hash = params[:data]
+          
+      	  if ((remove_hash and remove_hash.size > 0) or (new_hash and new_hash.size > 0))
+    		    source.lock(:md,timeout,raise_on_expire) do |s|
+    		      # get the objects from DB, remove prev attr data, add new attr data
+    		      update_keys = Set.new
+    		      update_keys += Set.new(remove_hash.keys) if remove_hash
+    		      update_keys += Set.new(new_hash.keys) if new_hash
+    		      objs_to_update = source.get_objects(:md, update_keys.to_a) || {}
+    		      diff_count = -objs_to_update.size
+    		      # remove old values from DB
+    		      source.delete_data(:md, objs_to_update)
+    		      # update data
+    		      remove_hash.each do |key, obj|
+    		        next unless objs_to_update[key]
+    		        obj.each do |attrib, value|
+    		          objs_to_update[key].delete(attrib)
+    		          objs_to_update.delete(key) if objs_to_update[key].empty?
+    		        end
+    		      end if remove_hash
+    		      new_hash.each do |key, obj|
+    		        objs_to_update[key] ||= {}
+    		        objs_to_update[key].merge!(obj)
+    		      end if new_hash
+    		      # store new data into DB
+    		      source.put_data(:md, objs_to_update, true)
+    		      diff_count += objs_to_update.size
+    		      source.update_count(:md_size,diff_count)
+    		    end
+    		    source.announce_changes
+        	end
+          'done'  
+        end
+
+        def do_fast_delete
+          source = _load_source
+          timeout = params[:timeout] || 10
+          raise_on_expire = params[:raise_on_expire] || false
+
+          delete_objs = params[:data]
+	        source.lock(:md,timeout,raise_on_expire) do |s|
+	          diff_count = -delete_objs.size
+	          source.delete_data(:md, delete_objs)
+	          source.update_count(:md_size,diff_count)
+	        end
+	        source.announce_changes
+          'done'
+        end
+
+        private
+        def _load_source
+          source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+          # if source does not exist create one for dynamic adapter
+          unless source
+            sconfig = Rhoconnect.source_config(params[:source_id])
+            source = Source.create(sconfig.merge!({:name => params[:source_id]}),{:user_id => params[:user_id], :app_id => APP_NAME})
+            current_app.sources << source.name
+          end
+          source  
+        end
+  	  end
+  	end
+  end
+end

data/lib/rhoconnect/handler/plugin_callbacks/runner.rb

@@ -0,0 +1,28 @@
+module Rhoconnect
+  module Handler
+  	module PluginCallbacks
+  	  class Runner
+  	  	attr_accessor :source, :model, :route_handler, :params
+
+  	  	include Rhoconnect::Handler::Helpers::Binding
+
+  	  	def initialize(route_handler, model, params = {})
+          raise ArgumentError.new(UNKNOWN_SOURCE) unless (model and model.source)
+          raise ArgumentError.new('Invalid app for source') unless model.source.app
+          raise ArgumentError.new('Invalid plugin_callbacks handler') unless route_handler
+          # if handler is not bound - bind it to self
+          # normally it should be bound to a Controller's instance
+        
+          @source = model.source
+          @model = model
+          @route_handler = bind_handler(:plugin_callback_handler_method, route_handler)  
+          @params = params
+  	  	end
+
+  	  	def run
+  	  	  @route_handler.call
+  	  	end
+  	  end
+  	end
+  end
+end

data/lib/rhoconnect/handler/plugin_callbacks.rb

@@ -0,0 +1,67 @@
+require 'rhoconnect/handler/plugin_callbacks/execute_methods.rb'
+require 'rhoconnect/handler/plugin_callbacks/runner.rb'
+
+module Rhoconnect
+  module Handler
+  	module PluginCallbacks
+  	  def self.registered(app)
+  	  	# push objects
+	    app.post "/push_objects", \
+	            { :rc_handler => :push_objects,
+	              :admin_required => true, 
+	              :login_required => false, 
+	              :source_required => false,
+	              :client_required => false,
+	              :deprecated_route => {:verb => :post, :url => ['/api/push_objects', '/api/source/push_objects']}
+	            } do
+	      @model.push_objects(params)
+	    end
+	    
+	    # push_deletes
+	    app.post "/push_deletes", \
+	          { :rc_handler => :push_deletes,
+	            :admin_required => true, 
+	            :login_required => false,
+	            :source_required => false, 
+	            :client_required => false,
+	            :deprecated_route => {:verb => :post, :url => ['/api/push_deletes', '/api/source/push_deletes']}
+	          } do
+	      @model.push_deletes(params)
+	    end
+	    
+	    # fast insert
+	    app.post "/fast_insert", \
+	          { :admin_required => true, 
+	            :login_required => false, 
+	            :source_required => false,
+	            :client_required => false,
+	            :deprecated_route => {:verb => :post, :url => ['/api/fast_insert', '/api/source/fast_insert']}
+	          } do
+	      do_fast_insert
+	    end
+	    
+	    # fast update
+	    app.post "/fast_update", \
+	          { :admin_required => true, 
+	            :login_required => false, 
+	            :source_required => false,
+	            :client_required => false,
+	            :deprecated_route => {:verb => :post, :url => ['/api/fast_update', '/api/source/fast_update']}
+	          } do
+	      do_fast_update
+	    end
+	    
+	    # fast object delete
+	    app.post "/fast_delete", \
+	            { :admin_required => true, 
+	              :login_required => false, 
+	              :source_required => false,
+	              :client_required => false,
+	              :deprecated_route => {:verb => :post, :url => ['/api/fast_delete', '/api/source/fast_delete']}
+	            } do
+	      do_fast_delete
+	    end
+  	  end
+  	end
+  end
+end

data/lib/rhoconnect/handler/query/engine.rb

@@ -0,0 +1,93 @@
+module Rhoconnect
+  module Handler
+    module Query
+      class Engine
+        attr_accessor :source, :model, :route_handler, :params
+
+        include Rhoconnect::Handler::Helpers::AuthMethod
+        include Rhoconnect::Handler::Helpers::SourceJob
+        include Rhoconnect::Handler::Helpers::Binding
+
+        def initialize(model, route_handler, params = {})
+          raise ArgumentError.new(UNKNOWN_SOURCE) unless (model and model.source)
+          raise ArgumentError.new('Invalid app for source') unless model.source.app
+          raise ArgumentError.new('Invalid sync handler') unless route_handler
+          # if handler is not bound - bind it to self
+        # normally it should be bound to a Controller's instance
+        
+          @source = model.source
+          @route_handler = bind_handler(:sync_handler_method, route_handler)  
+          @params = params
+          @model = model
+        end
+
+        def do_sync
+          query_res = nil
+          if source.query_queue or source.queue
+            query_res = async(:query, source.query_queue || source.queue, params[:query])
+          else
+            source.if_need_refresh do
+              Rhoconnect::Stats::Record.update("source:query:#{source.name}") do
+                if auth_method('login')
+                  query_res = run_query
+                  auth_method('logoff')
+                end
+                # re-wind refresh time in case of error
+                query_failure = source.exists?(:errors)
+                source.rewind_refresh_time(query_failure)
+              end
+            end
+          end
+          query_res
+        end
+
+        def run_query
+          errordoc = nil
+          docobj = nil
+          result = nil
+          begin
+            errordoc = :errors
+            docobj = @source
+            [:metadata,:schema].each do |method|
+              _get_model_data(method)
+            end
+            @model.before_query
+            @route_handler.call
+            result = @model.after_query
+            # operation,sync succeeded, remove errors
+            docobj.lock(errordoc) do
+              docobj.flush_data(errordoc)
+            end
+          rescue Exception => e
+            # store sync,operation exceptions to be sent to all clients for this source/user
+            log "Model raised query exception: #{e}"
+            log e.backtrace.join("\n")
+            docobj.lock(errordoc) do
+              docobj.put_data(errordoc,{"query-error"=>{'message'=>e.message}},true)
+            end
+          end
+          # pass through expects result hash
+          @source.is_pass_through? ? result : true
+        end
+
+        # Metadata Operation; source model returns json
+        def _get_model_data(method)
+          if @model.respond_to?(method)
+            data = @model.send(method)        
+            if data
+              @source.put_value(method,data)
+              if method == :schema
+                parsed = JSON.parse(data)
+                schema_version = parsed['version']
+                raise "Mandatory version key is not defined in model schema method" if schema_version.nil?
+                @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(schema_version))
+              else
+                @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(data))
+              end          
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/query/execute_methods.rb

@@ -0,0 +1,21 @@
+module Rhoconnect
+  module Handler
+    module Query
+      module ExecuteMethods
+        def execute_query_handler(route_handler)
+          content_type :json
+          @handler = nil
+          if not current_source.is_pass_through?
+            @handler = Rhoconnect::Handler::Query::Runner.new(@model, current_client, route_handler, params)
+          else
+            @handler = Rhoconnect::Handler::Query::PassThroughRunner.new(@model, current_client, route_handler, params)
+          end
+          formatted_res = @handler.run
+          response.headers[Rhoconnect::PAGE_TOKEN_HEADER] = formatted_res[1]['token']
+          response.headers[Rhoconnect::PAGE_OBJECT_COUNT_HEADER] = formatted_res[2]['count'].to_s
+          response.body = formatted_res.to_json     
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/query/pass_through_runner.rb

@@ -0,0 +1,35 @@
+module Rhoconnect
+  module Handler
+    module Query
+      class PassThroughRunner < Rhoconnect::Handler::Query::Runner
+        def run
+          token = params[:token]
+          ack_token(token) if token
+          res = []
+          query_result = engine.do_sync
+          res = send_new_page(query_result)
+          format_result(res[0],res[1],res[2],res[3])
+        end
+        
+        def send_new_page(data)
+          data ||= {}
+          data.each_key do |object_id| 
+            data[object_id].each { |attrib,value| data[object_id][attrib] = '' if value.nil? }
+          end
+          token = ''
+          compute_errors_page
+          res = build_page do |r|
+            r['insert'] = data
+            r['metadata'] = compute_metadata
+          end
+          if res['insert']
+            token = @client.compute_token(:page_token)
+          else
+            _delete_errors_page 
+          end    
+          [token,0,data.size,res]
+        end
+      end
+    end
+  end
+end