rhoconnect 3.4.5 → 4.0.0.beta.10

data/lib/rhoconnect/handler/query/runner.rb

@@ -0,0 +1,270 @@
+module Rhoconnect
+  # implementation classes
+  module Handler
+    module Query
+      class Runner
+      	attr_accessor :source,:client,:p_size,:engine,:params
+
+        def initialize(model,client,route_handler, params = {})
+          raise ArgumentError.new(UNKNOWN_CLIENT) unless client
+          raise ArgumentError.new(UNKNOWN_SOURCE) unless (model and model.source)
+          raise ArgumentError.new('Invalid app for source') unless model.source.app
+
+     	    @source,@client,@p_size = model.source,client,params[:p_size] ? params[:p_size].to_i : 500
+     	    @client.last_sync = Time.now if @client
+     	    @params = params
+     	    @engine = Rhoconnect::Handler::Query::Engine.new(model, route_handler, @params)
+        end
+
+        def run
+          res = []
+          token = params[:token]
+          if not ack_token(token)
+            res = resend_page(token)
+          else
+            query_result = @engine.do_sync
+            res = send_new_page
+          end
+          format_result(res[0],res[1],res[2],res[3])
+        end
+
+        # Resend token for a client, also sends exceptions
+        def resend_page(token=nil)
+          token,progress_count,total_count,res = '',0,0,{}
+          schema_page = @client.get_value(:schema_page)
+          if schema_page
+            res = {'schema-changed' => 'true'}
+          else  
+            res = build_page do |r|
+              r['insert'] = @client.get_data(:page)
+              r['delete'] = @client.get_data(:delete_page)
+              r['links'] = @client.get_data(:create_links_page)
+              r['metadata'] = @client.get_value(:metadata_page)
+              progress_count = 0
+              total_count = @client.get_value(:total_count_page).to_i
+            end
+          end
+          token = @client.get_value(:page_token)
+          [token,progress_count,total_count,res]
+        end
+
+      	def ack_token(token)
+          stored_token = @client.get_value(:page_token)
+          if stored_token 
+            if token and stored_token == token
+              @client.put_value(:page_token,nil)
+              @client.flush_data(:schema_page)
+              @client.flush_data(:metadata_page)
+              @client.flush_data(:create_links_page)
+              @client.flush_data(:page)
+              @client.flush_data(:delete_page)
+              _delete_errors_page
+              return true
+            end
+          else
+            return true    
+          end    
+          false
+        end
+
+        def format_result(token,progress_count,total_count,res)
+          count = 0
+          count += res['insert'].length if res['insert']
+          count += res['delete'].length if res['delete']
+          [ {'version'=>Rhoconnect::SYNC_VERSION},
+            {'token'=>(token ? token : '')},
+            {'count'=>count},
+            {'progress_count'=>progress_count},
+            {'total_count'=>total_count},
+            res ]
+        end
+
+        def build_page
+          res = {}
+          yield res
+          res.reject! {|key,value| value.nil? or value.empty?}
+          res.merge!(_send_errors)
+          res
+        end
+
+        def send_new_page
+          token,progress_count,total_count,res = '',0,0,{}
+          if schema_changed?
+            _expire_bulk_data
+            token = @client.compute_token(:page_token)
+            res = {'schema-changed' => 'true'}
+          else  
+            compute_errors_page
+            res = build_page do |r|
+              total_count,r['insert'],r['delete'] = compute_page
+              r['links'] = compute_links_page
+              r['metadata'] = compute_metadata
+            end
+            if res['insert'] or res['delete'] or res['links']
+              token = @client.compute_token(:page_token)
+            else
+              _delete_errors_page 
+            end
+          end
+          # TODO: progress count can not be computed properly
+          # without comparing what has actually changes
+          # so we need to obsolete it in the future versions
+          progress_count = 0
+          [token,progress_count,total_count,res]
+        end
+
+        def send_new_page_bruteforce
+          token,progress_count,total_count,res = '',0,0,{}
+          if schema_changed?
+            _expire_bulk_data
+            token = @client.compute_token(:page_token)
+            res = {'schema-changed' => 'true'}
+          else  
+            compute_errors_page
+            res = build_page do |r|
+              total_count,r['insert'],r['delete'] = compute_page_bruteforce
+              r['links'] = compute_links_page
+              r['metadata'] = compute_metadata
+            end
+            if res['insert'] or res['delete'] or res['links']
+              token = @client.compute_token(:page_token)
+            else
+              _delete_errors_page 
+            end
+          end
+          # TODO: progress count can not be computed properly
+          # without comparing what has actually changes
+          # so we need to obsolete it in the future versions
+          progress_count = 0
+          [token,progress_count,total_count,res]
+        end
+
+        # Checks if schema changed
+        def schema_changed?
+          if engine.model.respond_to?(:schema)
+            schema_sha1 = @source.get_value(:schema_sha1)
+            if @client.get_value(:schema_sha1).nil?
+              @client.put_value(:schema_sha1,schema_sha1)
+              return false
+            elsif @client.get_value(:schema_sha1) == schema_sha1
+              return false
+            end
+            @client.put_value(:schema_sha1,schema_sha1)
+            @client.put_value(:schema_page,schema_sha1)
+            return true
+          else
+            return false
+          end
+        end
+
+        # Computes the metadata sha1 and returns metadata if client's sha1 doesn't 
+        # match source's sha1
+        def compute_metadata
+          metadata_sha1,metadata = @source.lock(:metadata) do |s|
+            [s.get_value(:metadata_sha1),s.get_value(:metadata)]
+          end
+          return if @client.get_value(:metadata_sha1) == metadata_sha1
+          @client.put_value(:metadata_sha1,metadata_sha1)
+          @client.put_value(:metadata_page,metadata)
+          metadata
+        end
+        
+        
+        # Computes diffs between master doc and client doc, trims it to page size, 
+        # stores page, and returns page as hash  
+        def compute_page
+          inserts_elements_map,deletes_elements_map,total_count = @source.lock(:md) do |s| 
+            inserts_elements_map = @client.get_diff_data(:cd,s.docname(:md),@p_size)
+            total_count = s.get_value(:md_size).to_i
+            deletes_elements_map = s.get_diff_data(:md,@client.docname(:cd),@p_size)
+            [inserts_elements_map,deletes_elements_map,total_count]
+          end
+          # until sync is not done - set cd_size to 0
+          # once there are no changes, then, set cd_size to md_size
+          cd_size = inserts_elements_map.size > 0 ? 0 : total_count
+          @client.put_value(:cd_size, cd_size)
+          
+          # now, find the exact changes
+          inserts,deletes = Store.get_inserts_deletes(inserts_elements_map,deletes_elements_map)
+          
+          @client.put_data(:page,inserts)
+          @client.put_data(:delete_page,deletes,true)
+          @client.put_value(:total_count_page,total_count)
+          @client.update_elements(:cd,inserts_elements_map,deletes_elements_map)
+          
+          [total_count,inserts,deletes]
+        end
+        
+        # Computes diffs between master doc and client doc, trims it to page size, 
+        # stores page, and returns page as hash  
+        def compute_page_bruteforce
+          inserts_elements_map,deletes_elements_map,total_count = @source.lock(:md) do |s| 
+            inserts_elements_map,deletes_elements_map = @client.get_diff_data_bruteforce(:cd,s.docname(:md),@p_size)
+            total_count = s.get_value(:md_size).to_i
+            [inserts_elements_map,deletes_elements_map,total_count]
+          end
+          # until sync is not done - set cd_size to 0
+          # once there are no changes, then, set cd_size to md_size
+          cd_size = inserts_elements_map.size > 0 ? 0 : total_count
+          @client.put_value(:cd_size, cd_size)
+          
+          # now, find the exact changes
+          inserts,deletes = Store.get_inserts_deletes(inserts_elements_map,deletes_elements_map)
+          
+          @client.put_data(:page,inserts)
+          @client.put_data(:delete_page,deletes,true)
+          @client.put_value(:total_count_page,total_count)
+          @client.update_elements(:cd,inserts_elements_map,deletes_elements_map)
+          
+          [total_count,inserts,deletes]
+        end
+
+        # Computes errors for client and stores a copy as errors page
+        def compute_errors_page
+          ['create','update','delete'].each do |operation|
+            @client.lock("#{operation}_errors") do |c| 
+              c.rename("#{operation}_errors","#{operation}_errors_page")
+            end
+          end
+          @client.lock("update_rollback") do |c|
+            c.rename("update_rollback","update_rollback_page")
+          end
+        end
+        
+        # Computes create links for a client and stores a copy as links page
+        def compute_links_page
+          @client.lock(:create_links) do |c| 
+            c.rename(:create_links,:create_links_page)
+            c.get_data(:create_links_page)
+          end
+        end
+
+        private
+        def _delete_errors_page
+          ['create','update','delete'].each do |operation|
+            @client.flush_data("#{operation}_errors_page")
+          end
+          @client.flush_data("update_rollback_page")
+        end
+
+        def _send_errors
+          res = {}
+          ['create','update','delete'].each do |operation|
+            res["#{operation}-error"] = @client.get_data("#{operation}_errors_page")
+          end
+          res["source-error"] = @source.lock(:errors) { |s| s.get_data(:errors) }
+          res["update-rollback"] = @client.get_data(:update_rollback_page)
+          res.reject! {|key,value| value.nil? or value.empty?}
+          res
+        end
+
+        # expires the bulk data for the client
+        def _expire_bulk_data
+          [:user,:app].each do |partition|
+            Rhoconnect.expire_bulk_data(@client.user_id,partition)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/query.rb

@@ -0,0 +1,19 @@
+require 'rhoconnect/handler/helpers.rb'
+require 'rhoconnect/handler/query/execute_methods.rb'
+require 'rhoconnect/handler/query/runner.rb'
+require 'rhoconnect/handler/query/pass_through_runner.rb'
+require 'rhoconnect/handler/query/engine.rb'
+
+module Rhoconnect
+  module Handler
+  	module Query
+  	  def self.registered(app)
+	  	  app.get "/", :rc_handler => :query, :login_required => true, :admin_required => false,
+                     :source_required => true, :client_required => true, 
+                     :deprecated_route => {:verb => :get, :url => ['/api/application', '/application', '/api/application/query']} do
+      	  @model.query(params[:query])
+    	 end
+	    end
+  	end
+  end
+end

data/lib/rhoconnect/handler/search/engine.rb

@@ -0,0 +1,60 @@
+module Rhoconnect
+  module Handler
+    module Search
+      class Engine
+        attr_accessor :source, :client, :model, :route_handler, :params
+
+        include Rhoconnect::Handler::Helpers::AuthMethod
+        include Rhoconnect::Handler::Helpers::Binding
+
+        def initialize(model, client, route_handler, params = {})
+          raise ArgumentError.new(UNKNOWN_CLIENT) unless client
+          raise ArgumentError.new(UNKNOWN_SOURCE) unless (model and model.source)
+          raise ArgumentError.new('Invalid app for source') unless model.source.app
+          raise ArgumentError.new('Invalid sync handler') unless route_handler
+          
+          @client = client
+          @source = model.source
+          @model = model
+          # if handler is not bound - bind it to self
+          # normally it should be bound to a Controller's instance
+          @route_handler = bind_handler(:search_handler_method, route_handler)  
+          @params = params
+        end
+
+        def do_search
+          return if auth_method('login',client.id) == false
+          res = run_search
+          auth_method('logoff',client.id)
+          res
+        end
+
+        def run_search
+          errordoc = nil
+          docobj = nil
+          result = nil
+          begin
+            errordoc = :search_errors
+            docobj = client
+            client.compute_token(:search_token)
+            result = @route_handler.call
+            client.put_data(:search,result) unless @source.is_pass_through?
+            # operation,sync succeeded, remove errors
+            docobj.lock(errordoc) do
+              docobj.flush_data(errordoc)
+            end
+          rescue Exception => e
+            # store sync,operation exceptions to be sent to all clients for this source/user
+            log "Model raised search exception: #{e}"
+            log e.backtrace.join("\n")
+            docobj.lock(errordoc) do
+              docobj.put_data(errordoc,{"search-error"=>{'message'=>e.message}},true)
+            end
+          end
+          # pass through expects result hash
+          @source.is_pass_through? ? result : true
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/search/execute_methods.rb

@@ -0,0 +1,32 @@
+module Rhoconnect
+  module Handler
+    module Search
+      module ExecuteMethods
+        def execute_search_handler(route_handler)
+          content_type :json
+          return [].to_json unless params[:sources]
+          res = []
+          params[:sources].each do |source_params|
+            s = Source.load(source_params['name'],{:app_id => current_client.app_id,
+              :user_id => current_client.user_id})
+            current_client.source_name = source_params['name']
+            @model = Rhoconnect::Model::Base.create(s)
+
+            params[:token] = source_params['token'] if source_params['token']
+            if not s.is_pass_through?
+              @handler = Rhoconnect::Handler::Search::Runner.new(@model, current_client, route_handler, params)
+            else
+              @handler = Rhoconnect::Handler::Search::PassThroughRunner.new(@model, current_client, route_handler, params)
+            end
+            @model = @handler.engine.model
+            search_res = @handler.run
+            res << search_res if search_res
+          end
+          response.headers[Rhoconnect::PAGE_TOKEN_HEADER] = res[0][1]['token'] if res[0][1] and res[0][1]['token']
+          response.headers[Rhoconnect::PAGE_OBJECT_COUNT_HEADER] = res[0][3]['count'].to_s if res[0][3] and res[0][3]['count']
+          res.to_json     
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/search/pass_through_runner.rb

@@ -0,0 +1,18 @@
+module Rhoconnect
+  module Handler
+    module Search
+      # this class just overrides one method
+      class PassThroughRunner < Rhoconnect::Handler::Search::Runner
+        private
+        def _do_search
+          # call model search unless client is sending token for ack
+          res = @engine.do_search if params.nil? or !params[:token]
+          res,diffsize =  [res,res.size]
+          formatted_res = _format_search_result(res,diffsize)      
+          _delete_search if diffsize == 0
+          formatted_res
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/search/runner.rb

@@ -0,0 +1,104 @@
+module Rhoconnect
+  # implementation classes
+  module Handler
+    module Search
+      class Runner
+        attr_accessor :source,:client,:p_size,:model,:engine,:params
+
+        def initialize(model,client,route_handler, params = {})
+          raise ArgumentError.new(UNKNOWN_CLIENT) unless client
+          raise ArgumentError.new(UNKNOWN_SOURCE) unless (model and model.source)
+          raise ArgumentError.new('Invalid app for source') unless model.source.app
+
+          @source,@client,@p_size = model.source,client,params[:p_size] ? params[:p_size].to_i : 500
+          @model = model
+          @client.last_sync = Time.now if @client
+          @params = params
+          @engine = Rhoconnect::Handler::Search::Engine.new(@model, @client, route_handler, @params)
+        end
+
+        def run
+          if params
+            return _resend_search_result if params[:token] and params[:resend]
+            if params[:token] and !_ack_search(params[:token]) 
+              formatted_result = _format_search_result
+              _delete_search
+              return formatted_result
+            end
+          end
+          _do_search
+        end
+
+        private
+        def _do_search
+          # call model search unless client is sending token for ack
+          res = @engine.do_search if params.nil? or !params[:token]
+          res,diffsize =  _compute_search 
+          formatted_res = _format_search_result(res,diffsize)      
+          _delete_search if diffsize == 0
+          formatted_res
+        end
+        
+        # Computes search result, updates md for source and cd for client with the result
+        def _compute_search
+          cd_inserts_elements_map = @client.get_diff_data(:cd,@client.docname(:search),@p_size)
+          
+          @client.update_elements(:cd, cd_inserts_elements_map, {})
+          @client.update_count(:cd_size,cd_inserts_elements_map.size)
+          # remove previous search page and build new one
+          @client.flush_data(:search_page)
+          @client.update_elements(:search_page,cd_inserts_elements_map,{})
+          client_res = @client.get_data(:search_page)
+          
+          @source.lock(:md) do |s|
+            md_inserts_elements_map = s.get_diff_data(:md,@client.docname(:cd))
+            s.update_elements(:md, md_inserts_elements_map, {})
+            s.update_count(:md_size,md_inserts_elements_map.size)
+          end
+          
+          [client_res,client_res.size]
+        end
+        
+        def _resend_search_result
+          res = @client.get_data(:search_page)
+           _format_search_result(res,res.size)
+        end
+        
+        def _ack_search(search_token)
+          if @client.get_value(:search_token) != search_token
+            _delete_search
+            @client.put_data(:search_errors,
+              {'search-error'=>{'message'=>'Search error - invalid token'}}
+            )
+            return false
+          end
+          true
+        end
+        
+        def _delete_search
+          [:search, :search_page, :search_token, :search_errors].each do |search_doc|
+         	  @client.flush_data(search_doc)
+         	end
+        end
+        
+        def _format_search_result(res={},diffsize=nil)
+          error = @client.get_data(:search_errors)
+          if not error.empty?
+            [ {'version'=>Rhoconnect::SYNC_VERSION},
+              {'source'=>@source.name},
+              {'search-error'=>error} ]
+          else  
+            search_token = @client.get_value(:search_token)
+            search_token ||= ''
+            return [] if res.empty?
+            [ {'version'=>Rhoconnect::SYNC_VERSION},
+              {'token' => search_token},
+              {'source'=>@source.name},
+              {'count'=>res.size},
+              {'insert'=>res} ]
+           end
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/handler/search.rb

@@ -0,0 +1,26 @@
+require 'rhoconnect/handler/helpers'
+require 'rhoconnect/handler/search/execute_methods'
+require 'rhoconnect/handler/search/runner'
+require 'rhoconnect/handler/search/pass_through_runner'
+require 'rhoconnect/handler/search/engine'
+
+module Rhoconnect
+  module Handler
+  	module Search
+  	  def self.registered(app)
+	  	# search request
+	    app.post "/search", \
+	          { :login_required => true,
+	          	:client_required => true,
+	          	:source_required => false,
+	          	:admin_required => false,
+	            :rc_handler => :search,
+	            :deprecated_route => {:verb => :get, :url => ['/application/search', '/api/application/search']}
+	          } do
+	      search_params = params[:search]
+	      @model.search(search_params)
+	    end
+	  end
+  	end
+  end
+end

data/lib/rhoconnect/handler/sync.rb

@@ -0,0 +1,29 @@
+require 'rhoconnect/handler/query'
+require 'rhoconnect/handler/changes'
+require 'rhoconnect/handler/plugin_callbacks'
+
+module Rhoconnect
+  module Handler
+  	module Sync
+  	  def self.registered(app)
+  	  	app.set_default :admin_required, false
+    	  app.set_default :login_required, true
+    	  app.set_default :source_required, true
+    	  app.set_default :client_required, true
+
+  	  	app.register Rhoconnect::Handler::Query
+  	  	app.register Rhoconnect::Handler::Changes
+  	  	app.register Rhoconnect::Handler::PluginCallbacks
+
+        # source name is available inherently from controller
+        app.before do
+          params[:source_name] = app._rest_name
+          params[:source_id] = app._rest_name
+        end
+  	  end
+  	end
+  end
+end
+
+
+

data/lib/rhoconnect/jobs/source_job.rb

@@ -3,13 +3,22 @@ module Rhoconnect
     class << self
       attr_accessor :queue
     end
-    
+
     def self.perform(job_type,source_id,app_id,user_id,params)
       source = Source.load(source_id,{:app_id => app_id,:user_id => user_id})
-      source_sync = SourceSync.new(source)
+      params ||= {}
+
       case job_type.to_sym
-      when :query then source_sync.do_query(params)
-      when :cud then source_sync.do_cud
+      when :query 
+        handler_sync = lambda { @model.query(params[:query])}
+        @model = Rhoconnect::Model::Base.create(source)
+        source_sync = Rhoconnect::Handler::Query::Engine.new(@model, handler_sync, params)
+        source_sync.do_sync
+      when :cud
+        handler_cud = lambda { @model.send params[:operation].to_sym, params["#{params[:operation]}_object".to_sym] }
+        @model = Rhoconnect::Model::Base.create(source)
+        source_cud = Rhoconnect::Handler::Changes::Engine.new(['create', 'update', 'delete'], @model, handler_cud, params)
+        source_cud.do_cud
       end    
     end
   end

data/lib/rhoconnect/js_adapter.rb

@@ -0,0 +1,79 @@
+# module Rhoconnect
+#   class JsAdapter < SourceAdapter
+#     attr_accessor :name
+#     
+#     def initialize(source)
+#       @source = source
+#       @name = source.name
+#     end
+#     
+#     def self.method_missing method_name, *args
+#       #grab function and args and publish to channel
+#       begin
+#         json = create_json_hash(method_name,args)
+#         NodeChannel.publish_channel_and_wait(json)
+#       rescue Exception => e
+#         puts "#{method_name error} - #{e.message}"
+#       end
+#     end
+#       
+#     def login(params=nil)
+#       begin
+#         json = create_json_hash('get',params)
+#         NodeChannel.publish_channel_and_wait(json)
+#       rescue Exception=>e
+#         puts "error login #{e.message}\n #{e.backtrace}"
+#       end
+#     end
+#   
+#     def query(params=nil)
+#       begin
+#         json = create_json_hash('get',params)
+#         @result = NodeChannel.publish_channel_and_wait(json)
+#         puts "res is ******* #{@result}"
+#         @result
+#       rescue Exception=>e
+#         puts "error query #{e.message}\n#{e.backtrace}"
+#       end
+#     end
+#     
+#     # def search(params=nil)
+#     #    json = create_json_hash('search',params)
+#     #    NodeChannel.publish_channel_and_wait(json)
+#     #  end
+#     
+#     # def sync
+#     #      @result
+#     #      super
+#     #    end
+#     
+#     def create(create_hash)
+#       json = create_json_hash('post',create_hash)
+#       NodeChannel.publish_channel_and_wait(json)  
+#     end
+# 
+#     def update(update_hash)
+#       puts "update hash is #{update_hash}"
+#       json = create_json_hash('put',update_hash)
+#       NodeChannel.publish_channel_and_wait(json)  
+#     end
+# 
+#     def delete(delete_hash)
+#       puts "create hash is #{delete_hash}"
+#       json = create_json_hash('delete',delete_hash)
+#       NodeChannel.publish_channel_and_wait(json)  
+#     end
+#     
+#     
+#     private
+#     
+#     def create_json_hash(operation,params)
+#       json = {
+#         :url     => "#{operation}-/#{@name}",
+#         :args       => params,
+#         :route      => 'request'
+#       }
+#     end
+#   end
+#   
+# end