rhoconnect 3.4.5 → 4.0.0.beta.10

data/lib/rhoconnect/document.rb

@@ -1,87 +1,196 @@
 module Document
-  
+  class << self
+    def included(base)
+      base.extend ClassMethods
+    end
+  end
+
+  module ClassMethods
+    def define_valid_doctypes(doctypes = [])
+      @valid_doctypes ||= Set.new
+      doctypes.each do |doctype|
+        @valid_doctypes.add(doctype.to_sym)
+      end
+      @enforce_valid_doctypes ||= true
+    end
+
+    def valid_doctypes
+      @valid_doctypes ||= Set.new
+      @valid_doctypes
+    end
+
+    def enforce_valid_doctypes
+      @enforce_valid_doctypes ||= false
+    end
+    def enforce_valid_doctypes=(enforce_flag)
+      @enforce_valid_doctypes = enforce_flag
+    end
+  end
+
+  def set_db_doc(doctype, data, append=false)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).set_db_doc(docname(doctype), data, append)
+  end
+
+  def get_db_doc(doctype)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).get_db_doc(docname(doctype))
+  end
+
   # Store wrapper methods for document
   def get_data(doctype,type=Hash)
-    Store.get_data(docname(doctype), type)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).get_data(docname(doctype), type)
   end
-  
+
   def get_object(doctype, key)
-    Store.get_object(docname(doctype), key)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).get_object(docname(doctype), key)
   end
-  
+
   def get_objects(doctype, keys)
-    Store.get_objects(docname(doctype), keys)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).get_objects(docname(doctype), keys)
   end
-  
+
   def get_list(doctype)
-    Store.get_list(docname(doctype))
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).get_list(docname(doctype))
   end
-  
+
   def get_value(doctype)
-    Store.get_value(docname(doctype))
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).get_value(docname(doctype))
   end
-  
+
   def put_object(doctype, key, data={})
-    Store.put_object(docname(doctype), key, data)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).put_object(docname(doctype), key, data)
   end
-  
+
   def put_data(doctype,data,append=false)
-    Store.put_data(docname(doctype),data,append)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).put_data(docname(doctype),data,append)
   end
-  
+
+  def put_tmp_data(doctype, data, append=false)
+    Store.get_store(store_index(doctype)).put_tmp_data(docname(doctype),data,append)  
+  end
+
   def put_list(doctype,data,append=false)
-    Store.put_list(docname(doctype),data,append)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).put_list(docname(doctype),data,append)
   end
-  
+
   def update_objects(doctype,updates)
-    Store.update_objects(docname(doctype),updates)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).update_objects(docname(doctype),updates)
   end
-  
+
   def remove_objects(doctype,deletes)
-    Store.delete_objects(docname(doctype),deletes)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).delete_objects(docname(doctype),deletes)
   end
-  
+
   def put_value(doctype,data)
-    Store.put_value(docname(doctype),data)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).put_value(docname(doctype),data)
   end
-  
+
   def delete_data(doctype,data)
-    Store.delete_data(docname(doctype),data)
-  end
-  
-  def flash_data(doctype)
-    Store.flash_data(docname(doctype))
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).delete_data(docname(doctype),data)
   end
-  
-  def flash_source_data(doctype, from_source)
-    self.source_name=from_source
-    docnamestr = docname('') + doctype
-    Store.flash_data(docnamestr)
+
+  def flush_data(doctype)
+    verify_doctype(doctype)
+    Store.flush_data(docname(doctype))
   end
-  
+
   def rename(srcdoctype,dstdoctype)
-    Store.rename(docname(srcdoctype),docname(dstdoctype))
+    verify_doctype(srcdoctype)
+    verify_doctype(dstdoctype)
+    Store.get_store(store_index(srcdoctype)).rename(docname(srcdoctype),docname(dstdoctype))
   end
-  
-  def get_zdata(doctype)
-    Store.get_zdata(docname(doctype))
+
+  def rename_tmp_data(srcdoctype,dstdoctype)
+    verify_doctype(dstdoctype)
+    Store.get_store(store_index(srcdoctype)).rename_tmp_data(docname(srcdoctype),docname(dstdoctype))
   end
-  
-  def flush_zdata(doctype)
-    Store.flush_zdata(docname(doctype))
+
+  def clone(srcdoctype, dstdocname)
+    verify_doctype(srcdoctype)
+    Store.get_store(store_index(srcdoctype)).clone(docname(srcdoctype), dstdocname)
   end
-  
-  def put_zdata(doctype,assoc_key, data={},append=false)
-    Store.put_zdata(docname(doctype),assoc_key, data,append)
-  end
-  
+
   # Generate the fully-qualified docname
   def docname(doctype)
     "#{self.class.class_prefix(self.class)}:#{self.app_id}:#{self.doc_suffix(doctype)}"
   end
-  
+
+  # default data sharding
+  def store_index(doctype)
+    0
+  end
+
+  def compute_store_index(doctype, source, user_id)
+    index = 0
+    # app-partitioned sources go to 0
+    # everything else if sharded
+    if(source.partition == :user)
+      index_char = Digest::SHA1.hexdigest("#{user_id}:#{source.name}")[0]
+      # designate Store 0 only for system data
+      num_user_stores = Store.num_stores - 1
+      if num_user_stores > 0
+        index = index_char.hex/(16/num_user_stores)
+        if index >= num_user_stores
+          index = num_user_stores - 1
+        end
+        # make up for fact that user store index starts from 1
+        index += 1
+      end
+    end
+    index
+  end
+
+  def exists?(dockey)
+    verify_doctype(dockey)
+    Store.get_store(store_index(dockey)).exists?(docname(dockey))
+  end
+
   # Update count for a given document
   def update_count(doctype,count)
-    Store.update_count(docname(doctype), count)
-  end
-end
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).update_count(docname(doctype), count)
+  end
+
+  def verify_doctype(doctype)
+    # doctype must be in the list (or list must be empty - which is default for 'all documents are valid')
+    return true if !self.class.enforce_valid_doctypes or self.class.valid_doctypes.member?(doctype.to_sym) or (self.class.valid_doctypes.size == 0)
+    raise Rhoconnect::InvalidDocumentException.new("Invalid document type #{doctype} for #{self.class.name}")
+  end
+
+  # interface for doc diffs
+  def get_diff_data(srcdoctype, dstdocname, p_size = nil)
+    verify_doctype(srcdoctype)
+    Store.get_store(store_index(srcdoctype)).get_diff_data(docname(srcdoctype), dstdocname, p_size)
+  end
+
+  def get_diff_data_bruteforce(srcdoctype, dstdocname, p_size = nil)
+    verify_doctype(srcdoctype)
+    Store.get_store(store_index(srcdoctype)).get_diff_data_bruteforce(docname(srcdoctype), dstdocname, p_size)
+  end
+
+  def update_elements(doctype, inserts_elements_map, deletes_elements_map)
+    verify_doctype(doctype)
+    Store.get_store(store_index(doctype)).update_elements(docname(doctype), inserts_elements_map, deletes_elements_map)
+  end
+
+  # Computes token for a single client request
+  def compute_token(doc_key)
+    verify_doctype(doc_key)
+    token = get_token
+    Store.get_store(store_index(doc_key)).put_value(docname(doc_key),token)
+    token.to_s
+  end
+end

data/lib/rhoconnect/handler/authenticate/execute_methods.rb

@@ -0,0 +1,77 @@
+module Rhoconnect
+  module Handler
+  	module Authenticate
+  	  module ExecuteMethods
+  	  	def execute_authenticate_handler(route_handler)
+  	  	  _logout
+  	  	  _login(route_handler)
+  	  	end
+
+  	  	def execute_admin_authenticate_handler(route_handler)
+  	  	  token = ''
+  	  	  _logout
+  	  	  _login(route_handler)
+          u = User.load(params[:login])
+          token = _do_get_api_token(params, u)
+  	  	end
+
+        def execute_rps_authenticate_handler(route_handler)
+          _check_login do
+            _rps_login(route_handler) ? status(204) : status(401)
+          end
+        end
+
+  	  	# helper methods
+  	  	private
+  	  	def _logout
+  	  	  session[:login] = nil
+  	  	end
+
+  	  	def _login(route_handler)
+  	  	  _check_login do
+  	  	  	@handler = Rhoconnect::Handler::Authenticate::Runner.new(current_app, route_handler, params)
+  	  	  	user = @handler.run
+  	  	  	if user
+              session[:login] = user.login
+              session[:app_name] = APP_NAME
+              status(200)
+	          else
+	            raise LoginException.new("Unable to authenticate '#{params[:login]}'")
+	          end
+  	  	  end
+  	  	end
+
+        def _rps_login(route_handler)
+          res = false
+          if current_app
+            auth = Rack::Auth::Basic::Request.new(request.env)
+            if auth.provided? and auth.basic? and auth.credentials
+              params[:login] = auth.credentials.first
+              params[:password] = auth.credentials.last
+              res = route_handler.call 
+            end
+          end
+          res
+        end
+
+  	  	def _check_login
+          begin
+            yield
+          rescue LoginException => le
+            throw :halt, [401, le.message]
+          rescue Exception => e
+            throw :halt, [500, e.message]
+          end
+        end
+
+	    def _do_get_api_token(params,user)
+	      if user and user.admin == 1 and user.token
+	        user.token.value 
+	      else
+	        raise ApiException.new(422, "Invalid/missing API user")
+	      end
+	    end
+  	  end
+  	end
+  end
+end

data/lib/rhoconnect/handler/authenticate/runner.rb

@@ -0,0 +1,49 @@
+module Rhoconnect
+  module Handler
+  	module Authenticate
+  	  class Runner
+  	  	attr_accessor :app, :route_handler, :params
+
+  	  	include Rhoconnect::Handler::Helpers::Binding
+
+  	  	def initialize(app, route_handler, params = {})
+  	  	  raise ArgumentError.new('Invalid app') unless app
+  	  	  raise ArgumentError.new('Invalid authenticate handler') unless route_handler
+  	  	  @app = app
+  	  	  @route_handler = bind_handler(:authenricate_handler_method, route_handler)
+  	  	  @params = params
+  	  	end
+
+  	  	def run
+          if params[:login].nil? or params[:login].empty?
+            return false
+          end
+          user = _do_authenticate
+  	  	end
+
+  	  	private
+  	  	def _do_authenticate
+  	  	  user = nil
+  	  	  if params[:login] == 'rhoadmin'
+          	user = @route_handler.call
+  	  	  else 
+  	  	  	if Rhoconnect.appserver
+        	    auth_result = Rhoconnect::Model::DynamicAdapterModel.authenticate(params[:login],params[:password])
+      	    else
+        	    auth_result = @route_handler.call
+            end
+      	    if auth_result
+              params[:login] = auth_result if auth_result.is_a? String
+              user = User.load(params[:login]) if User.is_exist?(params[:login])
+              if not user
+                user = User.create(:login => params[:login])
+                app.users << user.id
+              end
+            end
+          end
+          user
+        end
+  	  end
+  	end
+  end
+end

data/lib/rhoconnect/handler/authenticate.rb

@@ -0,0 +1,3 @@
+require 'rhoconnect/handler/helpers'
+require 'rhoconnect/handler/authenticate/runner'
+require 'rhoconnect/handler/authenticate/execute_methods'

data/lib/rhoconnect/handler/bulk_data.rb

@@ -0,0 +1,28 @@
+require 'rhoconnect/handler/helpers'
+
+# Bulk Data extenstion
+module Rhoconnect
+  module Handler
+  	module BulkData
+  	  def self.registered(app)
+  	  	app.helpers Rhoconnect::Handler::Helpers::BulkData
+  	  	# bulk sync request
+      	app.post "/bulk_data", \
+          	{ :login_required => true,
+          	  :client_required => true,
+          	  :source_required => false,
+          	  :admin_required => false,
+              :deprecated_route => {:verb => :get, :url => ['/application/bulk_data', '/api/application/bulk_data']}
+          	} do
+          content_type :json
+          sources_param = params[:sources]
+          if sources_param.is_a?String
+            sources_param = sources_param.split(',')
+          end
+          data = do_bulk_data(params[:partition].to_sym,current_client, sources_param)
+          data.to_json
+        end
+      end
+  	end
+  end
+end

data/lib/rhoconnect/handler/changes/engine.rb

@@ -0,0 +1,271 @@
+module Rhoconnect
+  module Handler
+    module Changes
+      class Engine
+        attr_accessor :source, :model, :route_handler, :params, :operations
+
+        include Rhoconnect::Handler::Helpers::SourceJob
+        include Rhoconnect::Handler::Helpers::Binding
+        include Rhoconnect::Handler::Helpers::AuthMethod
+
+        def initialize(operations, model, route_handler, params = {})
+          raise ArgumentError.new(UNKNOWN_SOURCE) unless (model and model.source)
+          raise ArgumentError.new('Invalid app for source') unless model.source.app
+          raise ArgumentError.new('Invalid CUD handler') unless route_handler
+          
+          @source = model.source
+          # if handler is not bound - bind it to self
+          # normally it should be bound to a Controller's instance
+          @route_handler = bind_handler(:cud_handler_method, route_handler)  
+          @params = params
+          @model = model
+          @operations = operations
+        end
+
+        def do_cud
+          if source.cud_queue or source.queue
+            async(:cud,source.cud_queue || source.queue)
+          else
+            run_cud
+          end   
+        end
+
+        # Pass through CUD to adapter, no data stored
+        def do_pass_through_cud
+          return if auth_method('login') == false
+          res,processed_objects = {},[]
+          begin
+            operations.each do |op|
+              objects = params[op]
+              params[:operation] = op if objects
+              objects.each do |key,value|
+                case op
+                when 'create'
+                  params[:create_object] = value
+                  @route_handler.call
+                when 'update'
+                  value['id'] = key
+                  params[:update_object] = value
+                  @route_handler.call
+                when 'delete'
+                  value['id'] = key
+                  params[:delete_object] = value
+                  @route_handler.call
+                end
+                processed_objects << key
+              end if objects
+            end
+          rescue Exception => e
+            log "Error in pass through method: #{e.message}"
+            res['error'] = {'message' => e.message } 
+          end
+          auth_method('logoff')
+          res['processed'] = processed_objects
+          res.to_json
+        end
+        
+        def run_cud
+          return if auth_method('login') == false
+          operations.each do |op|
+            send op.to_sym
+          end
+          auth_method('logoff')
+        end
+
+        # CUD Operations
+        def create
+          _measure_and_process_cud('create')
+        end
+        
+        def update
+          _measure_and_process_cud('update')
+        end
+        
+        def delete
+          _measure_and_process_cud('delete')
+        end
+
+        private
+        def _measure_and_process_cud(operation)
+          Rhoconnect::Stats::Record.update("source:#{operation}:#{@source.name}") do
+            params[:operation] = operation
+            _process_cud(operation)
+            params.delete(:operation)
+          end
+        end
+
+        def _process_create(modified_recs,key,value)
+          params[:create_object] = value
+          link = route_handler.call
+          # Store object-id link for the client
+          # If we have a link, store object in client document
+          # Otherwise, store object for delete on client
+          modified_recs.each do |modified_rec|
+            if link
+              modified_rec[:links] ||= {}
+              modified_rec[:links][modified_rec[:key]] = { 'l' => link.to_s }
+              modified_rec[:creates] ||= {}
+              modified_rec[:creates][link.to_s] = value
+            else
+              modified_rec[:deletes] ||= {}
+              modified_rec[:deletes][modified_rec[:key]] = value
+            end
+          end
+        end
+        
+        def _process_update(modified_recs,key,value)
+          # Add id to object hash to forward to backend call
+          value['id'] = key
+          params[:update_object] = value
+          # Perform operation
+          route_handler.call
+        end
+        
+        def _process_delete(modified_recs,key,value)
+          value['id'] = key
+          params[:delete_object] = value
+          route_handler.call
+          # Perform operation
+          modified_recs.each do |modified_rec|
+            modified_rec[:dels] ||= {}
+            modified_rec[:dels][modified_rec[:key]] = value
+          end
+        end
+        
+        def _process_cud(operation)
+          # take everything from the queue and erase it
+          # so that no other process will be able to process it again
+          operation_hashes = []
+          client_ids = []
+          operation_hashes,client_ids = @source.process_queue(operation)
+          invalid_meta = @model.validate(operation,operation_hashes,client_ids)
+          
+          errors,links,deletes,creates,dels = {},{},{},{},{}
+          operation_hashes.each_with_index do |client_operation_data,index|
+            client_id = client_ids[index]
+            
+            current_invalid_meta = invalid_meta[index] || {}
+            client_operation_data.each do |key, value|
+              begin
+                continue_loop = true
+                modified_recs = [{:client_id => client_id, :key => key, :value => value }]
+                record_invalid_meta = current_invalid_meta[key] || {}
+                # Remove object from queue
+                client_operation_data.delete(key)
+                
+                # skip the rec - if it is a duplicate of some other record
+                next if record_invalid_meta[:duplicate_of]
+                
+                # prepare duplicate docs
+                duplicates = record_invalid_meta[:duplicates] || {}
+                duplicates.each do |duplicate|
+                  modified_recs << duplicate
+                end
+                
+                # raise conflict error if record is marked with one
+                raise Rhoconnect::Model::ObjectConflictErrorException.new(record_invalid_meta[:error]) if record_invalid_meta[:error]
+                
+                # Call on source adapter to process individual object
+                case operation
+                when 'create'
+                  _process_create(modified_recs,key,value)
+                when 'update'
+                  _process_update(modified_recs,key,value)
+                when 'delete'
+                  _process_delete(modified_recs,key,value)
+                end
+              rescue Exception => e
+                log "Model raised #{operation} exception: #{e}"
+                log e.backtrace.join("\n")
+                continue_loop = false
+                modified_recs.each do |modified_rec|
+                  modified_rec[:errors] ||= {}
+                  modified_rec[:errors][modified_rec[:key]] = modified_rec[:value]
+                  modified_rec[:errors]["#{modified_rec[:key]}-error"] = {'message'=>e.message}
+                end
+              end
+              
+              { :errors => errors, 
+                :links => links, 
+                :deletes => deletes, 
+                :creates => creates, 
+                :dels => dels }.each do |doc_name, doc|
+                modified_recs.each do |modified_rec|
+                  doc[modified_rec[:client_id]] ||= {}
+                  next unless modified_rec[doc_name]
+                  doc[modified_rec[:client_id]].merge!(modified_rec[doc_name])
+                end
+              end
+              break unless continue_loop
+            end
+          
+            # Record rest of queue (if something in the middle failed)
+            if not client_operation_data.empty?
+              @source.push_queue(operation,client_id,client_operation_data,true)
+            end
+          end
+          
+          # now, process all the modified docs
+          processed_clients = {}
+          client_ids.each do |client_id|
+            processed_clients[client_id] = Client.load(client_id,{:source_name => @source.name}) unless processed_clients[client_id]
+          end
+          { "delete_page" => deletes,
+            "#{operation}_links" => links,
+            "#{operation}_errors" => errors }.each do |doctype,client_docs|
+              client_docs.each do |client_id,data|
+                client = processed_clients[client_id]
+                client.put_data(doctype,data,true) unless data.empty?
+              end
+          end
+            
+          if operation == 'create'
+            total_creates = {}
+            creates.each do |client_id,client_doc|
+              next if client_doc.empty?
+              client = processed_clients[client_id]
+              client.put_data(:cd,client_doc,true)
+              client.update_count(:cd_size,client_doc.size)
+              total_creates.merge!(client_doc)
+            end
+            unless total_creates.empty?
+              @source.lock(:md) do |s| 
+                s.put_data(:md,total_creates,true)
+                s.update_count(:md_size,total_creates.size)
+              end
+            end
+          end
+            
+          if operation == 'delete'
+            # Clean up deleted objects from master document and corresponding client document
+            total_dels = {}
+            objs = {}
+            dels.each do |client_id,client_doc|
+              next if client_doc.empty?
+              client = processed_clients[client_id]         
+              client.delete_data(:cd,client_doc)
+              client.update_count(:cd_size,-client_doc.size)
+              total_dels.merge!(client_doc)
+            end
+            unless total_dels.empty?
+              @source.lock(:md) do |s| 
+                s.delete_data(:md,total_dels)
+                s.update_count(:md_size,-total_dels.size)
+              end
+            end
+          end
+          if operation == 'update'
+            errors.each do |client_id, error_doc|
+              next if error_doc.empty?
+              client = processed_clients[client_id]
+              cd = client.get_data(:cd)
+              error_doc.each do |key, value|
+                client.put_data(:update_rollback,{key => cd[key]},true) if cd[key]
+              end
+            end
+          end   
+        end
+      end
+    end
+  end
+end