rhoconnect 3.4.5 → 4.0.0.beta.10

data/lib/rhoconnect/server.rb

@@ -5,59 +5,79 @@ require 'json'
 require 'fileutils'
 require 'rhoconnect'
 
-# all middlewares
+# all middlewares, conditions, handlers - everything, that makes our product so cool!
+Dir[File.join(File.dirname(__FILE__),'handler', '*.rb')].each { |mw| require mw }
 Dir[File.join(File.dirname(__FILE__),'middleware','**','*.rb')].each { |mw| require mw }
+Dir[File.join(File.dirname(__FILE__),'condition','**','*.rb')].each { |mw| require mw }
 
 module Rhoconnect
-  
+
   class ApiException < Exception
     attr_accessor :error_code
     def initialize(error_code,message)
       super(message)
       @error_code = error_code
-    end  
+    end
   end
-    
+
   class Server < Sinatra::Base
     set :static,        true
     set :stats,         false
     # default secret
     @secret = '<changeme>'
-                                                                                         
+
+    class << self
+      def set_default(setting, value)
+        @default_settings ||= {}
+        @default_settings[setting] = value
+      end
+
+      # these methods will be called by the subclasses
+      # to prevent double inclusion
+      # of the base middleware
+      # def reset_rc_base!
+      #   #@middleware_configured = true
+      # end
+
+      # def self.inherited(subclass)
+      #   subclass.reset_rc_base!
+      # end
+    end
+
     # Setup route and mimetype for bulk data downloads
     # TODO: Figure out why "mime :data, 'application/octet-stream'" doesn't work
-    Rack::Mime::MIME_TYPES['.data'] = 'application/octet-stream'   
-     
+    Rack::Mime::MIME_TYPES['.data'] = 'application/octet-stream'
+
     include Rhoconnect
+    # common conditions
+    extend Rhoconnect::Condition::AdminRequired
+    extend Rhoconnect::Condition::LoginRequired
+    extend Rhoconnect::Condition::SourceRequired
+    extend Rhoconnect::Condition::ClientRequired
+    extend Rhoconnect::Condition::Verbs
+    extend Rhoconnect::Condition::VerifySuccess
+
+    # RC Handlers
+    include Rhoconnect::Handler::Query::ExecuteMethods
+    include Rhoconnect::Handler::Changes::ExecuteMethods
+    include Rhoconnect::Handler::Search::ExecuteMethods
+    include Rhoconnect::Handler::PluginCallbacks::ExecuteMethods
+    include Rhoconnect::Handler::Authenticate::ExecuteMethods
 
     # Set rhoconnect middleware
     set :use_middleware, Proc.new {
       return false if @middleware_configured # Middleware might be configured only once!
 
       use Rhoconnect::Middleware::XDomainSessionWrapper
-      use Rack::Cors do |cfg|
-        cfg.allow do |allow|
-          allow.origins(/.*/)
-          allow.resource '/application',   :headers => :any, :methods => [:get, :post, :put, :delete], :credentials => true
-          allow.resource '/application/*', :headers => :any, :methods => [:get, :post, :put, :delete], :credentials => true
-          allow.resource '/api/application',   :headers => :any, :methods => [:get, :post, :put, :delete], :credentials => true
-          allow.resource '/api/application/*', :headers => :any, :methods => [:get, :post, :put, :delete], :credentials => true
-          allow.resource "/app/#{Rhoconnect::API_VERSION}/*",   :headers => :any, :methods => [:get, :post, :put, :delete], :credentials => true
-          allow.resource "/rc/#{Rhoconnect::API_VERSION}/app/*",   :headers => :any, :methods => [:get, :post, :put, :delete], :credentials => true
-        end
-      end
       use Rhoconnect::Middleware::BodyContentTypeParser
       use Rhoconnect::Middleware::Stats
       Rhoconnect::Server.set :secret, @secret unless settings.respond_to?(:secret)
       use Rack::Session::Cookie,
         :key => 'rhoconnect_session', :expire_after => Rhoconnect.cookie_expire,
         :secret => Rhoconnect::Server.secret
-      use Rhoconnect::Middleware::CurrentRequest
       use Rhoconnect::Middleware::CurrentApp
       use Rhoconnect::Middleware::CurrentUser
-      use Rhoconnect::Middleware::AdminUser
-      use Rhoconnect::Middleware::LoginRequired
-      
+
       @middleware_configured ||= true
     }
 
@@ -66,64 +86,6 @@ module Rhoconnect
         request.env['PATH_INFO'].split('/').last
       end
 
-      def check_login
-        begin
-          yield
-        rescue LoginException => le
-          throw :halt, [401, le.message]
-        rescue Exception => e
-          throw :halt, [500, e.message]
-        end
-      end
-
-      def do_login
-        check_login do
-          if login 
-            status(200) 
-          else
-            raise LoginException.new("Unable to authenticate '#{params[:login]}'")
-          end
-        end
-      end
-
-      def do_ans_login
-        check_login do
-          ans_login ? status(204) : status(401)
-        end
-      end
-
-      def login
-        if params[:login].nil? or params[:login].empty? 
-          return false
-        end
-        if params[:login] == 'rhoadmin'
-          user = User.authenticate(params[:login], params[:password])
-        elsif current_app and current_app.can_authenticate? and params[:login]
-          user = current_app.authenticate(params[:login], params[:password], session)
-        end
-        if user
-          session[:login] = user.login
-          session[:app_name] = APP_NAME
-        else
-          false
-        end
-      end
-
-      def ans_login
-        if current_app and current_app.can_ans_authenticate?
-          auth = Rack::Auth::Basic::Request.new(request.env)
-          current_app.ans_authenticate(
-            auth.credentials.first, auth.credentials.last
-          ) if auth.provided? and auth.basic? and auth.credentials
-        else
-          false
-        end
-      end
-
-      def logout
-        session[:login] = nil
-      end
-
       def current_user
         @env[Rhoconnect::CURRENT_USER]
       end
@@ -132,43 +94,16 @@ module Rhoconnect
         @env[Rhoconnect::CURRENT_APP]
       end
 
-      def current_client
-        if @client.nil?
-          client_id = @env[Rhoconnect::CLIENT_ID_HEADER]
-          client_id = params[:client_id] unless client_id
-          if client_id
-            @client = Client.load(client_id.to_s,
-              params[:source_name] ? {:source_name => current_source.name} : {:source_name => '*'})
-            if @client and current_user and @client.user_id != current_user.login
-              @client.switch_user(current_user.login) 
-            end
-          end
-        end
-        @client  
+      def request_action
+        Rhoconnect.resource_action(request.env)
       end
 
-      def current_source
-        return @source if @source 
-        user = current_user
-        if params[:source_name] and user
-          @source = Source.load(params[:source_name],
-            {:user_id => user.login,:app_id => APP_NAME})
-             
-          # if source does not exist create one for dynamic adapter
-          unless @source
-            sconfig = Rhoconnect.source_config(params[:source_name])
-            @source = Source.create(sconfig.merge!({:name => params[:source_name]}),{:user_id => user.login, :app_id => APP_NAME})
-            current_app.sources << @source.name
-          end
-          @source
-        else
-          log "ERROR: Can't load source, no source_name provided.\n"
-          nil
-        end
+      def current_client
+        @env[Rhoconnect::CURRENT_CLIENT]
       end
 
-      def current_client_sync
-        ClientSync.new(current_source,current_client,params[:p_size])
+      def current_source
+        @env[Rhoconnect::CURRENT_SOURCE]
       end
 
       def catch_all
@@ -181,31 +116,28 @@ module Rhoconnect
           throw :halt, [500, e.message]
         end
       end
-      
-      def mark_deprecated_call_and_reroute(name, namespace, *params)
-        namespace_val = namespace.nil? ? "<namespace>" : "#{namespace}"
-        http_method = request.get? ? "GET" : "POST"
-        warning_message = "Use of the #{http_method} #{request.path} is deprecated. Use #{http_method} /api/#{namespace_val}/#{name} instead."
+
+      def mark_deprecated_call_and_reroute_api4(verb, new_route, old_verb, old_route, route_handler, filter_handler=nil, klass = nil)
+        warning_message = "Use of the #{old_verb.to_s.upcase} #{old_route} is deprecated. Use Rhoconnect API #{Rhoconnect::API_VERSION} instead."
         response.headers['Warning'] = warning_message
         Rhoconnect.log warning_message
-        if namespace != nil  
-          call env.merge('PATH_INFO' => "/api/#{namespace}/#{name}")
+        if klass
+          klass_instance = klass.new
+          klass_instance.helpers.call! env.merge('PATH_INFO' => new_route, 'REQUEST_METHOD' => verb.to_s.upcase)
         else
-          yield *params
+          execute_api_call(route_handler, filter_handler)
         end
       end
-      
-      def mark_deprecated_call_and_reroute_api4(verb, new_route, old_verb, old_route, client_call, &block)
-        warning_message = "Use of the #{old_verb.to_s.upcase} #{old_route} is deprecated. Use Rhoconnect API #{Rhoconnect::API_VERSION} instead."
-        response.headers['Warning'] = warning_message
-        Rhoconnect.log warning_message
-        
-        execute_api_call(client_call, &block)
-      end
-      
-      def execute_api_call(client_call = false)
+
+      def execute_api_call(route_handler, filter_handler = nil)
         catch_all do
-          res = yield params, current_user, self
+          proc = route_handler.bind(self)
+          res = nil
+          if filter_handler
+            res = send filter_handler, proc
+          else
+            res = proc.call
+          end
           if params.has_key? :warning
             Rhoconnect.log params[:warning]
             response.headers['Warning'] = params[:warning]
@@ -214,23 +146,22 @@ module Rhoconnect
         end
       end
     end
-    
-    private 
+
+    private
     def self._use_async_framework
       return false if settings.respond_to?(:use_async_model) and settings.use_async_model == false
       return false if @dispatch_framework_initialized
-      
+
       @dispatch_framework_initialized ||=true
       if RUBY_VERSION =~ /1.9/ and not defined?(JRUBY_VERSION)
-        begin 
-          require 'rhoconnect/async' 
+        begin
+          require 'rhoconnect/async'
           register Rhoconnect::Synchrony
-          helpers Rhoconnect::AsyncHelpers
         rescue LoadError => e
           # if it fails here - Async can not be used
           settings.use_async_model = false
           warning_for_async_gems = <<_INSTALL_ASYNC_GEMS
-          
+
 ***** WARNING *****
 Rhoconnect has detected that Ruby 1.9.x is used and tried to initialize Async Framework, but failed:
 
@@ -253,33 +184,35 @@ _INSTALL_ASYNC_GEMS
         set :use_async_model, false
       end
     end
-    
-    def self.new
-      # by default, enable this feature
-      if not settings.respond_to?(:use_async_model) or settings.use_async_model != false
-        set :use_async_model, true
-      end
-      # this must be called first - because
-      # it redefines some of the middleware
-      Rhoconnect::Server._use_async_framework
-      
-      if settings.respond_to?(:stats) and settings.send(:stats) == true
-        Rhoconnect.stats = true
-      else
-        Rhoconnect::Server.disable :stats
-        Rhoconnect.stats = false
+
+    class << self
+      def new
+        # by default, enable this feature
+        if not settings.respond_to?(:use_async_model) or settings.use_async_model != false
+          set :use_async_model, true
+        end
+        # this must be called first - because
+        # it redefines some of the middleware
+        _use_async_framework
+
+        if settings.respond_to?(:stats) and settings.send(:stats) == true
+          Rhoconnect.stats = true
+        else
+          Rhoconnect::Server.disable :stats
+          Rhoconnect.stats = false
+        end
+        Rhoconnect::Server.settings.use_middleware
+        #puts "Controller #{self.name} has been initialized with #{middleware.inspect}"
+        super
       end
-      settings.use_middleware
-      
-      super
     end
-        
+
     def initialize
       # Whine about default session secret
       check_default_secret!(Rhoconnect::Server.secret)
       super
     end
-    
+
     Rhoconnect.log "Rhoconnect Server v#{Rhoconnect::VERSION} started..."
 
     before do
@@ -287,46 +220,82 @@ _INSTALL_ASYNC_GEMS
       headers({'pragma'=>'no-cache'})
     end
 
-    get '/' do
-      redirect "/console/"
-    end
-    
-    def self.api(name, namespace = nil, verb = :post, &block)
-      # this method is deprecated - so we should warn the users
-      # that it will be removed in 4.0
-      warning_for_server_api = <<_DEPRECATE_SERVER_API
-***** WARNING *****
-      Server.api method is deprecated and will be removed in Rhoconnect 4.0
-      Please change your implementation to Server.api4
-_DEPRECATE_SERVER_API
-      puts warning_for_server_api
-                
-      old_api_prefix = (namespace == :application) ? :application : :api
-      client_call = (namespace == :application) ? true : false      
-      send verb, "/#{old_api_prefix}/#{name}" do 
-        mark_deprecated_call_and_reroute(name, namespace, params, &block)
-      end
-      
-      send verb, "/api/#{namespace}/#{name}" do
-        execute_api_call(client_call, &block)
-      end
-    end
-    
-    def self.api4(method_name, verb, route, admin = true, deprecated_route = nil, &block)
+    def self.api4(resource, route_url, verb = :post, options = {}, &block)
+      deprecated_route = options[:deprecated_route]
+      options.delete(:deprecated_route)
+
+      # TODO: Re-work deprecation handling as soon as client is updated
+      # the only reason we do criss-cross routing is because old-style routes
+      # had one root and new-style routes now reside in separate controllers
+      rc_handler = options[:rc_handler]
+      options.delete(:rc_handler)
+      rc_handler_method = nil
+      rc_handler_method = "execute_#{rc_handler}_handler".to_sym if rc_handler
       unless deprecated_route.nil?
         deprecated_urls = deprecated_route[:url].is_a?(String) ? [deprecated_route[:url]] : deprecated_route[:url]
         deprecated_urls.each do |deprecated_url|
-          send deprecated_route[:verb], deprecated_url do 
-            mark_deprecated_call_and_reroute_api4(verb, route, deprecated_route[:verb], deprecated_url, !admin, &block)
+          d_verb = deprecated_route[:verb].to_sym
+          dep_route_handler = Rhoconnect::DefaultServer.send(:generate_method, :dep_route_handler, &block)
+
+          # build deprecation hash to be used later at run time for re-directing
+          dep_info = {}
+          dep_info[:klass] = self
+          dep_info[:route_handler] = dep_route_handler
+          dep_info[:rc_handler_method] = rc_handler_method
+          dep_info[:verb] = verb
+          dep_info[:route_url] = route_url
+
+          Rhoconnect::DefaultServer.registered_routes(d_verb)[deprecated_url] ||= {}
+          Rhoconnect::DefaultServer.registered_routes(d_verb)[deprecated_url][self.name] = dep_info
+          Rhoconnect::DefaultServer.send d_verb, deprecated_url, options do
+            klass = nil
+            req_verb = env['REQUEST_METHOD'].downcase.to_sym
+            req_path = env['PATH_INFO']
+            # retrieve controller-specific deprecation handler
+            if params[:source_name]
+              controller_name = "#{params[:source_name]}Controller"
+              dep_info = Rhoconnect::DefaultServer.registered_routes(req_verb)[req_path][controller_name]
+              if dep_info
+                klass = dep_info[:klass]
+                verb = dep_info[:verb]
+                route_url = dep_info[:route_url]
+                dep_route_handler = dep_info[:route_handler]
+                rc_handler_method = dep_info[:rc_handler_method]
+              end
+            end
+            mark_deprecated_call_and_reroute_api4(verb, route_url, req_verb, req_path, dep_route_handler, rc_handler_method, klass)
           end
         end
       end
-      send verb, route do
-        execute_api_call(!admin, &block)
+
+      # turn block into UnboundMethod - so that we can bind it later with
+      # particular Controller instance
+      route_handler = send(:generate_method, :route_handler, &block)
+      route verb.to_s.upcase, route_url, options do
+        execute_api_call(route_handler, rc_handler_method)
       end
     end
   end
+
+  # serves OBSOLETED routes and root
+  # TODO - OBSOLETED routes should be removed
+  class DefaultServer < Rhoconnect::Server
+    helpers Rhoconnect::Handler::Helpers::BulkData
+
+    # to prevent registering the same route several times
+    def self.registered_routes(verb)
+      @registered_routes ||= {}
+      @registered_routes[verb] ||= {}
+      @registered_routes[verb]
+    end
+
+    get '/' do
+      redirect "/console/"
+    end
+  end
 end
 
 include Rhoconnect
-Dir[File.join(File.dirname(__FILE__),'api','**','*.rb')].each { |api| load api }
+# load all controllers , starting with base
+require 'rhoconnect/controller/base'
+Dir[File.join(File.dirname(__FILE__),'controller','**','*.rb')].each { |api| require api }

data/lib/rhoconnect/source.rb

@@ -1,5 +1,5 @@
 module Rhoconnect
-  class MemoryModel
+  class MemoryOrm
     @@model_data = {}
     @@string_fields = []
     @@integer_fields = []
@@ -85,9 +85,20 @@ module Rhoconnect
       end
       res
     end
+
+    def to_hash
+      res = {}
+      @@string_fields.each do |field|
+       res[field] = send(field.to_sym)
+      end
+      @@integer_fields.each do |field|
+       res[field] = send(field.to_sym)
+      end
+      res
+    end
   end
 
-  class Source < MemoryModel
+  class Source < MemoryOrm
     attr_accessor :app_id, :user_id
 
     validates_presence_of :name
@@ -99,6 +110,22 @@ module Rhoconnect
     define_fields([:id, :rho__id, :name, :url, :login, :password, :callback_url, :partition_type, :sync_type,
       :queue, :query_queue, :cud_queue, :belongs_to, :has_many, :pass_through, :push_notify], [:source_id, :priority, :retry_limit, :simulate_time])
 
+    define_valid_doctypes([:md,
+                          :md_size,
+                          :md_copy,
+                          :errors,
+                          :metadata,
+                          :metadata_sha1,
+                          :schema,
+                          :schema_sha1,
+                          :create,
+                          :update,
+                          :delete])
+    # we can not enforce valid doctypes at this point
+    # because QUERY creates temporary md docs
+    # TODO: re-work it to eliminate this obstacle
+    #enforce_valid_doctypes=false
+
     def initialize(fields)
       self.name = fields['name'] || fields[:name]
       update_fields(fields)
@@ -121,6 +148,12 @@ module Rhoconnect
       fields[:push_notify] = fields[:push_notify] ? fields[:push_notify] : 'false'
     end
 
+    def store_index(doctype)
+      # app-partitioned sources go to 0
+      # everything else if sharded
+      compute_store_index(doctype, self, self.user_id)
+    end
+
     def self.create(fields,params)
       fields = fields.with_indifferent_access # so we can access hash keys as symbols
       super(fields,params)
@@ -139,6 +172,7 @@ module Rhoconnect
       Rhoconnect.create_predefined_source(obj_id,params)
 
       model_hash = @@model_data[obj_id.to_sym]
+
       obj = new(model_hash) if model_hash
       if obj
         obj = obj.dup
@@ -172,7 +206,11 @@ module Rhoconnect
 
     def self.delete_all
       params = {:app_id => APP_NAME,:user_id => '*'}
-      @@model_data.each { |k,v| Source.load(k,params).flash_store_data }
+      @@model_data.each do |k,v|
+        s = Source.load(k,params)
+        s.flush_store_data
+        Store.flush_data("source:#{s.name}:*")
+      end
       @@model_data = {}
     end
 
@@ -201,10 +239,6 @@ module Rhoconnect
       self.class.set_defaults(fields)
     end
 
-    def clone(src_doctype,dst_doctype)
-      Store.clone(docname(src_doctype),docname(dst_doctype))
-    end
-
     def poll_interval
       value = Store.get_value(poll_interval_key)
       value ? value.to_i : nil
@@ -250,14 +284,69 @@ module Rhoconnect
       "#{user_by_partition}:#{self.name}:#{doctype.to_s}"
     end
 
-    def flash_store_data
+    def flush_store_data
       delete_user_read_state
-      flash_data('*')
-      flash_data(poll_interval_key)
+      # queues should be processed separately first
+      [:create, :update, :delete].each do |doctype|
+        flush_queue(doctype)
+      end
+      # then, all other doctypes
+      self.class.valid_doctypes.each do |doctype|
+        flush_data(doctype)
+      end
+    end
+
+    def queue_docname(dockey)
+      # currently, all queues are bound by user - not shared
+      docname(dockey)
+    end
+
+    # this data is not sharded
+    def lock_queue_doc(doctype)
+      Store.lock(queue_docname(doctype)) do
+        yield self
+      end
+    end
+
+    # this is an atomic operation
+    # - lock queue, get queue data, flush queue, unlock queue
+    def process_queue(doctype)
+      verify_doctype(doctype)
+      ret = []
+      keys = []
+      lock_queue_doc(doctype) do |s|
+        ret, keys = Store.get_store(0).get_zdata(s.queue_docname(doctype))
+        Store.get_store(0).flush_zdata(s.queue_docname(doctype))
+      end
+      [ret, keys]
+    end
+
+    def get_queue(doctype)
+      verify_doctype(doctype)
+      ret = []
+      keys = []
+      lock_queue_doc(doctype) do |s|
+        ret, keys = Store.get_store(0).get_zdata(s.queue_docname(doctype))
+      end
+      [ret, keys]
+    end
+
+    def flush_queue(doctype)
+      verify_doctype(doctype)
+      lock_queue_doc(doctype) do |s|
+        Store.get_store(0).flush_zdata(s.queue_docname(doctype))
+      end
+    end
+
+    def push_queue(doctype,assoc_key, data={},append=false)
+      verify_doctype(doctype)
+      lock_queue_doc(doctype) do |s|
+        Store.get_store(0).put_zdata(s.queue_docname(doctype),assoc_key, data,append)
+      end
     end
 
     def delete
-      flash_store_data
+      flush_store_data
       @@model_data.delete(rho__id.to_sym) if rho__id
     end
 

data/lib/rhoconnect/stats/record.rb

@@ -10,19 +10,19 @@ module Rhoconnect
           start = (Time.now.to_i / resolution(metric)) * resolution(metric)
           current, current_score = 0, start
           Store.lock(key(metric)) do
-            range = Store.db.zrevrange(key(metric), 0, 0)
+            range = Store.zrevrange(key(metric), 0, 0)
             if !range.empty?
               member = range[0]
               m_current = member.split(':')[0]
-              m_current_score = Store.db.zscore(key(metric), member).to_i
+              m_current_score = Store.zscore(key(metric), member).to_i
               if m_current_score > (start - resolution(metric))
-                Store.db.zrem(key(metric), member)
+                Store.zrem(key(metric), member)
                 current, current_score = m_current, m_current_score
               end
             end
             value = block_given? ? yield(current, value) : value 
-            Store.db.zadd(key(metric), current_score, "#{value}:#{start}")
-            Store.db.zremrangebyscore(key(metric), 0, start - record_size(metric))
+            Store.zadd(key(metric), current_score, "#{value}:#{start}")
+            Store.zremrangebyscore(key(metric), 0, start - record_size(metric))
           end
         end
       
@@ -56,15 +56,15 @@ module Rhoconnect
         end
       
         def keys(glob='*')
-          Store.db.keys(key(glob)).collect {|c| c[5..-1]}
+          Store.keys(key(glob)).collect {|c| c[5..-1]}
         end
       
         def reset(metric)
-          Store.db.del(key(metric))
+          Store.flush_data(key(metric))
         end
       
         def reset_all
-          Store.flash_data('stat:*')
+          Store.flush_data('stat:*')
         end
       
         # Returns simple string metric
@@ -79,7 +79,7 @@ module Rhoconnect
     
         # Returns the metric data, uses array indexing
         def range(metric, start, finish = -1)
-          Store.db.zrange(key(metric), start, finish)
+          Store.zrange(key(metric), start, finish)
         end
       
         # Returns the resolution for a given metric, default 60 seconds
@@ -96,7 +96,7 @@ module Rhoconnect
       
         # Returns redis object type for a record
         def rtype(metric)
-          Store.db.type(key(metric))
+          Store.doc_type(key(metric))
         end
       
         def key(metric)