rhosync 2.0.0.beta1

data/lib/rhosync/server.rb

@@ -0,0 +1,242 @@
+$:.unshift File.join(File.dirname(__FILE__),'..')
+require 'sinatra/base'
+require 'erb'
+require 'json'
+require 'fileutils'
+require 'rhosync'
+
+module Rhosync
+  
+  class ApiException < Exception
+    attr_accessor :error_code
+    def initialize(error_code,message)
+      super(message)
+      @error_code = error_code
+    end  
+  end
+    
+  class Server < Sinatra::Base
+    libdir = File.dirname(File.expand_path(__FILE__))
+    set :views,  "#{libdir}/server/views"
+    set :public, "#{libdir}/server/public"
+    set :static, true
+    
+    set :secret, '<changeme>' unless defined? Server.secret
+    
+    use Rack::Session::Cookie, :key => 'rhosync_session',
+                               :expire_after => 31536000,
+                               :secret => Server.secret
+                             
+    # Setup route and mimetype for bulk data downloads
+    # TODO: Figure out why "mime :data, 'application/octet-stream'" doesn't work
+    Rack::Mime::MIME_TYPES['.data'] = 'application/octet-stream'   
+     
+    include Rhosync
+                                                              
+    helpers do
+      def request_action
+        request.env['PATH_INFO'].split('/').last
+      end
+
+      def check_api_token
+        request_action == 'get_api_token' or 
+          (params[:api_token] and ApiToken.is_exist?(params[:api_token]))
+      end
+
+      def do_login
+        login ? status(200) : status(401)
+      end
+
+      def login_required
+        current_user.nil?
+      end
+
+      def login
+        if current_app and current_app.can_authenticate?
+          user = current_app.authenticate(params[:login], params[:password], session)
+        else
+          user = User.authenticate(params[:login], params[:password])
+        end
+        if user
+          session[:login] = user.login
+          session[:app_name] = APP_NAME
+          true
+        else
+          false
+        end
+      end
+
+      def logout
+        session[:login] = nil
+      end
+
+      def current_user
+        if @user.nil? and User.is_exist?(session[:login]) 
+          @user = User.load(session[:login])
+        end
+        if @user and (@user.admin == 1 || session[:app_name] == APP_NAME)
+          @user
+        else  
+          nil
+        end  
+      end
+
+      def api_user
+        request_action == 'get_api_token' ? current_user : ApiToken.load(params[:api_token]).user
+      end
+
+      def current_app
+        App.load(APP_NAME)
+      end
+
+      def current_source
+        return @source if @source 
+        user = current_user
+        if params[:source_name] and user
+          @source = Source.load(params[:source_name],
+            {:user_id => user.login,:app_id => APP_NAME}) 
+        else
+          nil
+        end
+      end
+
+      def current_client
+        if @client.nil? and params[:client_id]
+          @client = Client.load(params[:client_id].to_s,
+            params[:source_name] ? {:source_name => current_source.name} : {:source_name => '*'}) 
+        end  
+      end
+      
+      def source_config
+        { "sources" => Rhosync.get_config(Rhosync.base_directory)[:sources] }
+      end
+
+      def catch_all
+        begin
+          yield
+        rescue Exception => e
+          #log e.message + e.backtrace.join("\n")
+          throw :halt, [500, e.message]
+        end
+      end
+    end
+        
+    def initialize
+      # Whine about default session secret
+      check_default_secret!(Server.secret)
+      super
+    end
+    
+    Rhosync.log "Rhosync Server v#{Rhosync::VERSION} started..."
+
+    before do
+      if params["cud"]
+        cud = JSON.parse(params["cud"])
+        params.delete("cud")
+        params.merge!(cud)
+      end
+      if request.env['CONTENT_TYPE'] == 'application/json'
+        params.merge!(JSON.parse(request.body.read))
+        request.body.rewind
+      end      
+      if params[:version] and params[:version].to_i < 3
+        throw :halt, [404, "Server supports version 3 or higher of the protocol."]
+      end
+      #log "request params: #{params.inspect}"
+    end
+
+    %w[get post].each do |verb|
+      send(verb, "/application*") do
+        unless request_action == 'clientlogin'
+          throw :halt, [401, "Not authenticated"] if login_required
+        end
+        pass
+      end
+    end
+
+    get '/' do
+      erb :index
+    end
+
+    # Collection routes
+    post '/login' do
+      logout
+      do_login
+    end
+
+    post '/application/clientlogin' do
+      logout
+      do_login
+    end
+
+    get '/application/clientcreate' do
+      content_type :json
+      client = Client.create(:user_id => current_user.id,:app_id => current_app.id)
+      client.update_fields(params)
+      { "client" => { "client_id" =>  client.id.to_s } }.merge!(source_config).to_json
+    end
+
+    post '/application/clientregister' do
+      current_client.update_fields(params)
+      source_config.to_json
+    end
+
+    get '/application/clientreset' do
+      ClientSync.reset(current_client)
+      source_config.to_json
+    end
+
+    # Member routes
+    get '/application' do
+      catch_all do
+        content_type :json
+        cs = ClientSync.new(current_source,current_client,params[:p_size])
+        res = cs.send_cud(params[:token],params[:query]).to_json
+        res
+      end
+    end
+
+    post '/application' do
+      catch_all do
+        cs = ClientSync.new(current_source,current_client,params[:p_size]) 
+        cs.receive_cud(params)
+        status 200
+      end
+    end
+
+    get '/application/bulk_data' do
+      catch_all do
+        content_type :json
+        data = ClientSync.bulk_data(params[:partition].to_sym,current_client)
+        data.to_json
+      end
+    end
+
+    get '/application/search' do
+      catch_all do
+        content_type :json
+        ClientSync.search_all(current_client,params).to_json
+      end
+    end
+
+    def self.api(name)
+      post "/api/#{name}" do
+        if check_api_token
+          begin
+            yield params,api_user
+          rescue ApiException => ae
+            throw :halt, [ae.error_code, ae.message]  
+          rescue Exception => e
+            # log e.message + "\n" + e.backtrace.join("\n")
+            throw :halt, [500, e.message]
+          end
+        else
+          throw :halt, [422, "No API token provided"]
+        end
+      end
+    end
+  end
+end
+
+include Rhosync
+Dir[File.join(File.dirname(__FILE__),'api','**','*.rb')].each { |api| load api }

data/lib/rhosync/source.rb

@@ -0,0 +1,112 @@
+module Rhosync
+  class Source < Model
+    field :source_id,:integer
+    field :name,:string
+    field :url,:string
+    field :login,:string
+    field :password,:string
+    field :priority,:integer
+    field :callback_url,:string
+    field :poll_interval,:integer
+    field :partition_type,:string
+    field :sync_type,:string
+    field :queue,:string
+    field :query_queue,:string
+    field :cud_queue,:string
+    attr_accessor :app_id, :user_id
+    validates_presence_of :name #, :source_id
+    
+    include Document
+    include LockOps
+  
+    def self.set_defaults(fields)
+      fields[:url] ||= ''
+      fields[:login] ||= ''
+      fields[:password] ||= ''
+      fields[:priority] ||= 3
+      fields[:partition_type] ||= :user
+      fields[:poll_interval] ||= 300
+      fields[:sync_type] ||= :incremental
+    end
+        
+    def self.create(fields,params)
+      fields = fields.with_indifferent_access # so we can access hash keys as symbols
+      # validate_attributes(params)
+      fields[:id] = fields[:name]
+      set_defaults(fields)
+      super(fields,params)
+    end
+    
+    def self.load(id,params)
+      validate_attributes(params)
+      super(id,params)
+    end
+    
+    def update(fields)
+      self.class.set_defaults(fields)
+      super(fields)
+    end
+    
+    def clone(src_doctype,dst_doctype)
+      Store.clone(docname(src_doctype),docname(dst_doctype))
+    end
+    
+    # Return the user associated with a source
+    def user
+      @user ||= User.load(self.user_id)
+    end
+    
+    # Return the app the source belongs to
+    def app
+      @app ||= App.load(self.app_id)
+    end
+    
+    def read_state
+      id = {:app_id => self.app_id,:user_id => user_by_partition,
+        :source_name => self.name}
+      @read_state ||= ReadState.load(id)
+      @read_state ||= ReadState.create(id)   
+    end
+    
+    def doc_suffix(doctype)
+      "#{user_by_partition}:#{self.name}:#{doctype.to_s}"
+    end
+    
+    def delete
+      flash_data('*')
+      super
+    end
+    
+    def partition
+      self.partition_type.to_sym
+    end
+    
+    def partition=(value)
+      self.partition_type = value
+    end
+    
+    def user_by_partition
+      self.partition.to_sym == :user ? self.user_id : '__shared__'
+    end
+  
+    def check_refresh_time
+      self.poll_interval == 0 or 
+      (self.poll_interval != -1 and self.read_state.refresh_time <= Time.now.to_i)
+    end
+        
+    def if_need_refresh(client_id=nil,params=nil)
+      need_refresh = lock(:md) do |s|
+        check = check_refresh_time
+        s.read_state.refresh_time = Time.now.to_i + s.poll_interval if check
+        check
+      end
+      yield client_id,params if need_refresh
+    end
+    
+    private
+    def self.validate_attributes(params)
+      raise ArgumentError.new('Missing required attribute user_id') unless params[:user_id]
+      raise ArgumentError.new('Missing required attribute app_id') unless params[:app_id]
+    end
+  end
+end

data/lib/rhosync/source_adapter.rb

@@ -0,0 +1,95 @@
+module Rhosync
+  class SourceAdapterException < RuntimeError; end
+
+  # raise this to cause client to be logged out during a sync
+  class SourceAdapterLoginException < SourceAdapterException; end
+
+  class SourceAdapterLogoffException < SourceAdapterException; end
+
+  # raise these to trigger rhosync sending an error to the client
+  class SourceAdapterServerTimeoutException < SourceAdapterException; end
+  class SourceAdapterServerErrorException < SourceAdapterException; end
+
+  class SourceAdapter
+    attr_accessor :session
+    
+    def initialize(source,credential=nil)
+      @source = source
+    end
+    
+    # Returns an instance of a SourceAdapter by source name
+    def self.create(source,credential=nil)
+      adapter=nil
+      if source
+        begin
+          source.name.strip! if source.name
+          require under_score(source.name)
+          adapter=(Object.const_get(source.name)).new(source,credential) 
+        rescue Exception=>e
+          log "Failure to create adapter from class #{source.name}: #{e.inspect.to_s}"
+          raise e
+        end
+      end
+      adapter
+    end
+
+    def login; end
+  
+    def query(params=nil); end
+    
+    def search(params=nil); end
+    
+    def sync
+      return if _result_nil?
+      if @result.empty?
+        @source.lock(:md) do |s|
+          s.flash_data(:md)
+          s.put_value(:md_size,0)
+        end
+      else
+        tmp_docname = @source.docname(:md) + get_random_uuid
+        Store.put_data(tmp_docname,@result)
+        @source.lock(:md) do |s|
+          Store.rename(tmp_docname,s.docname(:md))
+          s.put_value(:md_size,@result.size)
+        end
+      end
+    end
+  
+    def create(name_value_list); end
+
+    def update(name_value_list); end
+
+    def delete(name_value_list); end
+    
+    def ask(params=nil); end
+
+    def logoff; end
+    
+    def save(docname)
+      return if _result_nil?
+      if @result.empty?
+        Store.flash_data(docname)
+      else
+        Store.put_data(docname,@result)
+      end
+    end
+  
+    # only implement this if you want RhoSync to install a callback into your backend
+    # def set_callback(notify_url)
+    # end
+  
+    MSG_NIL_RESULT_ATTRIB = "You might have expected a synchronization but the @result attribute was 'nil'"
+    
+    protected
+    def current_user
+      @source.user
+    end
+  
+    private 
+    def _result_nil? #:nodoc:
+      log MSG_NIL_RESULT_ATTRIB if @result.nil?
+      @result.nil?
+    end
+  end
+end

data/lib/rhosync/source_sync.rb

@@ -0,0 +1,245 @@
+module Rhosync
+  class SourceSync
+    attr_reader :adapter
+    
+    def initialize(source)
+      @source = source
+      raise InvalidArgumentError.new('Invalid source') if @source.nil?
+      raise InvalidArgumentError.new('Invalid app for source') unless @source.app
+      @adapter = SourceAdapter.create(@source)
+    end
+    
+    # CUD Operations
+    def create(client_id)
+      _process_cud('create',client_id)
+    end
+    
+    def update(client_id)
+      _process_cud('update',client_id)
+    end
+    
+    def delete(client_id)
+      _process_cud('delete',client_id)
+    end
+    
+    # Read Operation; params are query arguments
+    def read(client_id=nil,params=nil)
+      _read('query',client_id,params)
+    end
+    
+    def search(client_id=nil,params=nil)
+      return if _auth_op('login',client_id) == false
+      res = _read('search',client_id,params)
+      _auth_op('logoff',client_id)
+      res
+    end
+
+    def process_cud(client_id)
+      if @source.cud_queue or @source.queue
+        async(:cud,@source.cud_queue || @source.queue,client_id)
+      else
+        do_cud(client_id)
+      end   
+    end
+    
+    def do_cud(client_id)
+      return if _auth_op('login') == false
+      self.create(client_id)
+      self.update(client_id)
+      self.delete(client_id)
+      _auth_op('logoff')
+    end
+    
+    def process_query(params=nil)
+      if @source.query_queue or @source.queue
+        async(:query,@source.query_queue || @source.queue,nil,params)
+      else
+        do_query(params)
+      end   
+    end
+    
+    def do_query(params=nil)
+      @source.if_need_refresh do
+        return if _auth_op('login') == false
+        self.read(nil,params)
+        _auth_op('logoff')
+      end
+    end
+    
+    # Enqueue a job for the source based on job type
+    def async(job_type,queue_name,client_id=nil,params=nil)
+      SourceJob.queue = queue_name
+      Resque.enqueue(SourceJob,job_type,@source.id,
+        @source.app_id,@source.user_id,client_id,params)
+    end
+    
+    def push_objects(objects)
+      @source.lock(:md) do |s|
+        doc = @source.get_data(:md)
+        objects.each do |id,obj|
+          doc[id] ||= {}
+          doc[id].merge!(obj)
+        end  
+        @source.put_data(:md,doc)
+        @source.update_count(:md_size,doc.size)
+      end      
+    end    
+
+    def push_deletes(objects)
+      @source.lock(:md) do |s|
+        doc = @source.get_data(:md)
+        objects.each do |id|
+          doc.delete(id)
+        end  
+        @source.put_data(:md,doc)
+        @source.update_count(:md_size,doc.size)
+      end      
+    end
+    
+    private
+    def _auth_op(operation,client_id=-1)
+      edockey = client_id == -1 ? @source.docname(:errors) :
+        Client.load(client_id,{:source_name => @source.name}).docname(:search_errors)
+      begin
+        Store.flash_data(edockey) if operation == 'login'
+        @adapter.send operation
+      rescue Exception => e
+        log "SourceAdapter raised #{operation} exception: #{e}"
+        Store.put_data(edockey,{"#{operation}-error"=>{'message'=>e.message}},true)
+        return false
+      end
+      true
+    end
+    
+    def _process_create(client_id,key,value,links,creates,deletes)
+      # Perform operation
+      link = @adapter.create value
+      # Store object-id link for the client
+      # If we have a link, store object in client document
+      # Otherwise, store object for delete on client
+      if link
+        links ||= {}
+        links[key] = { 'l' => link.to_s }
+        creates ||= {}
+        creates[link.to_s] = value
+      else
+        deletes ||= {}
+        deletes[key] = value
+      end
+    end
+    
+    def _process_update(client_id,key,value)
+      # Add id to object hash to forward to backend call
+      value['id'] = key
+      # Perform operation
+      @adapter.update value
+    end
+    
+    def _process_delete(client_id,key,value,dels)
+      value['id'] = key
+      # Perform operation
+      @adapter.delete value
+      dels ||= {}
+      dels[key] = value
+    end
+    
+    def _process_cud(operation,client_id)
+      errors,links,deletes,creates,dels = {},{},{},{},{}
+      client = Client.load(client_id,{:source_name => @source.name})
+      modified = client.get_data(operation)
+      # Process operation queue, one object at a time
+      modified.each do |key,value|
+        begin
+          # Remove object from queue
+          modified.delete(key)
+          # Call on source adapter to process individual object
+          case operation
+          when 'create'
+            _process_create(client_id,key,value,links,creates,deletes)
+          when 'update'
+            _process_update(client_id,key,value)
+          when 'delete'
+            _process_delete(client_id,key,value,dels)
+          end
+        rescue Exception => e
+          log "SourceAdapter raised #{operation} exception: #{e}"
+          errors ||= {}
+          errors[key] = value
+          errors["#{key}-error"] = {'message'=>e.message}
+          break
+        end
+      end
+      # Record operation results
+      { "delete_page" => deletes,
+        "#{operation}_links" => links,
+        "#{operation}_errors" => errors }.each do |doctype,value|
+        client.put_data(doctype,value,true) unless value.empty?
+      end
+      unless operation != 'create' and creates.empty?
+        client.put_data(:cd,creates,true)
+        client.update_count(:cd_size,creates.size)
+        @source.lock(:md) do |s| 
+          s.put_data(:md,creates,true)
+          s.update_count(:md_size,creates.size)
+        end
+      end
+      if operation == 'delete'
+        # Clean up deleted objects from master document and corresponding client document
+        client.delete_data(:cd,dels)
+        client.update_count(:cd_size,-dels.size)
+        @source.lock(:md) do |s| 
+          s.delete_data(:md,dels)
+          s.update_count(:md_size,-dels.size)
+        end
+      end
+      # Record rest of queue (if something in the middle failed)
+      if modified.empty?
+        client.flash_data(operation)
+      else
+        client.put_data(operation,modified)
+      end
+      modified.size
+    end
+    
+    # Metadata Operation; source adapter returns json
+    def _get_metadata
+      if @adapter.respond_to?(:metadata)
+        metadata = @adapter.metadata 
+        if metadata
+          @source.put_value(:metadata,metadata)
+          @source.put_value(:metadata_sha1,Digest::SHA1.hexdigest(metadata))
+        end
+      end
+    end
+    
+    # Read Operation; params are query arguments
+    def _read(operation,client_id,params=nil)
+      errordoc = nil
+      begin
+        if operation == 'search'
+          client = Client.load(client_id,{:source_name => @source.name})
+          errordoc = client.docname(:search_errors)
+          compute_token(client.docname(:search_token))
+          @adapter.search(params)
+          @adapter.save(client.docname(:search))
+        else
+          errordoc = @source.docname(:errors)
+          _get_metadata
+          params ? @adapter.query(params) : @adapter.query
+          @adapter.sync
+        end
+        # operation,sync succeeded, remove errors
+        Store.lock(errordoc) do
+          Store.flash_data(errordoc)
+        end
+      rescue Exception => e
+        # store sync,operation exceptions to be sent to all clients for this source/user
+        log "SourceAdapter raised #{operation} exception: #{e}"
+        Store.lock(errordoc) do
+          Store.put_data(errordoc,{"#{operation}-error"=>{'message'=>e.message}},true)
+        end
+      end
+      true
+    end
+  end
+end