rhoconnect 3.0.0.beta1

data/lib/rhoconnect/source_sync.rb

@@ -0,0 +1,302 @@
+module Rhoconnect
+  class SourceSync
+    attr_reader :adapter
+    
+    def initialize(source)
+      @source = source
+      raise InvalidArgumentError.new('Invalid source') if @source.nil?
+      raise InvalidArgumentError.new('Invalid app for source') unless @source.app
+      @adapter = SourceAdapter.create(@source)
+    end
+    
+    # CUD Operations
+    def create(client_id)
+      _measure_and_process_cud('create',client_id)
+    end
+    
+    def update(client_id)
+      _measure_and_process_cud('update',client_id)
+    end
+    
+    def delete(client_id)
+      _measure_and_process_cud('delete',client_id)
+    end
+    
+    # Pass through CUD to adapter, no data stored
+    def pass_through_cud(cud_params,query_params)
+      return if _auth_op('login') == false
+      res,processed_objects = {},[]
+      begin
+        ['create','update','delete'].each do |op|
+          key,objects = op,cud_params[op]
+          objects.each do |key,value|
+            case op
+            when 'create'
+              @adapter.send(op.to_sym,value)
+            when 'update'
+              value['id'] = key
+              @adapter.send(op.to_sym,value)
+            when 'delete'
+              value['id'] = key
+              @adapter.send(op.to_sym,value)
+            end
+            processed_objects << key
+          end if objects
+        end
+      rescue Exception => e
+        log "Error in pass through method: #{e.message}"
+        res['error'] = {'message' => e.message } 
+      end
+      _auth_op('logoff')
+      res['processed'] = processed_objects
+      res.to_json
+    end
+    
+    # Read Operation; params are query arguments
+    def read(client_id=nil,params=nil)
+      _read('query',client_id,params)
+    end
+    
+    def search(client_id=nil,params=nil)
+      return if _auth_op('login',client_id) == false
+      res = _read('search',client_id,params)
+      _auth_op('logoff',client_id)
+      res
+    end
+
+    def process_cud(client_id)
+      if @source.cud_queue or @source.queue
+        async(:cud,@source.cud_queue || @source.queue,client_id)
+      else
+        do_cud(client_id)
+      end   
+    end
+    
+    def do_cud(client_id)
+      return if _auth_op('login') == false
+      self.create(client_id)
+      self.update(client_id)
+      self.delete(client_id)
+      _auth_op('logoff')
+    end
+    
+    def process_query(params=nil)
+      if @source.query_queue or @source.queue
+        async(:query,@source.query_queue || @source.queue,nil,params)
+      else
+        do_query(params)
+      end   
+    end
+    
+    def do_query(params=nil)
+      result = nil
+      @source.if_need_refresh do
+        Stats::Record.update("source:query:#{@source.name}") do
+          return if _auth_op('login') == false
+          result = self.read(nil,params)
+          _auth_op('logoff')
+        end  
+      end
+      result
+    end
+    
+    # Enqueue a job for the source based on job type
+    def async(job_type,queue_name,client_id=nil,params=nil)
+      SourceJob.queue = queue_name
+      Resque.enqueue(SourceJob,job_type,@source.id,
+        @source.app_id,@source.user_id,client_id,params)
+    end
+    
+    def push_objects(objects,timeout=10,raise_on_expire=false)
+      @source.lock(:md,timeout,raise_on_expire) do |s|
+        doc = @source.get_data(:md)
+        orig_doc_size = doc.size
+        objects.each do |id,obj|
+          doc[id] ||= {}
+          doc[id].merge!(obj)
+        end  
+        diff_count = doc.size - orig_doc_size
+        @source.put_data(:md,doc)
+        @source.update_count(:md_size,diff_count)
+      end      
+    end    
+
+    def push_deletes(objects,timeout=10,raise_on_expire=false)
+      @source.lock(:md,timeout,raise_on_expire) do |s|
+        doc = @source.get_data(:md)
+        orig_doc_size = doc.size
+        objects.each do |id|
+          doc.delete(id)
+        end  
+        diff_count = doc.size - orig_doc_size
+        @source.put_data(:md,doc)
+        @source.update_count(:md_size,diff_count)
+      end      
+    end
+    
+    private
+    def _auth_op(operation,client_id=-1)
+      edockey = client_id == -1 ? @source.docname(:errors) :
+        Client.load(client_id,{:source_name => @source.name}).docname(:search_errors)
+      begin
+        Store.flash_data(edockey) if operation == 'login'
+        @adapter.send operation
+      rescue Exception => e
+        log "SourceAdapter raised #{operation} exception: #{e}"
+        log e.backtrace.join("\n")
+        Store.put_data(edockey,{"#{operation}-error"=>{'message'=>e.message}},true)
+        return false
+      end
+      true
+    end
+    
+    def _process_create(client,key,value,links,creates,deletes)
+      # Perform operation
+      link = @adapter.create value
+      # Store object-id link for the client
+      # If we have a link, store object in client document
+      # Otherwise, store object for delete on client
+      if link
+        links ||= {}
+        links[key] = { 'l' => link.to_s }
+        creates ||= {}
+        creates[link.to_s] = value
+      else
+        deletes ||= {}
+        deletes[key] = value
+      end
+    end
+    
+    def _process_update(client,key,value)
+      begin
+        # Add id to object hash to forward to backend call
+        value['id'] = key
+        # Perform operation
+        @adapter.update value
+      rescue Exception => e
+        # TODO: This will be slow!
+        cd = client.get_data(:cd)
+        client.put_data(:update_rollback,{key => cd[key]},true) if cd[key]
+        raise e
+      end
+    end
+    
+    def _process_delete(client,key,value,dels)
+      value['id'] = key
+      # Perform operation
+      @adapter.delete value
+      dels ||= {}
+      dels[key] = value
+    end
+    
+    def _measure_and_process_cud(operation,client_id)
+      Stats::Record.update("source:#{operation}:#{@source.name}") do
+        _process_cud(operation,client_id)
+      end
+    end
+    
+    def _process_cud(operation,client_id)
+      errors,links,deletes,creates,dels = {},{},{},{},{}
+      client = Client.load(client_id,{:source_name => @source.name})
+      modified = client.get_data(operation)
+      # Process operation queue, one object at a time
+      modified.each do |key,value|
+        begin
+          # Remove object from queue
+          modified.delete(key)
+          # Call on source adapter to process individual object
+          case operation
+          when 'create'
+            _process_create(client,key,value,links,creates,deletes)
+          when 'update'
+            _process_update(client,key,value)
+          when 'delete'
+            _process_delete(client,key,value,dels)
+          end
+        rescue Exception => e
+          log "SourceAdapter raised #{operation} exception: #{e}"
+          log e.backtrace.join("\n")
+          errors ||= {}
+          errors[key] = value
+          errors["#{key}-error"] = {'message'=>e.message}
+          break
+        end
+      end
+      # Record operation results
+      { "delete_page" => deletes,
+        "#{operation}_links" => links,
+        "#{operation}_errors" => errors }.each do |doctype,value|
+        client.put_data(doctype,value,true) unless value.empty?
+      end
+      unless operation != 'create' and creates.empty?
+        client.put_data(:cd,creates,true)
+        client.update_count(:cd_size,creates.size)
+        @source.lock(:md) do |s| 
+          s.put_data(:md,creates,true)
+          s.update_count(:md_size,creates.size)
+        end
+      end
+      if operation == 'delete'
+        # Clean up deleted objects from master document and corresponding client document
+        client.delete_data(:cd,dels)
+        client.update_count(:cd_size,-dels.size)
+        @source.lock(:md) do |s| 
+          s.delete_data(:md,dels)
+          s.update_count(:md_size,-dels.size)
+        end
+      end
+      # Record rest of queue (if something in the middle failed)
+      if modified.empty?
+        client.flash_data(operation)
+      else
+        client.put_data(operation,modified)
+      end
+      modified.size
+    end
+    
+    # Metadata Operation; source adapter returns json
+    def _get_data(method)
+      if @adapter.respond_to?(method)
+        data = @adapter.send(method) 
+        if data
+          @source.put_value(method,data)
+          @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(data))
+        end
+      end
+    end
+    
+   # Read Operation; params are query arguments
+    def _read(operation,client_id,params=nil)
+      errordoc = nil
+      result = nil
+      begin
+        if operation == 'search'
+          client = Client.load(client_id,{:source_name => @source.name})
+          errordoc = client.docname(:search_errors)
+          compute_token(client.docname(:search_token))
+          result = @adapter.search(params)
+          @adapter.save(client.docname(:search)) unless @source.is_pass_through?
+        else
+          errordoc = @source.docname(:errors)
+          [:metadata,:schema].each do |method|
+            _get_data(method)
+          end
+          result = @adapter.do_query(params)
+        end
+        # operation,sync succeeded, remove errors
+        Store.lock(errordoc) do
+          Store.flash_data(errordoc)
+        end
+      rescue Exception => e
+        # store sync,operation exceptions to be sent to all clients for this source/user
+        log "SourceAdapter raised #{operation} exception: #{e}"
+        log e.backtrace.join("\n")
+        Store.lock(errordoc) do
+          Store.put_data(errordoc,{"#{operation}-error"=>{'message'=>e.message}},true)
+        end
+      end
+      # pass through expects result hash
+      @source.is_pass_through? ? result : true
+    end
+  end
+end

data/lib/rhoconnect/stats/middleware.rb

@@ -0,0 +1,20 @@
+module Rhoconnect
+  module Stats
+    class Middleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        start = Time.now.to_f
+        status, headers, body = @app.call(env)
+        finish = Time.now.to_f
+        metric = "http:#{env['REQUEST_METHOD']}:#{env['PATH_INFO']}"
+        source_name = env['rack.request.query_hash']["source_name"] if env['rack.request.query_hash']
+        metric << ":#{source_name}" if source_name
+        Record.save_average(metric,finish - start)
+        [status, headers, body]
+      end
+    end
+  end
+end

data/lib/rhoconnect/stats/record.rb

@@ -0,0 +1,108 @@
+module Rhoconnect
+  module Stats
+    class Record
+      class << self
+        
+        # Add a value to a metric.  If zset already has a member, 
+        # update the existing member with an incremented value by default.
+        # Also supports updating the value with a block (useful for averages)
+        def set(metric, value = 0)
+          start = (Time.now.to_i / resolution(metric)) * resolution(metric)
+          current, current_score = 0, start
+          Store.lock(key(metric)) do
+            range = Store.db.zrevrange(key(metric), 0, 0)
+            if !range.empty?
+              member = range[0]
+              m_current = member.split(':')[0]
+              m_current_score = Store.db.zscore(key(metric), member).to_i
+              if m_current_score > (start - resolution(metric))
+                Store.db.zrem(key(metric), member)
+                current, current_score = m_current, m_current_score
+              end
+            end
+            value = block_given? ? yield(current, value) : value 
+            Store.db.zadd(key(metric), current_score, "#{value}:#{start}")
+            Store.db.zremrangebyscore(key(metric), 0, start - record_size(metric))
+          end
+        end
+        
+        def add(metric, value = 1)
+          set(metric,value) { |current,value| current.to_i + value } 
+        end
+        
+        # Saves the accumulated average for a resolution in a metric
+        def save_average(metric, value)
+          set(metric,value) do |current,value| 
+            sum = value
+            if current.is_a?(String)
+              current,sum = current.split(',')
+              current = current.to_f
+              sum = sum.to_f+value
+            end
+            "#{current + 1},#{sum}"
+          end  
+        end
+        
+        def update(metric)
+          if Rhoconnect.stats
+            start = Time.now.to_f
+            # perform the operations
+            yield
+            finish = Time.now.to_f
+            save_average(metric, finish - start)
+          else
+            yield
+          end
+        end
+        
+        def keys(glob='*')
+          Store.db.keys(key(glob)).collect {|c| c[5..-1]}
+        end
+        
+        def reset(metric)
+          Store.db.del(key(metric))
+        end
+        
+        def reset_all
+          Store.flash_data('stat:*')
+        end
+        
+        # Returns simple string metric
+        def get_value(metric)
+          Store.get_value(key(metric))
+        end
+        
+        # Sets a string metric
+        def set_value(metric, value)
+          Store.set_value(key(metric), value)
+        end
+      
+        # Returns the metric data, uses array indexing
+        def range(metric, start, finish = -1)
+          Store.db.zrange(key(metric), start, finish)
+        end
+        
+        # Returns the resolution for a given metric, default 60 seconds
+        def resolution(metric)
+          resolution = STATS_RECORD_RESOLUTION rescue nil
+          resolution || 60 #=> 1 minute aggregate
+        end
+        
+        # Returns the # of records to save for a given metric
+        def record_size(metric)
+          size = STATS_RECORD_SIZE rescue nil
+          size || 60 * 24 * 31 #=> 44640 minutes
+        end
+        
+        # Returns redis object type for a record
+        def rtype(metric)
+          Store.db.type(key(metric))
+        end
+        
+        def key(metric)
+          "stat:#{metric}"
+        end
+      end
+    end
+  end
+end

data/lib/rhoconnect/store.rb

@@ -0,0 +1,232 @@
+module Rhoconnect
+  
+  class StoreLockException < RuntimeError; end
+  
+  class Store
+    RESERVED_ATTRIB_NAMES = ["attrib_type", "id"] unless defined? RESERVED_ATTRIB_NAMES
+    @@db = nil
+    
+    class << self
+      def db; @@db || @@db = _get_redis end
+      
+      def db=(server=nil)
+        @@db = _get_redis(server)
+      end
+      
+      def create(server=nil)
+        @@db ||= _get_redis(server)
+        raise "Error connecting to Redis store." unless @@db and 
+          (@@db.is_a?(Redis) or @@db.is_a?(Redis::Client))
+      end
+  
+      # Adds set with given data, replaces existing set
+      # if it exists or appends data to the existing set
+      # if append flag set to true
+      def put_data(dockey,data={},append=false)
+        if dockey and data
+          flash_data(dockey) unless append
+          # Inserts a hash or array
+          if data.is_a?(Hash)
+            @@db.pipelined do
+              data.each do |key,value|
+                raise ArgumentError, "Invalid value object: #{value.inspect}. Hash is expected." unless value.is_a?(Hash)   
+                value.each do |attrib,value|
+                  unless _is_reserved?(attrib,value)
+                    @@db.sadd(dockey,setelement(key,attrib,value))
+                  end
+                end
+              end
+            end
+          else
+            @@db.pipelined do
+              data.each do |value|
+                @@db.sadd(dockey,value)
+              end
+            end
+          end
+        end
+        true
+      end
+    
+      # Adds a simple key/value pair
+      def put_value(dockey,value)
+        if dockey
+          @@db.del(dockey)
+          @@db.set(dockey,value.to_s) if value
+        end
+      end
+    
+      # Retrieves value for a given key
+      def get_value(dockey)
+        @@db.get(dockey) if dockey
+      end
+      
+      def incr(dockey)
+        @@db.incr(dockey)
+      end
+      
+      def decr(dockey)
+        @@db.decr(dockey)
+      end
+  
+      # Retrieves set for given dockey,source,user
+      def get_data(dockey,type=Hash)
+        res = type == Hash ? {} : []
+        if dockey
+          members = @@db.smembers(dockey)
+          members.each do |element|
+            if type == Hash
+              key,attrib,value = getelement(element)
+              res[key] = {} unless res[key]
+              res[key].merge!({attrib => value})
+            else
+              res << element
+            end
+          end if members
+          res
+        end
+      end
+  
+      # Retrieves diff data hash between two sets
+      def get_diff_data(src_dockey,dst_dockey,p_size=nil)
+        res = {}
+        if src_dockey and dst_dockey
+          @@db.sdiff(dst_dockey,src_dockey).each do |element|
+            key,attrib,value = getelement(element)
+            res[key] = {} unless res[key]
+            res[key].merge!({attrib => value})
+          end
+        end
+        if p_size
+          diff = {}
+          page_size = p_size
+          res.each do |key,item|
+            diff[key] = item
+            page_size -= 1
+            break if page_size <= 0         
+          end
+          [diff,res.size]
+        else  
+          [res,res.size]
+        end
+      end
+
+      # Deletes data from a given doctype,source,user
+      def delete_data(dockey,data={})
+        if dockey and data
+          @@db.pipelined do
+            data.each do |key,value|
+              value.each do |attrib,val|
+                @@db.srem(dockey,setelement(key,attrib,val))
+              end
+            end
+          end
+        end
+        true
+      end
+    
+      # Deletes all keys matching a given mask
+      def flash_data(keymask)
+        @@db.keys(keymask).each do |key|
+          @@db.del(key)
+        end
+      end
+    
+      # Returns array of keys matching a given keymask
+      def get_keys(keymask)
+        @@db.keys(keymask)
+      end
+    
+      # Returns true if given item is a member of the given set
+      def ismember?(setkey,item)
+        @@db.sismember(setkey,item)
+      end
+      
+      # Lock a given key and release when provided block is finished
+      def lock(dockey,timeout=0,raise_on_expire=false)
+        m_lock = get_lock(dockey,timeout,raise_on_expire)
+        res = yield
+        release_lock(dockey,m_lock)
+        res
+      end
+      
+      def get_lock(dockey,timeout=0,raise_on_expire=false)
+        lock_key = _lock_key(dockey)
+        current_time = Time.now.to_i   
+        ts = current_time+(Rhoconnect.lock_duration || timeout)+1
+        loop do 
+          if not @@db.setnx(lock_key,ts)
+            if raise_on_expire or Rhoconnect.raise_on_expired_lock
+              if @@db.get(lock_key).to_i <= current_time
+                # lock expired before operation which set it up completed
+                # this process cannot continue without corrupting locked data 
+                raise StoreLockException, "Lock \"#{lock_key}\" expired before it was released"
+              end
+            else  
+              if @@db.get(lock_key).to_i <= current_time and 
+                @@db.getset(lock_key,ts).to_i <= current_time
+                # previous lock expired and we replaced it with our own
+                break
+              end
+            end
+            sleep(1)
+            current_time = Time.now.to_i
+          else
+            break #no lock was set, so we set ours and leaving
+          end
+        end
+        return ts
+      end
+      
+      # Due to redis bug #140, setnx always returns true so this doesn't work
+      # def get_lock(dockey,timeout=0)
+      #   lock_key = _lock_key(dockey)
+      #   until @@db.setnx(lock_key,1) do 
+      #     sleep(1) 
+      #   end
+      #   @@db.expire(lock_key,timeout+1)
+      #   Time.now.to_i+timeout+1
+      # end
+      
+      def release_lock(dockey,lock)
+        @@db.del(_lock_key(dockey)) if (lock >= Time.now.to_i)
+      end
+      
+      # Create a copy of srckey in dstkey
+      def clone(srckey,dstkey)
+        @@db.sdiffstore(dstkey,srckey,'')
+      end
+      
+      # Rename srckey to dstkey
+      def rename(srckey,dstkey)
+        @@db.rename(srckey,dstkey) if @@db.exists(srckey)
+      end
+      
+      alias_method :set_value, :put_value
+      alias_method :set_data, :put_data
+      
+      private
+      def _get_redis(server=nil)
+        if ENV[REDIS_URL]
+          Redis.connect(:url => ENV[REDIS_URL])
+        elsif server and server.is_a?(String)
+          host,port,db,password = server.split(':')
+          Redis.new(:thread_safe => true, :host => host,
+            :port => port, :db => db, :password => password)
+        elsif server and server.is_a?(Redis)
+          server
+        else
+          Redis.new(:thread_safe => true)
+        end
+      end
+      
+      def _lock_key(dockey)
+        "lock:#{dockey}"
+      end
+          
+      def _is_reserved?(attrib,value) #:nodoc:
+        RESERVED_ATTRIB_NAMES.include? attrib
+      end
+    end
+  end
+end