joffice_redis 0.1.1

data/lib/joffice_redis/redis_client_base.rb

@@ -0,0 +1,121 @@
+#encoding: utf-8
+=begin
+Copyright 2010 Denis Kokorin <virkdi@mail.ru>
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+=end
+
+
+require 'joffice/kernel/array'
+
+module JOffice
+  class RedisClientBase
+    
+    DEFAULT_KEYS_TO_SAVE= [:id].freeze
+    
+    def [](k)
+      db.get(key(k))
+    end
+    
+    def []=(k, value)
+      db.set!(key(k), value)
+    end
+    
+    def exists?(k)
+      db.exists?(key(k))   
+    end
+    
+    def delete_all
+      db.del!(db.keys("#{prefix}*"))
+    end
+    
+    def db_name
+      raise "Missed db name in class #{self}"
+    end
+    
+    def db
+      @db||=JOffice::RedisManager.instance.open_db_em(db_name)
+    end
+    
+    def prefix
+      ''
+    end
+    
+    def key(hash_key)
+      "#{prefix}#{hash_key.to_s.gsub(/ /,'_')}"      
+    end     
+    
+    def keys_to_save
+      DEFAULT_KEYS_TO_SAVE
+    end
+    
+    def flush_db
+      db.flush_db!
+    end
+    
+    def update_attribute(id, key, value)			
+      db.lset!("#{prefix}#{id}", keys_to_save.index(key), Marshal.dump(value))
+    end
+    
+    def set(id, value, raw=true)      
+      db.set!(key(id), raw ? value : Marshal.dump(value))
+    end
+    
+    def get(id, raw=true)      
+      value=db.get(key(id))
+       ((raw || !value) ? value : Marshal.load(value))
+    end
+    
+    def build_key(model)
+      key(model_id(model))
+    end
+    
+    def model_id(model)
+      model.id  
+    end
+    
+    def update_attributes(model, attributes)
+      id=model_id(model)
+      attributes.each do |name|
+        update_attribute(id, name, model.send(name))
+      end
+    end
+    
+    def add_or_update(model)
+      id=build_key(model)
+      keys_to_save.each do |key|
+        db.tail_push!(id, Marshal.dump(model.send(key)))
+      end
+      db.ltrim!(id, 0, keys_to_save.length)
+    end  
+    
+    def load_by_range(key, ranges)
+      result=[]
+      ranges.each do |range|        
+        db.lrange(key,range.first,range.last).each {|v| result << Marshal.load(v);   }        
+      end
+      result
+    end
+    
+    def load_values(id, *keys)
+      load_by_range(key(id), to_index_range(keys))      
+    end
+    
+    def to_index_range(keys)
+      keys.flatten.map {|key| keys_to_save.index(key.to_sym) }.to_ranges
+    end
+    private :to_index_range
+    
+  end
+  
+end

data/lib/joffice_redis/redis_manager.rb

@@ -0,0 +1,213 @@
+#encoding: utf-8
+=begin
+Copyright 2010 Denis Kokorin <virkdi@mail.ru>
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+=end
+
+
+require 'thread'
+
+module JOffice
+  class RedisManager
+    include Singleton
+    attr_reader :db_prefix
+    include JOffice::FiberEvents
+    def error_callback 
+      @error_callback||= lambda {|code| 
+        #$log.error("Redis"){"[RedisEM]Error code: #{code}"}
+        puts "[RedisEM]Error code: #{code}";          
+        #JOffice::LogOutputter.flush; 
+        #EM.next_tick{EM.stop} 
+      }
+    end
+    
+    def flush_all
+      open_redis.flush_all
+    end
+    
+    def increment_key
+      :"db.increment"
+    end
+    
+    private :increment_key
+    
+    def init_ping_timer
+      @@ping_fiber||=Fiber.new{
+        while Fiber.yield         
+          p "[Redis] ping"
+           (@@redis_em||={}).values.each do |r|            
+            r.exists!('0')
+          end          
+        end
+      }
+      EventMachine::PeriodicTimer.new(10.minutes) do
+        @@ping_fiber.resume true
+      end
+    end
+    private :init_ping_timer
+    
+    def start_ping_timer
+      @@ping_timer_thread||=init_ping_timer
+    end
+    private :start_ping_timer
+    
+    def config=(v)
+      v.symbolize_keys!
+      @config=v
+      @db_prefix=(config[:prefix] || 'test')
+      p "DB_PREFIX: #{db_prefix}"      
+    end
+    
+    def load_all_names(redis)
+      @db_names={}
+      a=redis.keys('db:*')
+      a.each do |name|
+        @db_names[name]=redis.get(name)
+      end if a
+    end
+    private :load_all_names
+    
+    def config
+      @config||={}
+    end
+    
+    def logger=(v)
+      @logger=v
+    end
+    
+    def logger
+      @logger||=nil
+    end
+    
+    def db_names
+      @db_names||={}
+    end
+    
+    private :db_names
+    
+    def last_access
+      @last_access||={}
+    end
+    private :last_access
+    
+    #def new_redis(db=1)
+    #  ::Redis.new({:logger=>logger,:db=>db}.merge(config))
+    #end
+    
+    #def get_or_open_redis_instance(db=1, forse=false)
+    #  key=:"redisdb_#{db_prefix}_#{db}"   
+    #  
+    #  if forse
+    #    p "Reopen db #{db}"
+    #    Thread.current[key]=new_redis(db)        
+    #  else
+    #    Thread.current[key]||=new_redis(db)
+    #  end
+    #  
+    #end
+    #private :get_or_open_redis_instance
+    
+    def force_open_em_redis(db)      
+      redis = EM::Protocols::Redis.connect((config[:host] || '127.0.0.1'), (config[:port] || 6379).to_i) 
+      redis.on_error(&error_callback)      
+      redis.select(db)      
+      redis
+    end
+    private :force_open_em_redis
+    
+    def redis_em
+      @@redis_em||=Hash.new { |hash, id| hash[id] = force_open_em_redis(id); };
+    end
+    
+    def open_em_redis(db=1)     
+      redis_em[db]
+    end
+    
+    #private :open_em_redis
+    def semaphore 
+      @semaphore||=Mutex.new
+    end
+    
+    def register_database(key)      
+      register_fiber_and_singleton_task("register_database(#{key}) ") do 
+        redis=open_em_redis      
+        id=redis.get(key)
+        if (new_db=id.nil?)
+          id=redis.incr(increment_key)
+          while id<2
+            id=redis.incr(increment_key)
+          end          
+          redis.set(key,id)
+        end          
+        p "Open db #{key} id=#{id} #{new_db}"
+        db_names[key]=id      
+      end
+    end
+    
+    # Lock the object so no other instances can modify it.
+    # This method implements the design pattern for locks
+    # described at: http://code.google.com/p/redis/wiki/SetnxCommand
+    #
+    # @see Model#mutex
+    def lock!
+      db=open_em_redis 
+      until db.setnx('_lock', lock_timeout)
+        next unless lock = db.get('_lock')
+        sleep(2) and next unless lock_expired?(lock)
+        break unless lock = db.getset('_lock', lock_timeout)
+        break if lock_expired?(lock)
+      end
+    end
+    
+    # Release the lock.
+    # @see Model#mutex
+    def unlock!
+      open_em_redis.del('_lock')
+    end    
+    
+    def lock_timeout
+      Time.now.to_f + 5
+    end
+    
+    def lock_expired? lock
+      lock.to_f < Time.now.to_f
+    end
+    
+    def open_db_em(name, force=false)
+      key= :"db:#{db_prefix}_#{name}"
+      open_em_redis(db_names[key] || register_database(key))              
+    end
+    
+    #    def opend_db(name, forse=false)
+    #      redis=nil	  
+    #      key= :"db:#{db_prefix}_#{name}"
+    #      new_db=false
+    #      id=(db_names[key] || (new_db=register_database(key)))
+    #      
+    #      redis=get_or_open_redis_instance(id, forse)
+    #      redis.flush_db if new_db
+    #      
+    #      if block_given?
+    #        begin			
+    #          yield redis
+    #        rescue
+    #          p "Error #{$!}"
+    #          yield get_or_open_redis_instance(id, true)
+    #        end			
+    #      end
+    #      #last_access[id]=Time.new.to_i
+    #      redis
+    #    end
+  end
+end

data/lib/joffice_redis/redis_method_factory.rb

@@ -0,0 +1,445 @@
+#encoding: utf-8
+=begin
+Copyright 2010 Denis Kokorin <virkdi@mail.ru>
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+=end
+
+
+require 'fileutils'
+require 'em-redis'
+require 'active_support'
+
+module EventMachine
+  module Protocols
+    class RedisMethodFactory
+      def initialize(parent)
+        @class_name='syncronized_redis_protocol'
+        #@buffer=['', "require 'em-redis'","require 'active_support'",'module EventMachine', 'module Protocols', 'module Redis']
+        @buffer=['']
+        yield self if block_given?
+        method 'ping' do |m|
+          m << 'exists?(0)'
+        end
+        method('blank?', 'args') do |m|
+          m << "args.blank? || args.flatten!.blank?"
+        end
+        #@buffer << 'end'
+        #@buffer << 'end'
+        #@buffer << 'end'
+        #generate
+        parent.class_eval @buffer.join("\n")
+      end
+      
+      def method(name, args=nil)      
+        buf=[]
+        yield buf
+        buf=buf.map{|v| "    #{v}"}.join("\n")
+        signature=(args ? "#{name}(#{args})" : name)
+        @buffer << ''
+        @buffer << "def #{signature}\n#{buf}\nend"
+        @buffer << "public :#{name}"
+        @buffer << ''
+      end
+      
+      def method_call_args
+        @method_call_args||=['', 'key', 'key, value']
+      end
+      
+      def method_call(name, options={})
+        name_sync="#{name}_sync"
+        name_async="#{name}_async"        
+        
+        args = (method_call_args[options[:args_count] || 1] || '*args')       
+        
+        call_args = case (options[:args_count] || 1)
+          when 2 then "[:#{name}, key, value]"
+          when 1 then "[:#{name}, key]"
+          when 0 then "[:#{name}]"
+        else "args.unshift(:#{name})"
+        end
+        
+        method(name_sync, args) do |m|
+          m << "return [] if blank?(args)" if (options[:args_count] || 1)>3
+          m << "c=Fiber.current"
+          m << "call_command(#{call_args}) { |a|  c.resume(a); }"
+          m << "Fiber.yield"
+        end
+        
+        method(name_async, args) do |m|
+          m << "return if blank?(args)" if (options[:args_count] || 1)>3
+          m << "call_command(#{call_args}) { |a| }"
+        end
+        
+        @buffer << "alias :#{name}! :#{name_async}"
+        @buffer << "alias :#{name} :#{options[:sync] ? name_sync : name_async}"
+        
+        if options[:alias]
+          options[:alias].each do |alias_name|
+            @buffer << "alias :#{alias_name} :#{name}"
+          end
+        end
+      end
+      
+      def method_alias(name, options={})      
+        em_name="#{name}_real"
+        name_sync="#{name}_sync"
+        name_async="#{name}_async"        
+        @buffer << "alias :#{em_name} :#{name}"
+        args = (method_call_args[options[:args_count].to_i || 1] || '*args')
+        
+        method(name_sync, args) do |m|
+          m << "return [] if blank?(args)" if (options[:args_count] || 1)>3
+          m << "c=Fiber.current"
+          m << "#{em_name}(#{args}) { |a|  c.resume(a); }"
+          m << "Fiber.yield"
+        end
+        
+        if options[:cache]
+          @buffer << "alias :#{name_sync}_no_cache :#{name_async}"
+          method(name_sync, args) do |m|
+            m << "@cache_#{name}||=#{name_sync}_no_cache(#{args})"
+          end
+        end
+        
+        method(name_async, args) do |m|
+          m << "return if blank?(args)" if (options[:args_count] || 1)>3
+          m << "#{em_name}(#{args}) { |a| }"
+        end
+        
+        @buffer << "alias :#{name}! :#{name_async}"
+        @buffer << "alias :#{name} :#{options[:sync] ? name_sync : name_async}"
+        
+        if options[:alias]
+          options[:alias].each do |alias_name|
+            @buffer << "alias :#{alias_name} :#{name}"
+          end
+        end
+      end
+      
+      def generate
+        dir=File.join(File.absolute_path(File.dirname(__FILE__)),'metadata')
+        FileUtils.mkdir_p(dir)
+        
+        path=File.join(dir, "#{@class_name}.rb")
+        f=File.open(path,'w')
+        f.write(@buffer.join("\n"))
+        f.close
+        #require path   
+      end
+    end
+
+  end
+end
+
+module EventMachine
+  module Protocols
+    module Redis
+    RedisMethodFactory.new(self) do |m|
+      #m.method_alias :set         , {:sync=>false, :args_count=>2}
+      #m.method_alias :sort        , {:sync=>true}
+      m.method_alias :incr        , {:sync=>true, :alias=>[:increment], :args_count=>1}
+      m.method_alias :decr        , {:sync=>false, :alias=>[:decrement], :args_count=>1}
+      m.method_alias :mapped_mget , {:sync=>true, :args_count=>999999}	  
+      m.method_alias :quit        , {:sync=>false}
+      m.method_alias :type        , {:sync=>true, :alias=>[:type?], :args_count=>1}
+      
+      m.method_call :keys         , {:sync=>true, :args_count=>1}
+      m.method_call :get          , {:sync=>true}
+      m.method_call :getset       , {:sync=>true, :args_count=>2}
+      #m.method_call :incrby       , {:sync=>true, :args_count=>2}
+      m.method_call :select       , {:sync=>false, :args_count=>1}
+      m.method_call :scard        , {:sync=>true, :alias=>[:set_count], :args_count=>1}      
+      m.method_call :smembers     , {:sync=>true, :alias=>[:set_member?], :args_count=>1}      
+      m.method_call :rename       , {:sync=>false, :args_count=>2}
+      m.method_call :flushall     , {:sync=>false, :alias=>[:flush_all], :args_count=>0}
+      m.method_call :flushdb      , {:sync=>false, :alias=>[:flush_db, :flush_db!], :args_count=>0}
+      m.method_call :srem         , {:sync=>false, :alias=>[:set_delete, :set_remove, :set_del], :args_count=>2}
+      m.method_call :sadd         , {:sync=>false, :alias=>[:set_add], :args_count=>2}
+      
+      m.method_call :smembers     , {:sync=>true, :alias=>[:members, :set_members], :args_count=>1}
+      m.method_call :srandmember  , {:sync=>true, :args_count=>1}      
+      
+      m.method_call :sismember    , {:sync=>true, :alias=>[:set_member?], :args_count=>2}
+      m.method_call :exists       , {:sync=>true, :alias=>[:exists?], :args_count=>1}
+      m.method_call :version      , {:sync=>true, :args_count=>0, :cache=>true}
+      
+      m.method_call :hset      , {:sync=>true, :args_count=>3}    
+      
+      [:multi, :exec, :discard].each do |name|
+        m.method_call name, {:sync=>false, :args_count=>0}   
+      end      
+      
+    end  
+    	
+    	
+      MULTI_BULK_COMMANDS['hset']=true
+      MULTI_BULK_COMMANDS['hdel']=true
+      
+      def bulk_array
+        @bulk_array||=[]
+      end
+      
+      def delete_array
+        @delete_array||=[]
+      end
+      
+      def bulk_expire_array
+        @bulk_expire_array||=[]
+      end
+      
+      def bulk_insert?
+        !@bulk_insert.nil?
+      end
+      
+=begin      
+      Добавление всех данных в одной транзакции
+      Redis send:
+        MULTI
+        COMMAND_1 ...
+        COMMAND_2 ...
+        COMMAND_N ...
+        EXEC or DISCARD
+=end      
+      def transaction
+        return unless block_given?
+        begin
+          multi
+          yield
+          exec
+        rescue Exception => e
+          discard 
+          raise e          
+        end
+      end
+      
+      def bulk_set
+        bulk_insert=@bulk_insert.nil?
+        begin
+          @bulk_insert=true
+          yield if block_given?
+          if bulk_insert
+            @bulk_insert=nil
+            del_real(delete_array)
+            mset_real(bulk_array)          
+            bulk_expire_array.each do |value|                
+              call_command(value) { |a| }
+            end
+          end
+        ensure
+          if bulk_insert             
+            delete_array.clear
+            bulk_array.clear
+            bulk_expire_array.clear
+          end
+        end
+      end
+      
+      #set
+      
+      def set_async(key, value)
+        if bulk_insert?
+          bulk_array << key
+          bulk_array << value
+        else
+          call_command(['set', key, value]) { |a| }
+        end
+      end            
+      
+      alias :set! :set_async
+      alias :set :set_async
+      
+      def mset_real(args)      
+        argv=args.flatten
+        call_command(argv.unshift('mset')) { |a| } unless argv.empty?   
+      end
+      
+      def mset_async(*args)
+        if bulk_insert?
+          bulk_array+=args          
+        else
+          mset_real(args)          
+        end
+      end            
+      
+      alias :mset! :mset_async
+      alias :mset :mset_async
+      
+      def expire_async(key, value)
+        if bulk_insert?
+          bulk_expire_array << ['expire', key, value]
+        else          
+          call_command(['expire', key, value]) { |a| }
+        end
+      end            
+      
+      alias :expire! :expire_async
+      alias :expire :expire_async
+      #end set      
+      
+      def del_real(keys)         
+        call_command(keys.unshift('del')) { |a| } unless keys.empty?        
+      end
+      
+      def del_async(*agrs)
+        values=agrs.flatten
+        if bulk_insert?
+          delete_array += values
+        else         
+          del_real(values)
+        end
+      end            
+      
+      alias :del! :del_async
+      alias :del :del_async
+      alias :delete :del_async
+      
+      def mget(keys)
+        return [] if keys.blank?
+        #keys.flatten!
+        #return [] if keys.empty?
+        #$log.debug("Redis"){"call mget (#{args.inspect})"} if $debug
+        
+        c=Fiber.current
+        call_command(keys.flatten.unshift('mget')) { |a|  c.resume(a); }
+        Fiber.yield
+      end
+      
+      def set_remove_if_empty(key_name, value)
+        set_remove(key_name, value)
+        del(key_name) if scard(key_name)==0
+      end
+      
+      def method_missing(*argv)
+        name=argv[0].to_s
+        p "call #{name} (#{argv[1..argv.length]})"
+        if $debug  && false
+          p "call #{name} (#{argv[1..argv.length]})" 
+          #pp Kernel.caller
+          #$log.debug("Redis"){"call #{name} (#{argv[1..argv.length]})"}
+          argv[0]=(name=name[0,name.length-1]) if name.end_with?('!')
+        end
+        if name.end_with?('!')
+          argv[0]=name[0,name.length-1]         
+          call_command(argv) {|a| }
+        else
+          c=Fiber.current
+          call_command(argv) { |a|  c.resume(a); }
+          Fiber.yield
+        end      
+      end
+      
+      def support_mset?    
+        @support_mset||=version >= "1.1" 
+      end
+      
+      def raw_call_command(args, &blk)
+        argv = args.flatten.map{|v| v.to_s}
+        
+        if MULTI_BULK_COMMANDS[argv.first]
+          command = "*#{argv.size}\r\n"
+          argv.each do |v|
+            command << "$#{get_size(v)}\r\n"
+            command << "#{v}\r\n"
+          end          
+        else
+          name = argv[0].downcase
+          argv[0] = (ALIASES[name] || name)
+          raise "#{name} command is disabled" if DISABLED_COMMANDS[name]
+          if argv.length > 2 and BULK_COMMANDS[name]
+            bulk=argv[-1]
+            argv[-1]=get_size(bulk)
+          end
+          command = "#{argv.join(' ')}\r\n"
+          command << "#{bulk}\r\n" if bulk
+        end
+        
+        puts "*** sending: #{command}" if $debug
+        @redis_callbacks << [REPLY_PROCESSOR[argv[0]], blk]
+        send_data command
+      end
+      
+      
+      def process_cmd(line)
+        puts "*** processing #{line}" if $debug
+        # first character of buffer will always be the response type
+        reply_type = line[0, 1]
+        reply_args = line.slice(1..-3) # remove type character and \r\n
+        case reply_type
+          
+          #e.g. -MISSING
+          when MINUS
+          @redis_callbacks.shift # throw away the cb?
+          if @err_cb
+            @err_cb.call(reply_args)
+          else
+            err = RedisError.new
+            err.code = reply_args
+            raise err, "Redis server returned error code: #{err.code}"
+          end
+          
+          # e.g. +OK
+          when PLUS
+          dispatch_response(reply_args)
+          
+          # e.g. $3\r\nabc\r\n
+          # 'bulk' is more complex because it could be part of multi-bulk
+          when DOLLAR
+          data_len = Integer(reply_args)
+          if data_len == -1 # expect no data; return nil
+            if @multibulk_n > 0 # we're in the middle of a multibulk reply
+              @values << nil
+              if @values.size == @multibulk_n # DING, we're done
+                dispatch_response(@values)
+                @values = []
+                @multibulk_n = 0
+              end
+            else
+              dispatch_response(nil)
+            end
+          elsif @buffer.size >= data_len + 2 # buffer is full of expected data
+            if @multibulk_n > 0 # we're in the middle of a multibulk reply
+              @values << @buffer.slice!(0, data_len)
+              if @values.size == @multibulk_n # DING, we're done
+                dispatch_response(@values)
+                @values = []
+                @multibulk_n = 0
+              end
+            else # not multibulk
+              value = @buffer.slice!(0, data_len)
+              dispatch_response(value)
+            end
+            @buffer.slice!(0,2) # tossing \r\n
+          else # buffer isn't full or nil
+            # FYI, ParseError puts command back on head of buffer, waits for
+            # more data complete buffer
+            raise ParserError 
+          end
+          
+          #e.g. :8
+          when COLON
+          dispatch_response(Integer(reply_args))
+          
+          #e.g. *2\r\n$1\r\na\r\n$1\r\nb\r\n 
+          when ASTERISK
+          @multibulk_n = Integer(reply_args)
+          dispatch_response(nil) if @multibulk_n == -1 || @multibulk_n == 0
+          
+          # Whu?
+        else
+          raise ProtocolError, "reply type not recognized: #{line.strip}"
+        end
+      end
+    end
+  end
+end