rdkit 0.0.1 → 0.1.4

data/lib/rdkit/rd_object.rb

@@ -0,0 +1,69 @@
+module RDKit
+  class RDObject
+    attr_accessor :type, :encoding, :value
+
+    def self.forward_to_value(*methods)
+      @forwarded_methods = methods
+    end
+
+    def method_missing(method, *args)
+      if forwarded_methods.include?(method)
+        value.__send__(method, *args)
+      else
+        super
+      end
+    end
+
+    private
+
+    def forwarded_methods
+      self.class.instance_variable_get(:@forwarded_methods) || []
+    end
+
+    module ClassMethods
+      def string(value)
+        new.tap do |object|
+          object.type = :string
+          object.value = value
+        end
+      end
+
+      def list(elements)
+        RDList.new.tap do |object|
+          object.type = :list
+          object.value = elements
+        end
+      end
+
+      def set(elements)
+        require "set"
+
+        RDSet.new.tap do |set|
+          set.type = :set
+          set.value = Set.new(elements)
+        end
+      end
+
+      def create_hash(key, value)
+        RDHash.new.tap do |hash|
+          hash.type  = :hash
+          hash.value = { key => value }
+        end
+      end
+    end
+
+    class << self; include ClassMethods; end
+  end
+
+  class RDList < RDObject
+    forward_to_value :unshift, :length, :shift, :empty?, :pop
+  end
+
+  class RDSet < RDObject
+    forward_to_value :add, :size, :to_a, :include?, :delete
+  end
+
+  class RDHash < RDObject
+    forward_to_value :has_key?, :[]=, :[], :size, :delete, :keys, :values
+  end
+end

data/lib/rdkit/resp.rb

@@ -5,6 +5,12 @@ module RDKit
     module ClassMethods
       def compose(data)
         case data
+        when *%w{ OK string list set hash zset none }
+          "+#{data}\r\n"
+        when true
+          ":1\r\n"
+        when false
+          ":0\r\n"
         when Integer
           ":#{data}\r\n"
         when Array
@@ -12,8 +18,10 @@ module RDKit
         when NilClass
           # Null Bulk String, not Null Array of "*-1\r\n"
           "$-1\r\n"
+        when WrongTypeError
+          "-WRONGTYPE #{data.message}\r\n"
         when StandardError
-          "-#{data.message}\r\n"
+          "-ERR #{data.message}\r\n"
         else
           # always Bulk String
           "$#{data.bytesize}\r\n#{data}\r\n"

data/lib/rdkit/{command_parser.rb → resp_parser.rb}

@@ -3,43 +3,31 @@ require "hiredis/reader"
 # Hiredis::Reader does not handle inline commands, so
 
 module RDKit
-  class CommandParser
+  class RESPParser
     def initialize
       @reader = Hiredis::Reader.new
       @buffer = []
       @regexp = Regexp.new("\\A(.+)\\r\\n\\z")
-      @error  = nil
+      @inline_mode = true
     end
 
     def feed(data)
-      if data =~ @regexp
+      if @inline_mode && (data =~ @regexp)
         @buffer << $1.split
       else
-        @reader.feed(data)
+        @inline_mode = false
 
-        read_into_buffer!
+        @reader.feed(data)
       end
     end
 
     def gets
-      raise @error unless @error.nil?
-
-      if result = @buffer.shift
+      if @inline_mode && (result = @buffer.shift)
 
         result
       else
         @reader.gets
       end
     end
-
-    private
-
-    def read_into_buffer!
-      until (reply = @reader.gets) == false
-        @buffer << reply
-      end
-    rescue RuntimeError => e
-      @error = ProtocolError.new(e) if e.message =~ /Protocol error/
-    end
   end
 end

data/lib/rdkit/resp_responder.rb

@@ -0,0 +1,105 @@
+module RDKit
+  class RESPResponder
+    def run(cmd)
+      if cmd.respond_to?(:call)
+        RESP.compose(cmd.call)
+      else
+        RESP.compose(call(cmd))
+      end
+    rescue StandardError => e
+      RESP.compose(e)
+    end
+
+    include SimpleCommands
+    include DBCommands
+
+    # 获取服务器状态
+    def info(section='default')
+      info = Introspection.info(section)
+
+      unless info.empty?
+        info.map do |type, value|
+          "# #{type.capitalize}\r\n" + value.map { |k, v| "#{k}:#{v}" }.join("\r\n") + "\r\n"
+        end.join("\r\n") + "\r\n"
+      end
+    end
+
+    def gc
+      GC.start
+
+      'OK'
+    end
+
+    def heapdump
+      require "objspace"
+
+      ObjectSpace.trace_object_allocations_start
+
+      GC.start
+
+      file = "tmp/heap-#{Process.pid}-#{Time.now.to_i}.json"
+
+      ObjectSpace.dump_all(output: File.open(file, "w"))
+
+      file
+    end
+
+    def monitor
+      server.monitors << server.current_client
+
+      'OK'
+    end
+
+    def shutdown
+      server.stop
+    end
+
+    include Subcommands
+    def config(cmd, *args)
+      execute_subcommand('config', %w{ get set resetstat }, cmd, *args)
+    end
+
+    def slowlog(cmd, *args)
+      execute_subcommand('slowlog', %w{ get reset len }, cmd, *args)
+    end
+
+    def client(cmd, *args)
+      execute_subcommand('client', %w{ list kill getname setname }, cmd, *args)
+    end
+
+    def debug(cmd, *args)
+      execute_subcommand('debug', %w{ sleep segfault }, cmd, *args)
+    end
+
+    def server
+      Server.instance
+    end
+
+    private
+
+    def call(cmd)
+      @logger ||= Logger.new
+
+      Introspection::Stats.incr(:total_commands_processed)
+
+      @logger.debug "running command: #{cmd}"
+      cmd, *args = cmd
+
+      cmd.downcase!
+
+      if self.respond_to?(cmd)
+        self.__send__(cmd, *args)
+      else
+        raise UnknownCommandError, "unknown command '#{cmd}'"
+      end
+    rescue ArgumentError => e
+      raise WrongNumberOfArgumentError, "wrong number of arguments for '#{cmd}' command"
+    end
+  end
+
+  class RESPRunner < RESPResponder
+    def self.inherited(base)
+      $stderr.puts "RESPRunner is deprecated, use RESPResponder instead"
+    end
+  end
+end

data/lib/rdkit/server.rb

@@ -1,61 +1,153 @@
-require 'newrelic_rpm'
+require "sigdump/setup"
+require 'thread/pool'
 
 module RDKit
   class Server
     HZ = 10
-    CYCLES_TIL_MEMORY_RESAMPLE = 1000
+
+    HANDLED_SIGNALS = [ :TERM, :INT, :HUP ]
 
     attr_reader :server_up_since
-    attr_reader :runner
+    attr_reader :current_client
+    attr_reader :current_db
     attr_reader :core
     attr_reader :host, :port
+    attr_reader :logger
+    attr_reader :monitors
+    attr_reader :cycles
+    attr_accessor :parser_class
+
+    def responder
+      @responder ||= (( @runner && $stderr.puts("@runner is deprecated, use @responder instead") ) || @runner)
+    end
 
     def initialize(host, port)
       @host, @port = host, port
 
       @cycles = 0
-      @peak_memory = 0
       @peak_connected_clients = 0
+      @client_id_seq = 0
 
-      @clients, @command_parsers = Hash.new, Hash.new
+      @clients = Hash.new
+      @blocked_clients = Hash.new
+      @monitors = []
 
-      @logger = Logger.new
+      @logger = Logger.new(ENV['RDKIT_LOG_PATH'])
+      @current_db = DB.new(0)
+      @all_dbs = [@current_db]
 
       Introspection.register(self)
 
       @server_up_since = Time.now
-    end
 
-    def sanity_check!
-      unless @host && @port
-        raise SDKRequirementNotMetError, '@host and @port are required for server to run'
-      end
+      @parser_class = RESPParser
 
-      if @core.nil?
-        raise SDKRequirementNotMetError, '@core is required to represent your business logics'
-      end
+      register_notification_observers!
 
-      if @runner.nil?
-        raise SDKRequirementNotMetError, '@runner is required to act as an RESP frontend'
-      end
+      Server.register(self)
+
+      # Self-pipe for deferred signal-handling http://www.sitepoint.com/the-self-pipe-trick-explained/
+      # Borrowed from `Foreman::Engine`
+      reader, writer = create_pipe
+      @selfpipe      = { :reader => reader, :writer => writer }
+      @signal_queue  = []
     end
 
     def start
       sanity_check!
 
+      register_signal_handlers
+
       @server_socket = TCPServer.new(@host, @port)
 
       run_acceptor
     end
 
     def stop
-      @logger.warn "signal caught, shutting down..."
+      @logger.warn "shutting down..."
       exit
     end
 
+    def create_pipe
+      IO.method(:pipe).arity.zero? ? IO.pipe : IO.pipe("BINARY")
+    end
+
+    def register_signal_handlers
+      HANDLED_SIGNALS.each do |sig|
+        if ::Signal.list.include? sig.to_s
+          trap(sig) { @signal_queue << sig ; notice_signal }
+        end
+      end
+    end
+
+    def notice_signal
+      @selfpipe[:writer].write_nonblock('.')
+    rescue Errno::EAGAIN
+      # Ignore writes that would block
+    rescue Errno::EINT
+      # Retry if another signal arrived while writing
+      retry
+    end
+
+    def handle_signals
+      while sig = @signal_queue.shift
+        handle_signal(sig)
+      end
+    end
+
+    # Invoke the real handler for signal +sig+. This shouldn't be called directly
+    # by signal handlers, as it might invoke code which isn't re-entrant.
+    #
+    # @param [Symbol] sig  the name of the signal to be handled
+    #
+    def handle_signal(sig)
+      case sig
+      when :TERM
+        handle_term_signal
+      when :INT
+        handle_interrupt
+      when :HUP
+        handle_hangup
+      else
+        system "unhandled signal #{sig}"
+      end
+    end
+
+    # Handle a TERM signal
+    #
+    def handle_term_signal
+      @logger.warn "SIGTERM received"
+      terminate_gracefully
+    end
+
+    # Handle an INT signal
+    #
+    def handle_interrupt
+      @logger.warn "SIGINT received"
+      terminate_gracefully
+    end
+
+    # Handle a HUP signal
+    #
+    def handle_hangup
+      @logger.warn "SIGHUP received"
+      terminate_gracefully
+    end
+
+    def terminate_gracefully
+      return if @terminating
+
+      @terminating = true
+
+      stop
+    end
+
+    include MemoryMonitoring
+
     def introspection
       {
         server: {
+          ruby_version: "#{RUBY_VERSION}p#{RUBY_PATCHLEVEL}",
           rdkit_version: RDKit::VERSION,
           multiplexing_api: 'select',
           process_id: Process.pid,
@@ -65,6 +157,7 @@ module RDKit
           hz: HZ,
         },
         clients: {
+          blocked_clients: @blocked_clients.size,
           connected_clients: @clients.size,
           connected_clients_peak: @peak_connected_clients
         },
@@ -75,118 +168,181 @@ module RDKit
       }
     end
 
-    private
-
-    def used_memory_rss_in_mb
-      update_peak_memory!
+    def delete(socket)
+      @clients.delete(socket)
+    end
 
-      '%0.2f' % used_memory_rss + 'M'
+    def clients
+      @clients.values
     end
 
-    def used_memory_peak_in_mb
-      '%0.2f' % @peak_memory + 'M'
+    def select_db!(index)
+      if db = @all_dbs.find { |db| db.index == index }
+        @current_db = db
+      else
+        @all_dbs << DB.new(index)
+
+        @current_db = @all_dbs.last
+      end
     end
 
-    def add_client
-      Introspection::Stats.incr(:total_connections_received)
+    def flushdb!
+      @current_db.flush!
+    end
 
-      socket = @server_socket.accept_nonblock
+    def flushall!
+      flushdb!
 
-      @command_parsers[socket] = CommandParser.new
+      @all_dbs = [@current_db]
+    end
 
-      @clients[socket] = Fiber.new do
-         with_error_handling(socket) do |io|
-           loop { process(io); Fiber.yield }
-         end
-      end
+    def blocking(on_success=nil, &block)
+      @blocked_clients[current_client.socket] = current_client
+      @clients.delete(current_client.socket)
 
-      @logger.debug "client #{socket} connected"
+      current_client.blocking(on_success, &block)
+    end
 
-      return @clients[socket]
+    def pool
+      @pool ||= Thread.pool((ENV['RDKIT_SERVER_THREAD_POOL_SIZE'] || 10).to_i)
     end
 
-    def process(io)
-      feed_parser(io)
+    include Callbacks
+
+    private
+
+    def register_notification_observers!
+      if webhook = ENV['RDKIT_SLOW_LOG_BEARYCHAT_WEBHOOK']
+        require "httpi"
+        require "multi_json"
+        HTTPI.logger = Logger.new('/dev/null')
+
+        NotificationCenter.subscribe('slowlog', self) do |cmd, usec|
+          cmd, *args = cmd
 
-      until (reply = get_parser_reply(io)) == false
-        send_response(io, reply)
+          text = "host=#{@host} port=#{@port} cmd=#{cmd}(#{args.join(',') }) usec=#{usec}"
+
+          pool.process { HTTPI.post(webhook, payload: MultiJson.dump({ text: text })) }
+        end
       end
     end
 
-    def feed_parser(io)
-      cmd = io.readpartial(1024)
+    def sanity_check!
+      unless @host && @port
+        raise SDKRequirementNotMetError, '@host and @port are required for server to run'
+      end
+
+      if @core.nil?
+        raise SDKRequirementNotMetError, '@core is required to represent your business logics'
+      end
 
-      @command_parsers[io].feed(cmd)
-    end
+      if responder.nil?
+        raise SDKRequirementNotMetError, '@responder is required to act as an RESP frontend'
+      end
 
-    def get_parser_reply(io)
-      @command_parsers[io].gets
+      if responder.server.nil?
+        raise SDKRequirementNotMetError, '@responder should have reference to server'
+      end
     end
 
-    def send_response(io, cmd)
-      resp = runner.resp(cmd)
+    def add_client
+      Introspection::Stats.incr(:total_connections_received)
 
-      @logger.debug(resp)
+      socket = @server_socket.accept_nonblock
 
-      io.write(resp)
-    end
+      client = @clients[socket] = Client.new(socket, self)
+      client.id = (@client_id_seq += 1)
 
-    def with_error_handling(socket, &block)
+      @logger.debug "client #{socket} connected"
+      client_connected(client)
 
-      block.call(socket)
+      update_peak_connected_clients!
 
-    rescue Errno::ECONNRESET, EOFError => e
-      # client disconnected
-      @logger.debug "client #{socket.inspect} has disconnected"
-      @logger.debug e
-      @command_parsers.delete(socket)
-      @clients.delete(socket)
-    rescue ProtocolError => e
-      # client protocol error, force disconnect
-      @logger.debug "client protocol error"
-      @logger.debug e
+      return @clients[socket]
+    end
+
+    def process(socket)
+      client = @clients[socket]
+      @current_client = client
+      client.resume
+      client_command_processed(client)
+    rescue ClientDisconnectedError => e
       socket.close
-      @command_parsers.delete(socket)
-      @clients.delete(socket)
+      @monitors.delete(client)
+      delete(socket)
+      client_disconnected(client)
     end
 
     def run_acceptor
-      @logger.info "accepting on shared socket (#{@host}:#{@port})"
+      @logger.info "accepting on shared socket (#{@host}:#{@port}), PID #{Process.pid}"
 
-      loop do
-        readable, _ = IO.select([@server_socket, @clients.keys].flatten, nil, nil, 1.0 / HZ)
-
-        if readable
-          readable.each do |socket|
-            if socket == @server_socket
-              add_client
-            else
-              # client is a Fiber
-              client = @clients[socket]
-              client.resume
-            end
-          end
-        end
+      server_started
 
-        update_peak_memory! if @cycles % CYCLES_TIL_MEMORY_RESAMPLE == 0
-        update_peak_connected_clients!
+      loop do
+        process_blocked_clients
+        process_clients
 
         @cycles += 1
 
         core.tick!
+
+        gc_pool.process if @cycles % 1000 == 0
       end
+    rescue Exception => e
+      @logger.warn e unless e.class == SystemExit
+      raise e
     end
 
-    def update_peak_memory!
-      @peak_memory = [@peak_memory, used_memory_rss].max
+    def gc_pool
+      @gc_pool ||= Thread.pool(1) do
+        _, usec = SlowLog.monitor('bg_gc') { GC.start }
+
+        Introspection::Commandstats.record('bg_gc', usec)
+      end
+    end
+
+    def process_blocked_clients
+      @blocked_clients.each do |socket, client|
+        if client.finished?
+          @clients[socket] = client
+          @blocked_clients.delete(socket)
+
+          client.unblock!
+        end
+      end
+    end
+
+    def process_clients
+      readable, _ = IO.select([@server_socket, @selfpipe[:reader], @clients.keys].flatten, nil, nil, 1.0 / HZ)
+
+      if readable
+        readable.each do |socket|
+          if socket == @server_socket
+            add_client
+          elsif socket == @selfpipe[:reader]
+            handle_signals
+          else
+            process(socket)
+          end
+
+          process_blocked_clients
+        end
+      end
     end
 
     def update_peak_connected_clients!
       @peak_connected_clients = [@peak_connected_clients, @clients.size].max
     end
 
-    def used_memory_rss
-      NewRelic::Agent::Samplers::MemorySampler.new.sampler.get_sample
+    module ClassMethods
+      def register(instance)
+        @@instance = instance
+      end
+
+      def instance
+        @@instance
+      end
     end
+    class << self; include ClassMethods; end
   end
 end