fiveruns-starling 0.9.7.5

data/lib/starling/client.rb

@@ -0,0 +1,151 @@
+require 'rubygems'
+require 'starling'
+require 'starling/worker'
+require 'logger'
+require 'eventmachine'
+require 'analyzer_tools/syslog_logger'
+
+module StarlingClient
+
+  VERSION = "0.9.7.5"
+  
+  class Base
+    attr_reader :logger
+
+    DEFAULT_HOST            = "localhost"
+    DEFAULT_PORT            = "22122"
+    DEFAULT_TEMPLATES_PATH  = File.join("tmp", "starling", "templates")
+    DEFAULT_WORKERS_PATH    = File.join("tmp", "starling", "workers")
+    DEFAULT_TIMEOUT         = 60
+
+    ##
+    # Initialize a new Starling client and immediately start processing
+    # requests.
+    #
+    # +opts+ is an optional hash, whose valid options are:
+    #
+    #   [:host]     Host on which to listen (default is 127.0.0.1).
+    #   [:port]     Port on which to listen (default is 22122).
+    #   [:path]     Path to Starling queue logs. Default is /tmp/starling/
+    #   [:timeout]  Time in seconds to wait before closing connections.
+    #   [:logger]   A Logger object, an IO handle, or a path to the log.
+    #   [:loglevel] Logger verbosity. Default is Logger::ERROR.
+    #
+    # Other options are ignored.
+
+    def self.start(opts = {})
+      server = self.new(opts)
+      server.run
+    end
+
+    ##
+    # Initialize a new Starling client, but do not start with working
+    #
+    # +opts+ is as for +start+
+
+    def initialize(opts = {})
+      @opts = {
+        :host           => DEFAULT_HOST,
+        :port           => DEFAULT_PORT,
+        :templates_path => DEFAULT_TEMPLATES_PATH,
+        :workers_path   => DEFAULT_WORKERS_PATH,
+        :timeout        => DEFAULT_TIMEOUT,
+        :continues_processing => true
+      }.merge(opts)
+
+      @stats = Hash.new(0)
+
+      FileUtils.mkdir_p(@opts[:templates_path])
+      FileUtils.mkdir_p(@opts[:workers_path])
+      
+      @client = Starling.new("#{@opts[:host]}:#{@opts[:port]}", :multithread => true)
+    end
+
+    ##
+    # Start listening and processing requests.
+
+    def run
+      @stats[:start_time] = Time.now
+
+      @@logger = case @opts[:logger]
+                 when IO, String; Logger.new(@opts[:logger])
+                 when Logger; @opts[:logger]
+                 else; Logger.new(STDERR)
+                 end
+      @@logger = SyslogLogger.new(@opts[:syslog_channel]) if @opts[:syslog_channel]
+
+      @@logger.level = @opts[:log_level] || Logger::ERROR
+
+      @@logger.info "Starling Client STARTUP"
+      
+      load_templates
+      
+      @pids = []
+      
+      load_workers.each do |worker|
+        @pids << fork {
+          worker = StarlingWorker.const_get(worker).new(:host => @opts[:host],
+                                                        :port => @opts[:port]).run
+          @@logger.info "Starling Client STARTUP"
+        }
+      end
+      
+      if load_workers.length == 0
+        @@logger.error "no workers found"
+        @shutdown = true
+      end
+      
+      loop {
+        sleep 60
+      } if @opts[:continues_processing]
+    end
+    
+    def stop
+      @pids.each do |pid|
+        Process.kill(0, pid)
+      end
+      
+      exit(0)
+    end
+    
+    def load_templates
+      templates = []
+      Dir.glob("#{@opts[:templates_path]}/*.rb").each do |file|
+        unless [".", ".."].include?(file)
+          load(file) 
+          templates << File.basename(file, ".rb").split('_').map{|w| w.capitalize}.join
+        end
+      end
+      
+      return templates
+    end
+    
+    def load_workers
+      workers = []
+      Dir.glob("#{@opts[:workers_path]}/*.rb").each do |file|
+        unless [".", ".."].include?(file)
+          load(file) 
+          workers << File.basename(file, ".rb").split('_').map{|w| w.capitalize}.join
+        end
+      end
+      
+      return workers
+    end
+    
+    def starling
+      return @client
+    end
+
+    def self.logger
+      @@logger
+    end
+
+    def stats(stat = nil) #:nodoc:
+      case stat
+      when nil; @stats
+      when :connections; 1
+      else; @stats[stat]
+      end
+    end
+  end
+end

data/lib/starling/client_runner.rb

@@ -0,0 +1,272 @@
+require File.join(File.dirname(__FILE__), 'client')
+require 'optparse'
+require 'yaml'
+
+module StarlingClient
+  class Runner
+
+    attr_accessor :options
+    private :options, :options=
+
+    def self.run
+      new
+    end
+    
+    def self.shutdown
+      @@instance.shutdown
+    end
+
+    def initialize
+      @@instance = self
+      parse_options
+
+      @process = ProcessHelper.new(options[:logger], options[:pid_file], options[:user], options[:group])
+
+      pid = @process.running?
+      if pid
+        STDERR.puts "There is already a Starling Client process running (pid #{pid}), exiting."
+        exit(1)
+      elsif pid.nil?
+        STDERR.puts "Cleaning up stale pidfile at #{options[:pid_file]}."
+      end
+
+      start
+    end
+
+    def load_config_file(filename)
+      YAML.load(File.open(filename))['starling'].each do |key, value|
+        # alias some keys
+        case key
+        when "queue_path" then key = "path"
+        when "log_file" then key = "logger"
+        end
+        options[key.to_sym] = value
+
+        if options[:log_level].instance_of?(String)
+          options[:log_level] = Logger.const_get(options[:log_level])
+        end
+      end
+    end
+
+    def parse_options
+      self.options = { :path => File.join(%w( / tmp workers )),
+                       :log_level => Logger::INFO,
+                       :daemonize => false,
+                       :timeout => 0,
+                       :pid_file => File.join(%w( / tmp starling starling_client.pid )) }
+
+      OptionParser.new do |opts|
+        opts.summary_width = 25
+
+        opts.banner = "Starling (#{StarlingClient::VERSION})\n\n",
+                      "usage: starling_client [options...]\n",
+                      "       starling_client --help\n",
+                      "       starling_client --version\n"
+
+        opts.separator ""
+        opts.separator "Configuration:"
+
+        opts.on("-f", "--config FILENAME",
+                "Config file (yaml) to load") do |filename|
+          load_config_file(filename)
+        end
+        
+        opts.on("-w", "--workers_path PATH",
+                :REQUIRED,
+                "Path to workers") do |queue_path|
+          options[:workers_path] = queue_path
+        end
+        
+        opts.on("-t", "--templates_path PATH",
+                :REQUIRED,
+                "Path to templates") do |queue_path|
+          options[:templates_path] = queue_path
+        end
+
+        opts.separator ""; opts.separator "Process:"
+
+        opts.on("-d", "Run as a daemon.") do
+          options[:daemonize] = true
+        end
+
+        opts.on("-PFILE", "--pid FILENAME", "save PID in FILENAME when using -d option.", "(default: #{options[:pid_file]})") do |pid_file|
+          options[:pid_file] = pid_file
+        end
+
+        opts.separator ""; opts.separator "Logging:"
+
+        opts.on("-L", "--log [FILE]", "Path to print debugging information.") do |log_path|
+          options[:logger] = log_path
+        end
+
+        opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
+          options[:syslog_channel] = channel
+        end
+
+        opts.on("-v", "Increase logging verbosity (may be used multiple times).") do
+          options[:log_level] -= 1
+        end
+
+        opts.separator ""; opts.separator "Miscellaneous:"
+
+        opts.on_tail("-?", "--help", "Display this usage information.") do
+          puts "#{opts}\n"
+          exit
+        end
+
+        opts.on_tail("-V", "--version", "Print version number and exit.") do
+          puts "Starling Client #{StarlingClient::VERSION}\n\n"
+          exit
+        end
+      end.parse!
+    end
+
+    def start
+      drop_privileges
+
+      @process.daemonize if options[:daemonize]
+
+      setup_signal_traps
+      @process.write_pid_file
+
+      STDOUT.puts "Starting."
+      @client = StarlingClient::Base.new(options)
+      @client.run
+
+      @process.remove_pid_file
+    end
+
+    def drop_privileges
+      Process.euid = options[:user] if options[:user]
+      Process.egid = options[:group] if options[:group]
+    end
+
+    def shutdown
+      begin
+        STDOUT.puts "Shutting down."
+        StarlingClient::Base.logger.info "Shutting down."
+        @client.stop
+      rescue Object => e
+        STDERR.puts "There was an error shutting down: #{e}"
+        exit(70)
+      end
+    end
+
+    def setup_signal_traps
+      Signal.trap("INT") { shutdown }
+      Signal.trap("TERM") { shutdown }
+    end
+  end
+
+  class ProcessHelper
+
+    def initialize(log_file = nil, pid_file = nil, user = nil, group = nil)
+      @log_file = log_file
+      @pid_file = pid_file
+      @user = user
+      @group = group
+    end
+
+    def safefork
+      begin
+        if pid = fork
+          return pid
+        end
+      rescue Errno::EWOULDBLOCK
+        sleep 5
+        retry
+      end
+    end
+
+    def daemonize
+      sess_id = detach_from_terminal
+      exit if pid = safefork
+
+      Dir.chdir("/")
+      File.umask 0000
+
+      close_io_handles
+      redirect_io
+
+      return sess_id
+    end
+
+    def detach_from_terminal
+      srand
+      safefork and exit
+
+      unless sess_id = Process.setsid
+        raise "Couldn't detache from controlling terminal."
+      end
+
+      trap 'SIGHUP', 'IGNORE'
+
+      sess_id
+    end
+
+    def close_io_handles
+      ObjectSpace.each_object(IO) do |io|
+        unless [STDIN, STDOUT, STDERR].include?(io)
+          begin
+            io.close unless io.closed?
+          rescue Exception
+          end
+        end
+      end
+    end
+
+    def redirect_io
+      begin; STDIN.reopen('/dev/null'); rescue Exception; end
+
+      if @log_file
+        begin
+          STDOUT.reopen(@log_file, "a")
+          STDOUT.sync = true
+        rescue Exception
+          begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+        end
+      else
+        begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+      end
+
+      begin; STDERR.reopen(STDOUT); rescue Exception; end
+      STDERR.sync = true
+    end
+
+    def rescue_exception
+      begin
+        yield
+      rescue Exception
+      end
+    end
+
+    def write_pid_file
+      return unless @pid_file
+      FileUtils.mkdir_p(File.dirname(@pid_file))
+      File.open(@pid_file, "w") { |f| f.write(Process.pid) }
+      File.chmod(0644, @pid_file)
+    end
+
+    def remove_pid_file
+      return unless @pid_file
+      File.unlink(@pid_file) if File.exists?(@pid_file)
+    end
+
+    def running?
+      return false unless @pid_file
+
+      pid = File.read(@pid_file).chomp.to_i rescue nil
+      pid = nil if pid == 0
+      return false unless pid
+
+      begin
+        Process.kill(0, pid)
+        return pid
+      rescue Errno::ESRCH
+        return nil
+      rescue Errno::EPERM
+        return pid
+      end
+    end
+  end
+end

data/lib/starling/handler.rb

@@ -0,0 +1,222 @@
+module StarlingServer
+
+  ##
+  # This is an internal class that's used by Starling::Server to handle the
+  # MemCache protocol and act as an interface between the Server and the
+  # QueueCollection.
+
+  class Handler < EventMachine::Connection
+
+    DATA_PACK_FMT = "Ia*".freeze
+
+    # ERROR responses
+    ERR_UNKNOWN_COMMAND = "CLIENT_ERROR bad command line format\r\n".freeze
+
+    # GET Responses
+    GET_COMMAND = /\Aget (.{1,250})\s*\r\n/m
+    GET_RESPONSE       = "VALUE %s %s %s\r\n%s\r\nEND\r\n".freeze
+    GET_RESPONSE_EMPTY = "END\r\n".freeze
+
+    # SET Responses
+    SET_COMMAND = /\Aset (.{1,250}) ([0-9]+) ([0-9]+) ([0-9]+)\r\n/m
+    SET_RESPONSE_SUCCESS  = "STORED\r\n".freeze
+    SET_RESPONSE_FAILURE  = "NOT STORED\r\n".freeze
+    SET_CLIENT_DATA_ERROR = "CLIENT_ERROR bad data chunk\r\nERROR\r\n".freeze
+
+    # STAT Response
+    STATS_COMMAND = /\Astats\r\n/m
+    STATS_RESPONSE = "STAT pid %d
+STAT uptime %d
+STAT time %d
+STAT version %s
+STAT rusage_user %0.6f
+STAT rusage_system %0.6f
+STAT curr_items %d
+STAT total_items %d
+STAT bytes %d
+STAT curr_connections %d
+STAT total_connections %d
+STAT cmd_get %d
+STAT cmd_set %d
+STAT get_hits %d
+STAT get_misses %d
+STAT bytes_read %d
+STAT bytes_written %d
+STAT limit_maxbytes %d
+%sEND\r\n".freeze
+    QUEUE_STATS_RESPONSE = "STAT queue_%s_items %d
+STAT queue_%s_total_items %d
+STAT queue_%s_logsize %d
+STAT queue_%s_expired_items %d
+STAT queue_%s_age %d\n".freeze
+
+    SHUTDOWN_COMMAND = /\Ashutdown\r\n/m
+
+
+    @@next_session_id = 1
+
+    ##
+    # Creates a new handler for the MemCache protocol that communicates with a
+    # given client.
+
+    def initialize(options = {})
+      @opts = options
+    end
+
+    ##
+    # Process incoming commands from the attached client.
+
+    def post_init
+      @stash = []
+      @data = ""
+      @data_buf = ""
+      @server = @opts[:server]
+      @logger = StarlingServer::Base.logger
+      @expiry_stats = Hash.new(0)
+      @expected_length = nil
+      @server.stats[:total_connections] += 1
+      set_comm_inactivity_timeout @opts[:timeout]
+      @queue_collection = @opts[:queue]
+
+      @session_id = @@next_session_id
+      @@next_session_id += 1
+
+      peer = Socket.unpack_sockaddr_in(get_peername)
+      #@logger.debug "(#{@session_id}) New session from #{peer[1]}:#{peer[0]}"
+    end
+
+    def receive_data(incoming)
+      @server.stats[:bytes_read] += incoming.size
+      @data << incoming
+
+      while data = @data.slice!(/.*?\r\n/m)
+        response = process(data)
+      end
+
+      send_data response if response
+    end
+
+    def process(data)
+      data = @data_buf + data if @data_buf.size > 0
+      # our only non-normal state is consuming an object's data
+      # when @expected_length is present
+      if @expected_length && data.size == @expected_length
+        response = set_data(data)
+        @data_buf = ""
+        return response
+      elsif @expected_length
+        @data_buf = data
+        return
+      end
+
+      case data
+      when SET_COMMAND
+        @server.stats[:set_requests] += 1
+        set($1, $2, $3, $4.to_i)
+      when GET_COMMAND
+        @server.stats[:get_requests] += 1
+        get($1)
+      when STATS_COMMAND
+        stats
+      when SHUTDOWN_COMMAND
+        # no point in responding, they'll never get it.
+        Runner::shutdown
+      else
+        logger.warn "Unknown command: #{data}."
+        respond ERR_UNKNOWN_COMMAND
+      end
+    rescue => e
+      logger.error "Error handling request: #{e}."
+      logger.debug e.backtrace.join("\n")
+      respond GET_RESPONSE_EMPTY
+    end
+
+    def unbind
+      #@logger.debug "(#{@session_id}) connection ends"
+    end
+
+  private
+    def respond(str, *args)
+      response = sprintf(str, *args)
+      @server.stats[:bytes_written] += response.length
+      response
+    end
+
+    def set(key, flags, expiry, len)
+      @expected_length = len + 2
+      @stash = [ key, flags, expiry ]
+      nil
+    end
+
+    def set_data(incoming)
+      key, flags, expiry = @stash
+      data = incoming.slice(0...@expected_length-2)
+      @stash = []
+      @expected_length = nil
+
+      internal_data = [expiry.to_i, data].pack(DATA_PACK_FMT)
+      if @queue_collection.put(key, internal_data)
+        respond SET_RESPONSE_SUCCESS
+      else
+        respond SET_RESPONSE_FAILURE
+      end
+    end
+
+    def get(key)
+      now = Time.now.to_i
+
+      while response = @queue_collection.take(key)
+        expiry, data = response.unpack(DATA_PACK_FMT)
+
+        break if expiry == 0 || expiry >= now
+
+        @expiry_stats[key] += 1
+        expiry, data = nil
+      end
+
+      if data
+        respond GET_RESPONSE, key, 0, data.size, data
+      else
+        respond GET_RESPONSE_EMPTY
+      end
+    end
+
+    def stats
+      respond STATS_RESPONSE,
+        Process.pid, # pid
+        Time.now - @server.stats(:start_time), # uptime
+        Time.now.to_i, # time
+        StarlingServer::VERSION, # version
+        Process.times.utime, # rusage_user
+        Process.times.stime, # rusage_system
+        @queue_collection.stats(:current_size), # curr_items
+        @queue_collection.stats(:total_items), # total_items
+        @queue_collection.stats(:current_bytes), # bytes
+        @server.stats(:connections), # curr_connections
+        @server.stats(:total_connections), # total_connections
+        @server.stats(:get_requests), # get count
+        @server.stats(:set_requests), # set count
+        @queue_collection.stats(:get_hits),
+        @queue_collection.stats(:get_misses),
+        @server.stats(:bytes_read), # total bytes read
+        @server.stats(:bytes_written), # total bytes written
+        0, # limit_maxbytes
+        queue_stats
+    end
+
+    def queue_stats
+      @queue_collection.queues.inject("") do |m,(k,v)|
+        m + sprintf(QUEUE_STATS_RESPONSE,
+                      k, v.length,
+                      k, v.total_items,
+                      k, v.logsize,
+                      k, @expiry_stats[k],
+                      k, v.current_age)
+      end
+    end
+
+    def logger
+      @logger
+    end
+  end
+end