mmmurf-starling 0.9.7.10

data/lib/starling/server.rb

@@ -0,0 +1,113 @@
+require 'socket'
+require 'logger'
+require 'rubygems'
+require 'eventmachine'
+require 'analyzer_tools/syslog_logger'
+
+here = File.dirname(__FILE__)
+
+require File.join(here, 'queue_collection')
+require File.join(here, 'handler')
+
+module StarlingServer
+
+  VERSION = "0.9.7.9"
+  
+  class Base
+    attr_reader :logger
+
+    DEFAULT_HOST    = '127.0.0.1'
+    DEFAULT_PORT    = 22122
+    DEFAULT_PATH    = "/tmp/starling/"
+    DEFAULT_TIMEOUT = 60
+
+    ##
+    # Initialize a new Starling server and immediately start processing
+    # requests.
+    #
+    # +opts+ is an optional hash, whose valid options are:
+    #
+    #   [:host]     Host on which to listen (default is 127.0.0.1).
+    #   [:port]     Port on which to listen (default is 22122).
+    #   [:path]     Path to Starling queue logs. Default is /tmp/starling/
+    #   [:timeout]  Time in seconds to wait before closing connections.
+    #   [:logger]   A Logger object, an IO handle, or a path to the log.
+    #   [:loglevel] Logger verbosity. Default is Logger::ERROR.
+    #
+    # Other options are ignored.
+
+    def self.start(opts = {})
+      server = self.new(opts)
+      server.run
+    end
+
+    ##
+    # Initialize a new Starling server, but do not accept connections or
+    # process requests.
+    #
+    # +opts+ is as for +start+
+
+    def initialize(opts = {})
+      @opts = {
+        :host    => DEFAULT_HOST,
+        :port    => DEFAULT_PORT,
+        :path    => DEFAULT_PATH,
+        :timeout => DEFAULT_TIMEOUT,
+        :server  => self
+      }.merge(opts)
+
+      @stats = Hash.new(0)
+
+      FileUtils.mkdir_p(@opts[:path])
+
+    end
+
+    ##
+    # Start listening and processing requests.
+
+    def run
+      @stats[:start_time] = Time.now
+
+      @@logger = case @opts[:logger]
+                 when IO, String; Logger.new(@opts[:logger])
+                 when Logger; @opts[:logger]
+                 else; Logger.new(STDERR)
+                 end
+      @@logger = SyslogLogger.new(@opts[:syslog_channel]) if @opts[:syslog_channel]
+
+      @opts[:queue] = QueueCollection.new(@opts[:path])
+      @@logger.level = @opts[:log_level] || Logger::ERROR
+
+      @@logger.info "Starling STARTUP on #{@opts[:host]}:#{@opts[:port]}"
+
+      EventMachine.epoll
+      EventMachine.set_descriptor_table_size(4096)
+      EventMachine.run do
+        EventMachine.start_server(@opts[:host], @opts[:port], Handler, @opts)
+      end
+
+      # code here will get executed on shutdown:
+      @opts[:queue].close
+    end
+
+    def self.logger
+      @@logger
+    end
+
+
+    ##
+    # Stop accepting new connections and shutdown gracefully.
+
+    def stop
+      EventMachine.stop_event_loop
+    end
+
+    def stats(stat = nil) #:nodoc:
+      case stat
+      when nil; @stats
+      when :connections; 1
+      else; @stats[stat]
+      end
+    end
+  end
+end

data/lib/starling/server_runner.rb

@@ -0,0 +1,292 @@
+require File.join(File.dirname(__FILE__), 'server')
+require 'optparse'
+require 'yaml'
+
+module StarlingServer
+  class Runner
+
+    attr_accessor :options
+    private :options, :options=
+
+    def self.run
+      new
+    end
+    
+    def self.shutdown
+      @@instance.shutdown
+    end
+
+    def initialize
+      @@instance = self
+      parse_options
+
+      @process = ProcessHelper.new(options[:logger], options[:pid_file], options[:user], options[:group])
+
+      pid = @process.running?
+      if pid
+        STDERR.puts "There is already a Starling process running (pid #{pid}), exiting."
+        exit(1)
+      elsif pid.nil?
+        STDERR.puts "Cleaning up stale pidfile at #{options[:pid_file]}."
+      end
+
+      start
+    end
+
+    def load_config_file(filename)
+      YAML.load(File.open(filename))['starling'].each do |key, value|
+        # alias some keys
+        case key
+        when "queue_path" then key = "path"
+        when "log_file" then key = "logger"
+        end
+        options[key.to_sym] = value
+
+        if options[:log_level].instance_of?(String)
+          options[:log_level] = Logger.const_get(options[:log_level])
+        end
+      end
+    end
+
+    def parse_options
+      self.options = { :host => '127.0.0.1',
+                       :port => 22122,
+                       :path => File.join(%w( / tmp starling spool )),
+                       :log_level => Logger::INFO,
+                       :daemonize => false,
+                       :timeout => 0,
+                       :pid_file => File.join(%w( / tmp starling starling.pid )) }
+
+      OptionParser.new do |opts|
+        opts.summary_width = 25
+
+        opts.banner = "Starling (#{StarlingServer::VERSION})\n\n",
+                      "usage: starling [options...]\n",
+                      "       starling --help\n",
+                      "       starling --version\n"
+
+        opts.separator ""
+        opts.separator "Configuration:"
+
+        opts.on("-f", "--config FILENAME",
+                "Config file (yaml) to load") do |filename|
+          load_config_file(filename)
+        end
+
+        opts.on("-q", "--queue_path PATH",
+                :REQUIRED,
+                "Path to store Starling queue logs", "(default: #{options[:path]})") do |queue_path|
+          options[:path] = queue_path
+        end
+
+        opts.separator ""; opts.separator "Network:"
+
+        opts.on("-hHOST", "--host HOST", "Interface on which to listen (default: #{options[:host]})") do |host|
+          options[:host] = host
+        end
+
+        opts.on("-pHOST", "--port PORT", Integer, "TCP port on which to listen (default: #{options[:port]})") do |port|
+          options[:port] = port
+        end
+
+        opts.separator ""; opts.separator "Process:"
+
+        opts.on("-d", "Run as a daemon.") do
+          options[:daemonize] = true
+        end
+
+        opts.on("-PFILE", "--pid FILENAME", "save PID in FILENAME when using -d option.", "(default: #{options[:pid_file]})") do |pid_file|
+          options[:pid_file] = pid_file
+        end
+
+        opts.on("-u", "--user USER", "User to run as") do |user|
+          options[:user] = user.to_i == 0 ? Etc.getpwnam(user).uid : user.to_i
+        end
+
+        opts.on("-gGROUP", "--group GROUP", "Group to run as") do |group|
+          options[:group] = group.to_i == 0 ? Etc.getgrnam(group).gid : group.to_i
+        end
+
+        opts.separator ""; opts.separator "Logging:"
+
+        opts.on("-L", "--log [FILE]", "Path to print debugging information.") do |log_path|
+          options[:logger] = log_path
+        end
+
+        opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
+          options[:syslog_channel] = channel
+        end
+
+        opts.on("-v", "Increase logging verbosity (may be used multiple times).") do
+          options[:log_level] -= 1
+        end
+
+        opts.on("-t", "--timeout [SECONDS]", Integer,
+                "Time in seconds before disconnecting inactive clients (0 to disable).",
+                "(default: #{options[:timeout]})") do |timeout|
+          options[:timeout] = timeout
+        end
+
+        opts.separator ""; opts.separator "Miscellaneous:"
+
+        opts.on_tail("-?", "--help", "Display this usage information.") do
+          puts "#{opts}\n"
+          exit
+        end
+
+        opts.on_tail("-V", "--version", "Print version number and exit.") do
+          puts "Starling #{StarlingServer::VERSION}\n\n"
+          exit
+        end
+      end.parse!
+    end
+
+    def start
+      drop_privileges
+
+      @process.daemonize if options[:daemonize]
+
+      setup_signal_traps
+      @process.write_pid_file
+
+      STDOUT.puts "Starting at #{options[:host]}:#{options[:port]}."
+      @server = StarlingServer::Base.new(options)
+      @server.run
+
+      @process.remove_pid_file
+    end
+
+    def drop_privileges
+      Process.egid = options[:group] if options[:group]
+      Process.euid = options[:user] if options[:user]
+    end
+
+    def shutdown
+      begin
+        STDOUT.puts "Shutting down."
+        StarlingServer::Base.logger.info "Shutting down."
+        @server.stop
+      rescue Object => e
+        STDERR.puts "There was an error shutting down: #{e}"
+        exit(70)
+      end
+    end
+
+    def setup_signal_traps
+      Signal.trap("INT") { shutdown }
+      Signal.trap("TERM") { shutdown }
+    end
+  end
+
+  class ProcessHelper
+
+    def initialize(log_file = nil, pid_file = nil, user = nil, group = nil)
+      @log_file = log_file
+      @pid_file = pid_file
+      @user = user
+      @group = group
+    end
+
+    def safefork
+      begin
+        if pid = fork
+          return pid
+        end
+      rescue Errno::EWOULDBLOCK
+        sleep 5
+        retry
+      end
+    end
+
+    def daemonize
+      sess_id = detach_from_terminal
+      exit if pid = safefork
+
+      Dir.chdir("/")
+      File.umask 0000
+
+      close_io_handles
+      redirect_io
+
+      return sess_id
+    end
+
+    def detach_from_terminal
+      srand
+      safefork and exit
+
+      unless sess_id = Process.setsid
+        raise "Couldn't detache from controlling terminal."
+      end
+
+      trap 'SIGHUP', 'IGNORE'
+
+      sess_id
+    end
+
+    def close_io_handles
+      ObjectSpace.each_object(IO) do |io|
+        unless [STDIN, STDOUT, STDERR].include?(io)
+          begin
+            io.close unless io.closed?
+          rescue Exception
+          end
+        end
+      end
+    end
+
+    def redirect_io
+      begin; STDIN.reopen('/dev/null'); rescue Exception; end
+
+      if @log_file
+        begin
+          STDOUT.reopen(@log_file, "a")
+          STDOUT.sync = true
+        rescue Exception
+          begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+        end
+      else
+        begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+      end
+
+      begin; STDERR.reopen(STDOUT); rescue Exception; end
+      STDERR.sync = true
+    end
+
+    def rescue_exception
+      begin
+        yield
+      rescue Exception
+      end
+    end
+
+    def write_pid_file
+      return unless @pid_file
+      FileUtils.mkdir_p(File.dirname(@pid_file))
+      File.open(@pid_file, "w") { |f| f.write(Process.pid) }
+      File.chmod(0644, @pid_file)
+    end
+
+    def remove_pid_file
+      return unless @pid_file
+      File.unlink(@pid_file) if File.exists?(@pid_file)
+    end
+
+    def running?
+      return false unless @pid_file
+
+      pid = File.read(@pid_file).chomp.to_i rescue nil
+      pid = nil if pid == 0
+      return false unless pid
+
+      begin
+        Process.kill(0, pid)
+        return pid
+      rescue Errno::ESRCH
+        return nil
+      rescue Errno::EPERM
+        return pid
+      end
+    end
+  end
+end

data/test/test_starling_server.rb

@@ -0,0 +1,205 @@
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'rubygems'
+require 'fileutils'
+require 'memcache'
+require 'digest/md5'
+
+require 'starling/server'
+
+class StarlingServer::PersistentQueue
+  remove_const :SOFT_LOG_MAX_SIZE
+  SOFT_LOG_MAX_SIZE = 16 * 1024 # 16 KB
+end
+
+def safely_fork(&block)
+  # anti-race juice:
+  blocking = true
+  Signal.trap("USR1") { blocking = false }
+  
+  pid = Process.fork(&block)
+  
+  while blocking
+    sleep 0.1
+  end
+  
+  pid
+end
+
+describe "StarlingServer" do
+  before do
+    @tmp_path = File.join(File.dirname(__FILE__), "tmp")
+    
+    begin
+      Dir::mkdir(@tmp_path)
+    rescue Errno::EEXIST
+    end
+
+    @server_pid = safely_fork do
+      server = StarlingServer::Base.new(:host => '127.0.0.1',
+                                        :port => 22133,
+                                        :path => @tmp_path,
+                                        :logger => Logger.new(STDERR),
+                                        :log_level => Logger::FATAL)
+      Signal.trap("INT") { server.stop }
+      Process.kill("USR1", Process.ppid)
+      server.run
+    end
+
+    @client = MemCache.new('127.0.0.1:22133')
+  end
+  
+  it "should test if temp_path exists and is writeable" do
+    File.exist?(@tmp_path).should be_true
+    File.directory?(@tmp_path).should be_true
+    File.writable?(@tmp_path).should be_true
+  end
+
+  it "should set and get" do
+    v = rand((2**32)-1)
+    @client.get('test_set_and_get_one_entry').should be_nil
+    @client.set('test_set_and_get_one_entry', v)
+    @client.get('test_set_and_get_one_entry').should eql(v)
+  end
+  
+  
+  it "should expire entries" do
+    v = rand((2**32)-1)
+    @client.get('test_set_with_expiry').should be_nil
+    now = Time.now.to_i
+    @client.set('test_set_with_expiry', v + 2, now)
+    @client.set('test_set_with_expiry', v)
+    sleep(1.0)
+    @client.get('test_set_with_expiry').should eql(v)
+  end
+
+  it "should have age stat" do
+    now = Time.now.to_i
+    @client.set('test_age', 'nibbler')
+    sleep(1.0)
+    @client.get('test_age').should eql('nibbler')
+
+    stats = @client.stats['127.0.0.1:22133']
+    stats.has_key?('queue_test_age_age').should be_true
+    (stats['queue_test_age_age'] >= 1000).should be_true
+  end
+  
+  it "should rotate log" do
+    log_rotation_path = File.join(@tmp_path, 'test_log_rotation')
+
+    Dir.glob("#{log_rotation_path}*").each do |file|
+      File.unlink(file) rescue nil
+    end
+    @client.get('test_log_rotation').should be_nil
+
+    v = 'x' * 8192
+
+    @client.set('test_log_rotation', v)
+    File.size(log_rotation_path).should eql(8207)
+    @client.get('test_log_rotation')
+
+    @client.get('test_log_rotation').should be_nil
+
+    @client.set('test_log_rotation', v)
+    @client.get('test_log_rotation').should eql(v)
+
+    File.size(log_rotation_path).should eql(1)
+    # rotated log should be erased after a successful roll.
+    Dir.glob("#{log_rotation_path}*").size.should eql(1)
+  end
+
+  it "should output statistics per server" do
+    stats = @client.stats
+    assert_kind_of Hash, stats
+    assert stats.has_key?('127.0.0.1:22133')
+    
+    server_stats = stats['127.0.0.1:22133']
+    
+    basic_stats = %w( bytes pid time limit_maxbytes cmd_get version
+                      bytes_written cmd_set get_misses total_connections
+                      curr_connections curr_items uptime get_hits total_items
+                      rusage_system rusage_user bytes_read )
+
+    basic_stats.each do |stat|
+      server_stats.has_key?(stat).should be_true
+    end
+  end
+
+  it "should return valid response with unkown command" do
+    response = @client.add('blah', 1)
+    response.should eql("CLIENT_ERROR bad command line format\r\n")
+  end
+
+  it "should disconnect and reconnect again" do
+    v = rand(2**32-1)
+    @client.set('test_that_disconnecting_and_reconnecting_works', v)
+    @client.reset
+    @client.get('test_that_disconnecting_and_reconnecting_works').should eql(v)
+  end
+  
+  it "should use epoll on linux" do
+    # this may take a few seconds.
+    # the point is to make sure that we're using epoll on Linux, so we can
+    # handle more than 1024 connections.
+    
+    unless IO::popen("uname").read.chomp == "Linux"
+      raise "(Skipping epoll test: not on Linux)"
+      skip = true
+    end
+    fd_limit = IO::popen("bash -c 'ulimit -n'").read.chomp.to_i
+    unless fd_limit > 1024
+      raise "(Skipping epoll test: 'ulimit -n' = #{fd_limit}, need > 1024)"
+      skip = true
+    end
+    
+    unless skip
+      v = rand(2**32 - 1)
+      @client.set('test_epoll', v)
+
+      # we can't open 1024 connections to memcache from within this process,
+      # because we will hit ruby's 1024 fd limit ourselves!
+      pid1 = safely_fork do
+        unused_sockets = []
+        600.times do
+          unused_sockets << TCPSocket.new("127.0.0.1", 22133)
+        end
+        Process.kill("USR1", Process.ppid)
+        sleep 90
+      end
+      pid2 = safely_fork do
+        unused_sockets = []
+        600.times do
+          unused_sockets << TCPSocket.new("127.0.0.1", 22133)
+        end
+        Process.kill("USR1", Process.ppid)
+        sleep 90
+      end
+
+      begin
+        client = MemCache.new('127.0.0.1:22133')
+        client.get('test_epoll').should eql(v)
+      ensure
+        Process.kill("TERM", pid1)
+        Process.kill("TERM", pid2)
+      end
+    end
+  end
+  
+  it "should raise error if queue collection is an invalid path" do
+    invalid_path = nil
+    while invalid_path.nil? || File.exist?(invalid_path)
+      invalid_path = File.join('/', Digest::MD5.hexdigest(rand(2**32-1).to_s)[0,8])
+    end
+
+    lambda {
+      StarlingServer::QueueCollection.new(invalid_path)
+    }.should raise_error(StarlingServer::InaccessibleQueuePath)
+  end
+  
+  after do
+    Process.kill("INT", @server_pid)
+    Process.wait(@server_pid)
+    @client.reset
+    FileUtils.rm(Dir.glob(File.join(@tmp_path, '*')))
+  end
+end