timshadel-starling 0.9.8.01.20080924

data/lib/starling/persistent_queue.rb

@@ -0,0 +1,151 @@
+module StarlingServer
+
+  ##
+  # PersistentQueue is a subclass of Ruby's thread-safe Queue class. It adds a
+  # transactional log to the in-memory Queue, which enables quickly rebuilding
+  # the Queue in the event of a sever outage.
+
+  class PersistentQueue < Queue
+
+    ##
+    # When a log reaches the SOFT_LOG_MAX_SIZE, the Queue will wait until
+    # it is empty, and will then rotate the log file.
+
+    SOFT_LOG_MAX_SIZE = 16 * (1024**2) # 16 MB
+
+    TRX_CMD_PUSH = "\000".freeze
+    TRX_CMD_POP = "\001".freeze
+
+    TRX_PUSH = "\000%s%s".freeze
+    TRX_POP = "\001".freeze
+
+    attr_reader :initial_bytes
+    attr_reader :total_items
+    attr_reader :logsize
+    attr_reader :current_age
+
+    ##
+    # Create a new PersistentQueue at +persistence_path+/+queue_name+.
+    # If a queue log exists at that path, the Queue will be loaded from
+    # disk before being available for use.
+
+    def initialize(persistence_path, queue_name, debug = false)
+      @persistence_path = persistence_path
+      @queue_name = queue_name
+      @total_items = 0
+      super()
+      @initial_bytes = replay_transaction_log(debug)
+      @current_age = 0
+    end
+
+    ##
+    # Pushes +value+ to the queue. By default, +push+ will write to the
+    # transactional log. Set +log_trx=false+ to override this behaviour.
+
+    def push(value, log_trx = true)
+      if log_trx
+        raise NoTransactionLog unless @trx
+        size = [value.size].pack("I")
+        transaction sprintf(TRX_PUSH, size, value)
+      end
+
+      @total_items += 1
+      super([now_usec, value])
+    end
+
+    ##
+    # Retrieves data from the queue.
+
+    def pop(log_trx = true)
+      raise NoTransactionLog if log_trx && !@trx
+
+      begin
+        rv = super(!log_trx)
+      rescue ThreadError
+        puts "WARNING: The queue was empty when trying to pop(). Technically this shouldn't ever happen. Probably a bug in the transactional underpinnings. Or maybe shutdown didn't happen cleanly at some point. Ignoring."
+        rv = [now_usec, '']
+      end
+      transaction "\001" if log_trx
+      @current_age = (now_usec - rv[0]) / 1000
+      rv[1]
+    end
+
+    ##
+    # Safely closes the transactional queue.
+
+    def close
+      # Ok, yeah, this is lame, and is *technically* a race condition. HOWEVER,
+      # the QueueCollection *should* have stopped processing requests, and I don't
+      # want to add yet another Mutex around all the push and pop methods. So we
+      # do the next simplest thing, and minimize the time we'll stick around before
+      # @trx is nil.
+      @not_trx = @trx
+      @trx = nil
+      @not_trx.close
+    end
+
+    private
+
+    def log_path #:nodoc:
+      File.join(@persistence_path, @queue_name)
+    end
+
+    def reopen_log #:nodoc:
+      @trx = File.new(log_path, File::CREAT|File::RDWR)
+      @logsize = File.size(log_path)
+    end
+
+    def rotate_log #:nodoc:
+      @trx.close
+      backup_logfile = "#{log_path}.#{Time.now.to_i}"
+      File.rename(log_path, backup_logfile)
+      reopen_log
+      File.unlink(backup_logfile)
+    end
+
+    def replay_transaction_log(debug) #:nodoc:
+      reopen_log
+      bytes_read = 0
+
+      print "Reading back transaction log for #{@queue_name} " if debug
+
+      while !@trx.eof?
+        cmd = @trx.read(1)
+        case cmd
+        when TRX_CMD_PUSH
+          print ">" if debug
+          raw_size = @trx.read(4)
+          next unless raw_size
+          size = raw_size.unpack("I").first
+          data = @trx.read(size)
+          next unless data
+          push(data, false)
+          bytes_read += data.size
+        when TRX_CMD_POP
+          print "<" if debug
+          bytes_read -= pop(false).size
+        else
+          puts "Error reading transaction log: " +
+               "I don't understand '#{cmd}' (skipping)." if debug
+        end
+      end
+
+      print " done.\n" if debug
+
+      return bytes_read
+    end
+
+    def transaction(data) #:nodoc:
+      raise "no transaction log handle. that totally sucks." unless @trx
+
+      @trx.write_nonblock data
+      @logsize += data.size
+      rotate_log if @logsize > SOFT_LOG_MAX_SIZE && self.length == 0
+    end
+    
+    def now_usec
+      now = Time.now
+      now.to_i * 1000000 + now.usec
+    end
+  end
+end

data/lib/starling/queue_collection.rb

@@ -0,0 +1,141 @@
+require 'thread'
+require 'starling/persistent_queue'
+
+module StarlingServer
+  class InaccessibleQueuePath < Exception #:nodoc:
+  end
+
+  ##
+  # QueueCollection is a proxy to a collection of PersistentQueue instances.
+
+  class QueueCollection
+
+    ##
+    # Create a new QueueCollection at +path+
+
+    def initialize(path)
+      unless File.directory?(path) && File.writable?(path)
+        raise InaccessibleQueuePath.new("'#{path}' must exist and be read-writable by #{Etc.getpwuid(Process.uid).name}.")
+      end
+
+      @shutdown_mutex = Mutex.new
+
+      @path = path
+      @logger = StarlingServer::Base.logger
+
+      @queues = {}
+      @queue_init_mutexes = {}
+
+      @stats = Hash.new(0)
+    end
+
+    ##
+    # Puts +data+ onto the queue named +key+
+
+    def put(key, data)
+      queue = queues(key)
+      return nil unless queue
+
+      @stats[:current_bytes] += data.size
+      @stats[:total_items] += 1
+
+      queue.push(data)
+
+      return true
+    end
+
+    ##
+    # Retrieves data from the queue named +key+
+
+    def take(key)
+      queue = queues(key)
+      if queue.nil? || queue.length == 0
+        @stats[:get_misses] += 1
+        return nil
+      else
+        @stats[:get_hits] += 1
+      end
+      result = queue.pop
+      @stats[:current_bytes] -= result.size
+      result
+    end
+
+    ##
+    # Returns all active queues.
+
+    def queues(key=nil)
+      return nil if @shutdown_mutex.locked?
+
+      return @queues if key.nil?
+
+      # First try to return the queue named 'key' if it's available.
+      return @queues[key] if @queues[key]
+
+      # If the queue wasn't available, create or get the mutex that will
+      # wrap creation of the Queue.
+      @queue_init_mutexes[key] ||= Mutex.new
+
+      # Otherwise, check to see if another process is already loading
+      # the queue named 'key'.
+      if @queue_init_mutexes[key].locked?
+        # return an empty/false result if we're waiting for the queue
+        # to be loaded and we're not the first process to request the queue
+        return nil
+      else
+        begin
+          @queue_init_mutexes[key].lock
+          # we've locked the mutex, but only go ahead if the queue hasn't
+          # been loaded. There's a race condition otherwise, and we could
+          # end up loading the queue multiple times.
+          if @queues[key].nil?
+            @queues[key] = PersistentQueue.new(@path, key)
+            @stats[:current_bytes] += @queues[key].initial_bytes
+          end
+        rescue Object => exc
+          puts "ZOMG There was an exception reading back the queue. That totally sucks."
+          puts "The exception was: #{exc}. Backtrace: #{exc.backtrace.join("\n")}"
+        ensure
+          @queue_init_mutexes[key].unlock
+        end
+      end
+
+      return @queues[key]
+    end
+
+    ##
+    # Returns statistic +stat_name+ for the QueueCollection.
+    #
+    # Valid statistics are:
+    #
+    #   [:get_misses]    Total number of get requests with empty responses
+    #   [:get_hits]      Total number of get requests that returned data
+    #   [:current_bytes] Current size in bytes of items in the queues
+    #   [:current_size]  Current number of items across all queues
+    #   [:total_items]   Total number of items stored in queues.
+
+    def stats(stat_name)
+      case stat_name
+      when nil; @stats
+      when :current_size; current_size
+      else; @stats[stat_name]
+      end
+    end
+
+    ##
+    # Safely close all queues.
+
+    def close
+      @shutdown_mutex.lock
+      @queues.each_pair do |name,queue|
+        queue.close
+        @queues.delete(name)
+      end
+    end
+
+    private
+
+    def current_size #:nodoc:
+      @queues.inject(0) { |m, (k,v)| m + v.length }
+    end
+  end
+end

data/lib/starling/server.rb

@@ -0,0 +1,125 @@
+require 'socket'
+require 'logger'
+require 'rubygems'
+require 'eventmachine'
+
+here = File.dirname(__FILE__)
+
+require File.join(here, 'queue_collection')
+require File.join(here, 'handler')
+
+module StarlingServer
+
+  VERSION = "0.9.8"
+  
+  class Base
+    attr_reader :logger
+
+    DEFAULT_HOST    = '127.0.0.1'
+    DEFAULT_PORT    = 22122
+    DEFAULT_PATH    = "/tmp/starling/"
+    DEFAULT_TIMEOUT = 60
+
+    ##
+    # Initialize a new Starling server and immediately start processing
+    # requests.
+    #
+    # +opts+ is an optional hash, whose valid options are:
+    #
+    #   [:host]     Host on which to listen (default is 127.0.0.1).
+    #   [:port]     Port on which to listen (default is 22122).
+    #   [:path]     Path to Starling queue logs. Default is /tmp/starling/
+    #   [:timeout]  Time in seconds to wait before closing connections.
+    #   [:logger]   A Logger object, an IO handle, or a path to the log.
+    #   [:loglevel] Logger verbosity. Default is Logger::ERROR.
+    #
+    # Other options are ignored.
+
+    def self.start(opts = {})
+      server = self.new(opts)
+      server.run
+    end
+
+    ##
+    # Initialize a new Starling server, but do not accept connections or
+    # process requests.
+    #
+    # +opts+ is as for +start+
+
+    def initialize(opts = {})
+      @opts = {
+        :host    => DEFAULT_HOST,
+        :port    => DEFAULT_PORT,
+        :path    => DEFAULT_PATH,
+        :timeout => DEFAULT_TIMEOUT,
+        :server  => self
+      }.merge(opts)
+
+      @stats = Hash.new(0)
+
+      FileUtils.mkdir_p(@opts[:path])
+
+    end
+
+    ##
+    # Start listening and processing requests.
+
+    def run
+      @stats[:start_time] = Time.now
+
+      if @opts[:syslog_channel]
+        begin
+          require 'syslog_logger'
+          @@logger = SyslogLogger.new(@opts[:syslog_channel])
+        rescue LoadError
+          # SyslogLogger isn't available, so we're just going to use Logger
+        end
+      end
+
+      @@logger ||= case @opts[:logger]
+                   when IO, String; Logger.new(@opts[:logger])
+                   when Logger; @opts[:logger]
+                   else; Logger.new(STDERR)
+                   end
+
+      begin
+        @opts[:queue] = QueueCollection.new(@opts[:path])
+      rescue InaccessibleQueuePath => e
+        puts "Error: #{e.message}"
+        exit 1
+      end
+      @@logger.level = @opts[:log_level] || Logger::ERROR
+
+      @@logger.info "Starling STARTUP on #{@opts[:host]}:#{@opts[:port]}"
+
+      EventMachine.epoll
+      EventMachine.set_descriptor_table_size(4096)
+      EventMachine.run do
+        EventMachine.start_server(@opts[:host], @opts[:port], Handler, @opts)
+      end
+
+      # code here will get executed on shutdown:
+      @opts[:queue].close
+    end
+
+    def self.logger
+      @@logger
+    end
+
+
+    ##
+    # Stop accepting new connections and shutdown gracefully.
+
+    def stop
+      EventMachine.stop_event_loop
+    end
+
+    def stats(stat = nil) #:nodoc:
+      case stat
+      when nil; @stats
+      when :connections; 1
+      else; @stats[stat]
+      end
+    end
+  end
+end

data/lib/starling/server_runner.rb

@@ -0,0 +1,297 @@
+require File.join(File.dirname(__FILE__), 'server')
+require 'optparse'
+require 'yaml'
+
+module StarlingServer
+  class Runner
+
+    attr_accessor :options
+    private :options, :options=
+
+    def self.run
+      new
+    end
+    
+    def self.shutdown
+      @@instance.shutdown
+    end
+
+    def initialize
+      @@instance = self
+      parse_options
+
+      @process = ProcessHelper.new(options[:logger], options[:pid_file], options[:user], options[:group])
+
+      pid = @process.running?
+      if pid
+        STDERR.puts "There is already a Starling process running (pid #{pid}), exiting."
+        exit(1)
+      elsif pid.nil?
+        STDERR.puts "Cleaning up stale pidfile at #{options[:pid_file]}."
+      end
+
+      start
+    end
+
+    def load_config_file(filename)
+      YAML.load(File.open(filename))['starling'].each do |key, value|
+        # alias some keys
+        case key
+        when "queue_path" then key = "path"
+        when "log_file" then key = "logger"
+        end
+        options[key.to_sym] = value
+
+        if options[:log_level].instance_of?(String)
+          options[:log_level] = Logger.const_get(options[:log_level])
+        end
+      end
+    end
+
+    def parse_options
+      self.options = { :host => '127.0.0.1',
+                       :port => 22122,
+                       :path => File.join('', 'var', 'spool', 'starling'),
+                       :log_level => Logger::INFO,
+                       :daemonize => false,
+                       :timeout => 0,
+                       :pid_file => File.join('', 'var', 'run', 'starling.pid') }
+
+      OptionParser.new do |opts|
+        opts.summary_width = 25
+
+        opts.banner = "Starling (#{StarlingServer::VERSION})\n\n",
+                      "usage: starling [options...]\n",
+                      "       starling --help\n",
+                      "       starling --version\n"
+
+        opts.separator ""
+        opts.separator "Configuration:"
+
+        opts.on("-f", "--config FILENAME",
+                "Config file (yaml) to load") do |filename|
+          load_config_file(filename)
+        end
+
+        opts.on("-q", "--queue_path PATH",
+                :REQUIRED,
+                "Path to store Starling queue logs", "(default: #{options[:path]})") do |queue_path|
+          options[:path] = File.expand_path(queue_path)
+        end
+
+        opts.separator ""; opts.separator "Network:"
+
+        opts.on("-hHOST", "--host HOST", "Interface on which to listen (default: #{options[:host]})") do |host|
+          options[:host] = host
+        end
+
+        opts.on("-pHOST", "--port PORT", Integer, "TCP port on which to listen (default: #{options[:port]})") do |port|
+          options[:port] = port
+        end
+
+        opts.separator ""; opts.separator "Process:"
+
+        opts.on("-d", "Run as a daemon.") do
+          options[:daemonize] = true
+        end
+
+        opts.on("-PFILE", "--pid FILENAME", "save PID in FILENAME when using -d option.", "(default: #{options[:pid_file]})") do |pid_file|
+          options[:pid_file] = File.expand_path(pid_file)
+        end
+
+        opts.on("-u", "--user USER", "User to run as") do |user|
+          options[:user] = user.to_i == 0 ? Etc.getpwnam(user).uid : user.to_i
+        end
+
+        opts.on("-gGROUP", "--group GROUP", "Group to run as") do |group|
+          options[:group] = group.to_i == 0 ? Etc.getgrnam(group).gid : group.to_i
+        end
+
+        opts.separator ""; opts.separator "Logging:"
+
+        opts.on("-L", "--log [FILE]", "Path to print debugging information.") do |log_path|
+          options[:logger] = File.expand_path(log_path)
+        end
+
+        begin
+          require 'syslog_logger'
+
+          opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
+            options[:syslog_channel] = channel
+          end
+        rescue LoadError
+        end
+
+        opts.on("-v", "Increase logging verbosity (may be used multiple times).") do
+          options[:log_level] -= 1
+        end
+
+        opts.on("-t", "--timeout [SECONDS]", Integer,
+                "Time in seconds before disconnecting inactive clients (0 to disable).",
+                "(default: #{options[:timeout]})") do |timeout|
+          options[:timeout] = timeout
+        end
+
+        opts.separator ""; opts.separator "Miscellaneous:"
+
+        opts.on_tail("-?", "--help", "Display this usage information.") do
+          puts "#{opts}\n"
+          exit
+        end
+
+        opts.on_tail("-V", "--version", "Print version number and exit.") do
+          puts "Starling #{StarlingServer::VERSION}\n\n"
+          exit
+        end
+      end.parse!
+    end
+
+    def start
+      drop_privileges
+
+      @process.daemonize if options[:daemonize]
+
+      setup_signal_traps
+      @process.write_pid_file
+
+      STDOUT.puts "Starting at #{options[:host]}:#{options[:port]}."
+      @server = StarlingServer::Base.new(options)
+      @server.run
+
+      @process.remove_pid_file
+    end
+
+    def drop_privileges
+      Process.egid = options[:group] if options[:group]
+      Process.euid = options[:user] if options[:user]
+    end
+
+    def shutdown
+      begin
+        STDOUT.puts "Shutting down."
+        StarlingServer::Base.logger.info "Shutting down."
+        @server.stop
+      rescue Object => e
+        STDERR.puts "There was an error shutting down: #{e}"
+        exit(70)
+      end
+    end
+
+    def setup_signal_traps
+      Signal.trap("INT") { shutdown }
+      Signal.trap("TERM") { shutdown }
+    end
+  end
+
+  class ProcessHelper
+
+    def initialize(log_file = nil, pid_file = nil, user = nil, group = nil)
+      @log_file = log_file
+      @pid_file = pid_file
+      @user = user
+      @group = group
+    end
+
+    def safefork
+      begin
+        if pid = fork
+          return pid
+        end
+      rescue Errno::EWOULDBLOCK
+        sleep 5
+        retry
+      end
+    end
+
+    def daemonize
+      sess_id = detach_from_terminal
+      exit if pid = safefork
+
+      Dir.chdir("/")
+      File.umask 0000
+
+      close_io_handles
+      redirect_io
+
+      return sess_id
+    end
+
+    def detach_from_terminal
+      srand
+      safefork and exit
+
+      unless sess_id = Process.setsid
+        raise "Couldn't detach from controlling terminal."
+      end
+
+      trap 'SIGHUP', 'IGNORE'
+
+      sess_id
+    end
+
+    def close_io_handles
+      ObjectSpace.each_object(IO) do |io|
+        unless [STDIN, STDOUT, STDERR].include?(io)
+          begin
+            io.close unless io.closed?
+          rescue Exception
+          end
+        end
+      end
+    end
+
+    def redirect_io
+      begin; STDIN.reopen('/dev/null'); rescue Exception; end
+
+      if @log_file
+        begin
+          STDOUT.reopen(@log_file, "a")
+          STDOUT.sync = true
+        rescue Exception
+          begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+        end
+      else
+        begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+      end
+
+      begin; STDERR.reopen(STDOUT); rescue Exception; end
+      STDERR.sync = true
+    end
+
+    def rescue_exception
+      begin
+        yield
+      rescue Exception
+      end
+    end
+
+    def write_pid_file
+      return unless @pid_file
+      FileUtils.mkdir_p(File.dirname(@pid_file))
+      File.open(@pid_file, "w") { |f| f.write(Process.pid) }
+      File.chmod(0644, @pid_file)
+    end
+
+    def remove_pid_file
+      return unless @pid_file
+      File.unlink(@pid_file) if File.exists?(@pid_file)
+    end
+
+    def running?
+      return false unless @pid_file
+
+      pid = File.read(@pid_file).chomp.to_i rescue nil
+      pid = nil if pid == 0
+      return false unless pid
+
+      begin
+        Process.kill(0, pid)
+        return pid
+      rescue Errno::ESRCH
+        return nil
+      rescue Errno::EPERM
+        return pid
+      end
+    end
+  end
+end