starling 0.9.3

data/lib/starling/persistent_queue.rb

@@ -0,0 +1,156 @@
+if RUBY_VERSION >= "1.8.6"
+  require 'thread'
+else
+  require 'fastthread'
+end
+
+module StarlingServer
+
+  ##
+  # PersistentQueue is a subclass of Ruby's thread-safe Queue class. It adds a
+  # transactional log to the in-memory Queue, which enables quickly rebuilding
+  # the Queue in the event of a sever outage.
+
+  class PersistentQueue < Queue
+
+    ##
+    # When a log reaches the SOFT_LOG_MAX_SIZE, the Queue will wait until
+    # it is empty, and will then rotate the log file.
+
+    SOFT_LOG_MAX_SIZE = 16 * (1024**2) # 16 MB
+
+    TRX_CMD_PUSH = "\000".freeze
+    TRX_CMD_POP = "\001".freeze
+
+    TRX_PUSH = "\000%s%s".freeze
+    TRX_POP = "\001".freeze
+
+    attr_reader :initial_bytes
+    attr_reader :total_items
+    attr_reader :logsize
+
+    ##
+    # Create a new PersistentQueue at +persistence_path+/+queue_name+.
+    # If a queue log exists at that path, the Queue will be loaded from
+    # disk before being available for use.
+    
+    def initialize(persistence_path, queue_name, debug = false)
+      @persistence_path = persistence_path
+      @queue_name = queue_name
+      @transaction_mutex = Mutex.new
+      @total_items = 0
+      super()
+      @initial_bytes = replay_transaction_log(debug)
+    end
+
+    ##
+    # Pushes +value+ to the queue. By default, +push+ will write to the
+    # transactional log. Set +log_trx=false+ to override this behaviour.
+
+    def push(value, log_trx = true)
+      if log_trx
+        raise NoTransactionLog unless @trx
+        size = [value.size].pack("I")
+        transaction sprintf(TRX_PUSH, size, value)
+      end
+
+      @total_items += 1
+      super(value)
+    end
+
+    ##
+    # Retrieves data from the queue.
+    
+    def pop(log_trx = true)
+      raise NoTransactionLog if log_trx && !@trx
+      
+      begin
+        rv = super(!log_trx)
+      rescue ThreadError
+        puts "WARNING: The queue was empty when trying to pop(). Technically this shouldn't ever happen. Probably a bug in the transactional underpinnings. Or maybe shutdown didn't happen cleanly at some point. Ignoring."
+        rv = ''
+      end
+      transaction "\001" if log_trx
+      rv
+    end
+
+    ##
+    # Safely closes the transactional queue.
+    
+    def close
+      @transaction_mutex.lock
+
+      # Ok, yeah, this is lame, and is *technically* a race condition. HOWEVER,
+      # the QueueCollection *should* have stopped processing requests, and I don't
+      # want to add yet another Mutex around all the push and pop methods. So we
+      # do the next simplest thing, and minimize the time we'll stick around before
+      # @trx is nil.
+      @not_trx = @trx
+      @trx = nil
+      @not_trx.close
+    end
+
+    private
+
+    def log_path #:nodoc:
+      File.join(@persistence_path, @queue_name)
+    end
+
+    def reopen_log #:nodoc:
+      @trx = File.new(log_path, File::CREAT|File::RDWR)
+      @logsize = File.size(log_path)
+    end
+
+    def rotate_log #:nodoc:
+      @trx.close
+      File.rename(log_path, "#{log_path}.#{Time.now.to_i}")
+      reopen_log
+    end
+
+    def replay_transaction_log(debug) #:nodoc:
+      reopen_log
+      bytes_read = 0
+
+      print "Reading back transaction log for #{@queue_name} " if debug
+
+      while !@trx.eof?
+        cmd = @trx.read(1)
+        case cmd
+        when TRX_CMD_PUSH
+          print ">" if debug
+          raw_size = @trx.read(4)
+          next unless raw_size
+          size = raw_size.unpack("I").first
+          data = @trx.read(size)
+          next unless data
+          push(data, false)
+          bytes_read += data.size
+        when TRX_CMD_POP
+          print "<" if debug
+          bytes_read -= pop(false).size
+        else
+          puts "Error reading transaction log: " +
+               "I don't understand '#{cmd}' (skipping)." if debug
+        end
+      end
+
+      print " done.\n" if debug
+
+      return bytes_read
+    end
+
+    def transaction(data) #:nodoc:
+      raise "no transaction log handle. that totally sucks." unless @trx
+
+      begin
+        @transaction_mutex.lock
+        @trx.write data
+        @trx.fsync
+        @logsize += data.size
+        rotate_log if @logsize > SOFT_LOG_MAX_SIZE && self.length == 0
+      ensure
+        @transaction_mutex.unlock
+      end
+    end
+  end
+end

data/lib/starling/queue_collection.rb

@@ -0,0 +1,140 @@
+require 'thread'
+require 'starling/persistent_queue'
+
+module StarlingServer
+  class InaccessibleQueuePath < Exception #:nodoc:
+  end
+
+  ##
+  # QueueCollection is a proxy to a collection of PersistentQueue instances.
+  
+  class QueueCollection
+
+    ##
+    # Create a new QueueCollection at +path+
+    
+    def initialize(path)
+      unless File.directory?(path) && File.writable?(path)
+        raise InaccessibleQueuePath.new(path)
+      end
+
+      @shutdown_mutex = Mutex.new
+
+      @path = path
+
+      @queues = {}
+      @queue_init_mutexes = {}
+
+      @stats = Hash.new(0)
+    end
+
+    ##
+    # Puts +data+ onto the queue named +key+
+
+    def put(key, data)
+      queue = queues(key)
+      return nil unless queue
+
+      @stats[:current_bytes] += data.size
+      @stats[:total_items] += 1
+
+      queue.push(data)
+      
+      return true
+    end
+
+    ##
+    # Retrieves data from the queue named +key+
+
+    def take(key)
+      queue = queues(key)
+      if queue.nil? || queue.length == 0
+        @stats[:get_misses] += 1
+        return nil
+      else
+        @stats[:get_hits] += 1
+      end
+      result = queue.pop
+      @stats[:current_bytes] -= result.size
+      result
+    end
+
+    ##
+    # Returns all active queues. 
+
+    def queues(key=nil)
+      return nil if @shutdown_mutex.locked?
+
+      return @queues if key.nil?
+
+      # First try to return the queue named 'key' if it's available.
+      return @queues[key] if @queues[key]
+
+      # If the queue wasn't available, create or get the mutex that will
+      # wrap creation of the Queue.
+      @queue_init_mutexes[key] ||= Mutex.new
+
+      # Otherwise, check to see if another process is already loading
+      # the queue named 'key'.
+      if @queue_init_mutexes[key].locked?
+        # return an empty/false result if we're waiting for the queue
+        # to be loaded and we're not the first process to request the queue
+        return nil
+      else
+        begin
+          @queue_init_mutexes[key].lock
+          # we've locked the mutex, but only go ahead if the queue hasn't
+          # been loaded. There's a race condition otherwise, and we could
+          # end up loading the queue multiple times.
+          if @queues[key].nil?
+            @queues[key] = PersistentQueue.new(@path, key)
+            @stats[:current_bytes] += @queues[key].initial_bytes
+          end
+        rescue Object => exc
+          puts "ZOMG There was an exception reading back the queue. That totally sucks."
+          puts "The exception was: #{exc}. Backtrace: #{exc.backtrace.join("\n")}"
+        ensure
+          @queue_init_mutexes[key].unlock
+        end
+      end
+
+      return @queues[key]
+    end
+
+    ##
+    # Returns statistic +stat_name+ for the QueueCollection.
+    #
+    # Valid statistics are:
+    #
+    #   [:get_misses]    Total number of get requests with empty responses
+    #   [:get_hits]      Total number of get requests that returned data
+    #   [:current_bytes] Current size in bytes of items in the queues
+    #   [:current_size]  Current number of items across all queues
+    #   [:total_items]   Total number of items stored in queues.
+
+    def stats(stat_name)
+      case stat_name
+      when nil; @stats
+      when :current_size; current_size
+      else; @stats[stat_name]
+      end
+    end
+
+    ##
+    # Safely close all queues.
+
+    def close
+      @shutdown_mutex.lock
+      @queues.each_pair do |name,queue|
+        queue.close
+        @queues.delete(name)
+      end
+    end
+
+    private
+
+    def current_size #:nodoc:
+      @queues.inject(0) { |m, (k,v)| m + v.length }
+    end
+  end
+end

data/lib/starling/runner.rb

@@ -0,0 +1,253 @@
+require File.join(File.dirname(__FILE__), 'server')
+require 'optparse'
+
+module StarlingServer
+  class Runner
+
+    attr_accessor :options
+    private :options, :options=
+
+    def self.run
+      new
+    end
+   
+    def initialize
+      parse_options
+
+      @process = ProcessHelper.new(options[:log_file], options[:pid_file], options[:user], options[:group])
+
+      pid = @process.running?
+      if pid
+        STDERR.puts "There is already a Starling process running (pid #{pid}), exiting."
+        exit(1)
+      elsif pid.nil?
+        STDERR.puts "Cleaning up stale pidfile at #{options[:pid_file]}."
+      end
+
+      start
+    end
+
+    def parse_options
+      self.options = { :host => '127.0.0.1',
+                       :port => 22122,
+                       :path => File.join(%w( / var spool starling )),
+                       :log_level => 0,
+                       :daemonize => false,
+                       :pid_file => File.join(%w( / var run starling.pid )) }
+
+      OptionParser.new do |opts|
+        opts.summary_width = 25
+
+        opts.banner = "Starling (#{StarlingServer::VERSION})\n\n",
+                      "usage: starling [-v] [-q path] [-h host] [-p port]\n",
+                      "                [-d [-P pidfile]] [-u user] [-g group] [-l log]\n",
+                      "       starling --help\n",
+                      "       starling --version\n"
+
+        opts.separator ""
+        opts.separator "Configuration:"
+        opts.on("-q", "--queue_path PATH",
+                :REQUIRED,
+                "Path to store Starling queue logs", "(default: #{options[:path]})") do |queue_path|
+          options[:path] = queue_path
+        end
+
+        opts.separator ""; opts.separator "Network:"
+
+        opts.on("-hHOST", "--host HOST", "Interface on which to listen (default: #{options[:host]})") do |host|
+          options[:host] = host
+        end
+
+        opts.on("-pHOST", "--port PORT", Integer, "TCP port on which to listen (default: #{options[:port]})") do |port|
+          options[:port] = port
+        end
+
+        opts.separator ""; opts.separator "Process:"
+
+        opts.on("-d", "Run as a daemon.") do
+          options[:daemonize] = true
+        end
+
+        opts.on("-PFILE", "--pid FILE", "save PID in FILE when using -d option.", "(default: #{options[:pid_file]})") do |pid_file|
+          options[:pid_file] = pid_file
+        end
+
+        opts.on("-u", "--user USER", Integer, "User to run as") do |user|
+          options[:user] = user
+        end
+
+        opts.on("-gGROUP", "--group GROUP", "Group to run as") do |group|
+          options[:group] = group
+        end
+
+        opts.separator ""; opts.separator "Logging:"
+
+        opts.on("-l", "--log [FILE]", "Path to print debugging information.") do |log_path|
+          options[:log] = log_path
+        end
+
+        opts.on("-v", "Increase logging verbosity.") do
+          options[:log_level] += 1
+        end
+
+        opts.separator ""; opts.separator "Miscellaneous:"
+
+        opts.on_tail("-?", "--help", "Display this usage information.") do
+          puts "#{opts}\n"
+          exit
+        end
+
+        opts.on_tail("-V", "--version", "Print version number and exit.") do
+          puts "Starling #{StarlingServer::VERSION}\n\n"
+          exit
+        end
+      end.parse!
+    end
+
+    def start
+      drop_privileges
+
+      @process.daemonize if options[:daemonize]
+
+      setup_signal_traps
+      @process.write_pid_file
+
+      @server, @thread = StarlingServer::Base.start(options)
+      @thread.join
+
+      @process.remove_pid_file
+    end
+
+    def drop_privileges
+      Process.euid = options[:user] if options[:user]
+      Process.egid = options[:group] if options[:group]
+    end
+
+    def shutdown
+      begin
+        STDOUT.puts "Shutting down."
+        @server.logger.info "Shutting down."
+        @server.stop
+      rescue Object => e
+        STDERR.puts "There was an error shutting down: #{e}"
+        exit(70)
+      end
+    end
+
+    def setup_signal_traps
+      Signal.trap("INT") { shutdown }
+      Signal.trap("TERM") { shutdown }
+    end
+  end
+
+  class ProcessHelper
+
+    def initialize(log_file = nil, pid_file = nil, user = nil, group = nil)
+      @log_file = log_file
+      @pid_file = pid_file
+      @user = user
+      @group = group
+    end
+
+    def safefork
+      begin
+        if pid = fork
+          return pid
+        end
+      rescue Errno::EWOULDBLOCK
+        sleep 5
+        retry
+      end
+    end
+
+    def daemonize
+      sess_id = detach_from_terminal
+      exit if pid = safefork
+
+      Dir.chdir("/")
+      File.umask 0000
+
+      close_io_handles
+      redirect_io
+
+      return sess_id
+    end
+
+    def detach_from_terminal
+      srand
+      safefork and exit
+
+      unless sess_id = Process.setsid
+        raise "Couldn't detache from controlling terminal."
+      end
+
+      trap 'SIGHUP', 'IGNORE'
+
+      sess_id
+    end
+
+    def close_io_handles
+      ObjectSpace.each_object(IO) do |io|
+        unless [STDIN, STDOUT, STDERR].include?(io)
+          begin
+            io.close unless io.closed?
+          rescue Exception
+          end
+        end
+      end
+    end
+    
+    def redirect_io
+      begin; STDIN.reopen('/dev/null'); rescue Exception; end
+
+      if @log_file
+        begin
+          STDOUT.reopen(@log_file, "a")
+          STDOUT.sync = true
+        rescue Exception
+          begin; STDOUT.reopen('/dev/null'); rescue Exception; end 
+        end
+      else
+        begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+      end
+
+      begin; STDERR.reopen(STDOUT); rescue Exception; end
+      STDERR.sync = true
+    end
+
+    def rescue_exception
+      begin
+        yield
+      rescue Exception
+      end
+    end
+
+    def write_pid_file
+      return unless @pid_file
+      File.open(@pid_file, "w") { |f| f.write(Process.pid) }
+      File.chmod(0644, @pid_file)
+    end
+
+    def remove_pid_file
+      return unless @pid_file
+      File.unlink(@pid_file) if File.exists?(@pid_file)
+    end
+
+    def running?
+      return false unless @pid_file
+
+      pid = File.read(@pid_file).chomp.to_i rescue nil
+      pid = nil if pid == 0
+      return false unless pid
+
+      begin
+        Process.kill(0, pid)
+        return pid
+      rescue Errno::ESRCH
+        return nil
+      rescue Errno::EPERM
+        return pid
+      end
+    end
+  end
+end