kim-toms-starling 1.0.0

data/lib/starling/server_runner.rb

@@ -0,0 +1,317 @@
+require File.join(File.dirname(__FILE__), 'server')
+require 'erb'
+require 'fileutils'
+require 'optparse'
+require 'yaml'
+require 'fileutils'
+
+module StarlingServer
+  class Runner
+
+    attr_accessor :options
+    private :options, :options=
+
+    def self.run
+      new
+    end
+    
+    def self.shutdown
+      @@instance.shutdown
+    end
+
+    def initialize
+      @@instance = self
+      parse_options
+
+      @process = ProcessHelper.new(options[:logger], options[:pid_file], options[:user], options[:group])
+
+      pid = @process.running?
+      if pid
+        STDERR.puts "There is already a Starling process running (pid #{pid}), exiting."
+        exit(1)
+      elsif pid.nil?
+        STDERR.puts "Cleaning up stale pidfile at #{options[:pid_file]}."
+      end
+
+      start
+    end
+
+    def load_config_file(filename)
+      config = YAML.load(ERB.new(File.read(filename)).result)
+
+      unless config.is_a?(Hash)
+        STDERR.puts "Config file does not contain a hash: #{filename}, exiting."
+        exit(1)
+      end
+
+      if config['starling'].nil?
+        STDERR.puts "Missing starling section in config file: #{filename}, exiting."
+        exit(1)
+      end
+
+      config['starling'].each do |key, value|
+        # alias some keys
+        case key
+        when "queue_path" then key = "path"
+        when "log_file" then key = "logger"
+        end
+
+        if %w(logger path pid_file).include?(key)
+          value = File.expand_path(value)
+        end
+
+        options[key.to_sym] = value
+
+        if options[:log_level].instance_of?(String)
+          options[:log_level] = Logger.const_get(options[:log_level])
+        end
+      end
+    end
+
+    def parse_options
+      self.options = { :host => '127.0.0.1',
+                       :port => 22122,
+                       :path => File.join('', 'var', 'spool', 'starling'),
+                       :log_level => Logger::INFO,
+                       :daemonize => false,
+                       :timeout => 0,
+                       :pid_file => File.join('', 'var', 'run', 'starling.pid') }
+
+      OptionParser.new do |opts|
+        opts.summary_width = 25
+
+        opts.banner = "Starling (#{StarlingServer::VERSION})\n\n",
+                      "usage: starling [options...]\n",
+                      "       starling --help\n",
+                      "       starling --version\n"
+
+        opts.separator ""
+        opts.separator "Configuration:"
+
+        opts.on("-f", "--config FILENAME",
+                "Config file (yaml) to load") do |filename|
+          load_config_file(filename)
+        end
+
+        opts.on("-q", "--queue_path PATH",
+                :REQUIRED,
+                "Path to store Starling queue logs", "(default: #{options[:path]})") do |queue_path|
+          options[:path] = File.expand_path(queue_path)
+        end
+
+        opts.separator ""; opts.separator "Network:"
+
+        opts.on("-hHOST", "--host HOST", "Interface on which to listen (default: #{options[:host]})") do |host|
+          options[:host] = host
+        end
+
+        opts.on("-pHOST", "--port PORT", Integer, "TCP port on which to listen (default: #{options[:port]})") do |port|
+          options[:port] = port
+        end
+
+        opts.separator ""; opts.separator "Process:"
+
+        opts.on("-d", "Run as a daemon.") do
+          options[:daemonize] = true
+        end
+
+        opts.on("-PFILE", "--pid FILENAME", "save PID in FILENAME when using -d option.", "(default: #{options[:pid_file]})") do |pid_file|
+          options[:pid_file] = File.expand_path(pid_file)
+        end
+
+        opts.on("-u", "--user USER", "User to run as") do |user|
+          options[:user] = user.to_i == 0 ? Etc.getpwnam(user).uid : user.to_i
+        end
+
+        opts.on("-gGROUP", "--group GROUP", "Group to run as") do |group|
+          options[:group] = group.to_i == 0 ? Etc.getgrnam(group).gid : group.to_i
+        end
+
+        opts.separator ""; opts.separator "Logging:"
+
+        opts.on("-L", "--log [FILE]", "Path to print debugging information.") do |log_path|
+          options[:logger] = File.expand_path(log_path)
+        end
+
+        begin
+          require 'syslog_logger'
+
+          opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
+            options[:syslog_channel] = channel
+          end
+        rescue LoadError
+        end
+
+        opts.on("-v", "Increase logging verbosity (may be used multiple times).") do
+          options[:log_level] -= 1
+        end
+
+        opts.on("-t", "--timeout [SECONDS]", Integer,
+                "Time in seconds before disconnecting inactive clients (0 to disable).",
+                "(default: #{options[:timeout]})") do |timeout|
+          options[:timeout] = timeout
+        end
+
+        opts.separator ""; opts.separator "Miscellaneous:"
+
+        opts.on_tail("-?", "--help", "Display this usage information.") do
+          puts "#{opts}\n"
+          exit
+        end
+
+        opts.on_tail("-V", "--version", "Print version number and exit.") do
+          puts "Starling #{StarlingServer::VERSION}\n\n"
+          exit
+        end
+      end.parse!
+    end
+
+    def start
+      drop_privileges
+
+      @process.daemonize if options[:daemonize]
+
+      setup_signal_traps
+      @process.write_pid_file
+
+      STDOUT.puts "Starting at #{options[:host]}:#{options[:port]}."
+      @server = StarlingServer::Base.new(options)
+      @server.run
+
+      @process.remove_pid_file
+    end
+
+    def drop_privileges
+      Process.egid = options[:group] if options[:group]
+      Process.euid = options[:user] if options[:user]
+    end
+
+    def shutdown
+      begin
+        STDOUT.puts "Shutting down."
+        StarlingServer::Base.logger.info "Shutting down."
+        @server.stop
+      rescue Object => e
+        STDERR.puts "There was an error shutting down: #{e}"
+        exit(70)
+      end
+    end
+
+    def setup_signal_traps
+      Signal.trap("INT") { shutdown }
+      Signal.trap("TERM") { shutdown }
+    end
+  end
+
+  class ProcessHelper
+
+    def initialize(log_file = nil, pid_file = nil, user = nil, group = nil)
+      @log_file = log_file
+      @pid_file = pid_file
+      @user = user
+      @group = group
+    end
+
+    def safefork
+      begin
+        if pid = fork
+          return pid
+        end
+      rescue Errno::EWOULDBLOCK
+        sleep 5
+        retry
+      end
+    end
+
+    def daemonize
+      sess_id = detach_from_terminal
+      exit if pid = safefork
+
+      Dir.chdir("/")
+      File.umask 0000
+
+      close_io_handles
+      redirect_io
+
+      return sess_id
+    end
+
+    def detach_from_terminal
+      srand
+      safefork and exit
+
+      unless sess_id = Process.setsid
+        raise "Couldn't detach from controlling terminal."
+      end
+
+      trap 'SIGHUP', 'IGNORE'
+
+      sess_id
+    end
+
+    def close_io_handles
+      ObjectSpace.each_object(IO) do |io|
+        unless [STDIN, STDOUT, STDERR].include?(io)
+          begin
+            io.close unless io.closed?
+          rescue Exception
+          end
+        end
+      end
+    end
+
+    def redirect_io
+      begin; STDIN.reopen('/dev/null'); rescue Exception; end
+
+      if @log_file
+        begin
+          STDOUT.reopen(@log_file, "a")
+          STDOUT.sync = true
+        rescue Exception
+          begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+        end
+      else
+        begin; STDOUT.reopen('/dev/null'); rescue Exception; end
+      end
+
+      begin; STDERR.reopen(STDOUT); rescue Exception; end
+      STDERR.sync = true
+    end
+
+    def rescue_exception
+      begin
+        yield
+      rescue Exception
+      end
+    end
+
+    def write_pid_file
+      return unless @pid_file
+      FileUtils.mkdir_p(File.dirname(@pid_file))
+      File.open(@pid_file, "w") { |f| f.write(Process.pid) }
+      File.chmod(0644, @pid_file)
+    end
+
+    def remove_pid_file
+      return unless @pid_file
+      File.unlink(@pid_file) if File.exists?(@pid_file)
+    end
+
+    def running?
+      return false unless @pid_file
+
+      pid = File.read(@pid_file).chomp.to_i rescue nil
+      pid = nil if pid == 0
+      return false unless pid
+
+      begin
+        Process.kill(0, pid)
+        return pid
+      rescue Errno::ESRCH
+        return nil
+      rescue Errno::EPERM
+        return pid
+      end
+    end
+  end
+end

data/lib/starling.rb

@@ -0,0 +1,139 @@
+require 'memcache'
+
+class Starling < MemCache
+
+  WAIT_TIME = 0.25
+  alias_method :_original_get, :get
+  alias_method :_original_delete, :delete
+
+  ##
+  # fetch an item from a queue.
+
+  def get(*args)
+    loop do
+      response = _original_get(*args)
+      return response unless response.nil?
+      sleep WAIT_TIME
+    end
+  end
+  
+  ##
+  # will return the next item or nil
+
+  def fetch(*args)
+    _original_get(*args)
+  end
+
+  ##
+  # Delete the key (queue) from all Starling servers. This is necessary
+  # because the random way a server is chosen in #get_server_for_key
+  # implies that the queue could easily be spread across the entire
+  # Starling cluster.
+
+  def delete(key, expiry = 0)
+    with_servers do
+      _original_delete(key, expiry)
+    end
+  end
+
+  ##
+  # Provides a way to work with a specific list of servers by
+  # forcing all calls to #get_server_for_key to use a specific
+  # server, and changing that server each time that the call
+  # yields to the block provided.  This helps work around the
+  # normally random nature of the #get_server_for_key method.
+  #
+  def with_servers(my_servers = self.servers.dup)
+    return unless block_given?
+    my_servers.each do |server|
+      yield
+    end
+  end
+  
+  ##
+  # insert +value+ into +queue+.
+  #
+  # +expiry+ is expressed as a UNIX timestamp
+  #
+  # If +raw+ is true, +value+ will not be Marshalled. If +raw+ = :yaml, +value+
+  # will be serialized with YAML, instead.
+
+  def set(queue, value, expiry = 0, raw = false)
+    retries = 0
+    begin
+      if raw == :yaml
+        value = YAML.dump(value)
+        raw = true
+      end
+
+      super(queue, value, expiry, raw)
+    rescue MemCache::MemCacheError => e
+      retries += 1
+      sleep WAIT_TIME
+      retry unless retries > 3
+      raise e
+    end
+  end
+
+  ##
+  # returns the number of items in +queue+. If +queue+ is +:all+, a hash of all
+  # queue sizes will be returned.
+
+  def sizeof(queue, statistics = nil)
+    statistics ||= stats
+
+    if queue == :all
+      queue_sizes = {}
+      available_queues(statistics).each do |queue|
+        queue_sizes[queue] = sizeof(queue, statistics)
+      end
+      return queue_sizes
+    end
+
+    statistics.inject(0) { |m,(k,v)| m + v["queue_#{queue}_items"].to_i }
+  end
+
+  ##
+  # returns a list of available (currently allocated) queues.
+
+  def available_queues(statistics = nil)
+    statistics ||= stats
+
+    res = []
+    # Each server returns its own statistics
+    statistics.each_pair { |k,v| res = res + v.keys }
+    # Here we rely on the name of one of the statistic names to find the queue.
+    # The name of the queue is embedded in the statistic name
+    res.collect! {|k| /^queue_(.*)_total_items/.match(k)[1] rescue nil }.compact!
+  end
+
+  ##
+  # iterator to flush +queue+. Each element will be passed to the provided
+  # +block+
+
+  def flush(queue)
+    sizeof(queue).times do
+      v = get(queue)
+      yield v if block_given?
+    end
+  end
+
+  private
+
+  def get_server_for_key(key)
+    raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
+    raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
+    raise MemCacheError, "No servers available" if servers.empty?
+ 
+    # Ignores server weights, oh well
+    srvs = servers.dup
+    srvs.size.times do |try|
+      n = rand(srvs.size)
+      server = srvs[n]
+      return server if server.alive?
+      srvs.delete_at(n)
+    end
+  
+    raise MemCacheError, "No servers available (all dead)"
+  end
+end

data/starling.gemspec

@@ -0,0 +1,22 @@
+# this file is automatically generated
+  Gem::Specification.new do |s|
+    s.name = "starling"
+    s.version = "1.0.0"
+    s.authors = ["Blaine Cook", "Chris Wanstrath", "Britt Selvitelle", "Glenn Rempe", "Abdul-Rahman Advany", "Seth Fitzsimmons", "Haarm Arts", "Chris Gaffney", "Kim Toms"]
+    s.email = ["blaine@twitter.com", "chris@ozmm.org", "abdulrahman@advany.com", "starlingmq@groups.google.com", "harmaarts@gmail.com", "gaffneyc@gmail.com", "kim.toms@gmail.com"]
+    s.homepage = "http://github.com/starling/starling/"
+    s.summary = "Starling is a lightweight, transactional, distributed queue server"
+    s.description = s.summary
+    
+    s.files = ["./CHANGELOG", "./LICENSE", "./README.rdoc", "./Rakefile", "./bin/starling", "./bin/starling_top", "./etc/sample-config.yml", "./etc/starling.redhat", "./etc/starling.ubuntu", "./lib/starling.rb", "./lib/starling/handler.rb", "./lib/starling/persistent_queue.rb", "./lib/starling/queue_collection.rb", "./lib/starling/server.rb", "./lib/starling/server_runner.rb", "./starling.gemspec"]
+    
+    s.executables = ["starling", "starling_top"]
+    s.require_paths = ["lib"]
+
+    s.has_rdoc = true
+    s.rdoc_options = ["--quiet", "--title", "starling documentation", "--opname", "index.html", "--line-numbers", "--main", "README.rdoc", "--inline-source"]
+    s.extra_rdoc_files = ["README.rdoc", "CHANGELOG", "LICENSE"]
+    
+    s.add_dependency 'fiveruns-memcache-client'
+    s.add_dependency 'eventmachine', [">= 0.12.0"]
+  end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification 
+name: kim-toms-starling
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Blaine Cook
+- Chris Wanstrath
+- Britt Selvitelle
+- Glenn Rempe
+- Abdul-Rahman Advany
+- Seth Fitzsimmons
+- Haarm Arts
+- Chris Gaffney
+- Kim Toms
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-04-08 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fiveruns-memcache-client
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.0
+    version: 
+description: Starling is a lightweight, transactional, distributed queue server
+email: 
+- blaine@twitter.com
+- chris@ozmm.org
+- abdulrahman@advany.com
+- starlingmq@groups.google.com
+- harmaarts@gmail.com
+- gaffneyc@gmail.com
+- kim.toms@gmail.com
+executables: 
+- starling
+- starling_top
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- CHANGELOG
+- LICENSE
+files: 
+- ./CHANGELOG
+- ./LICENSE
+- ./README.rdoc
+- ./Rakefile
+- ./bin/starling
+- ./bin/starling_top
+- ./etc/sample-config.yml
+- ./etc/starling.redhat
+- ./etc/starling.ubuntu
+- ./lib/starling.rb
+- ./lib/starling/handler.rb
+- ./lib/starling/persistent_queue.rb
+- ./lib/starling/queue_collection.rb
+- ./lib/starling/server.rb
+- ./lib/starling/server_runner.rb
+- ./starling.gemspec
+- README.rdoc
+- CHANGELOG
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/starling/starling/
+post_install_message: 
+rdoc_options: 
+- --quiet
+- --title
+- starling documentation
+- --opname
+- index.html
+- --line-numbers
+- --main
+- README.rdoc
+- --inline-source
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Starling is a lightweight, transactional, distributed queue server
+test_files: []
+