starling-starling 0.9.7.10 → 0.9.8.200810061510

data/CHANGELOG

@@ -1,3 +1,20 @@
+== 0.9.9
+* remove dependency on SyslogLogger so that starling works on windows.
+* fix config file loading so relative paths are expanded properly <jacob@jacobatzen.dk>
+* clean up redhat init.d script <gaffneyc@gmail.com>
+
+== 0.9.8
+* add fix to enable relative paths <david@motorator.com>
+* fix tests so they don't run 10 times due to a stupid bug with how the server is forked <seth@mojodna.net>
+* fix some other tests <romeda@gmail.com>
+* fix some error messages <romeda@gmail.com>
+* probably some other things <romeda@gmail.com>
+
+== 0.9.7.9
+* properly complain if the spool directory isn't writable <seth@mojodna.net>
+* assume group and user privileges in a working order <seth@mojodna.net>
+* support string user / group names in addition to uid/gids <seth@mojodna.net>
+
 == 0.9.7.7
 * added init.d scripts for redhat and ubuntu by Mike Perham <mperham@gmail.com>
 * fixed dependencies for SyslogLogger, eventmachine and memcache-client by Mike Perham <mperham@gmail.com>

data/Rakefile

@@ -2,6 +2,8 @@ require 'rubygems'
 require 'rake/rdoctask'
 require 'spec/rake/spectask'
 
+task :default => :spec
+
 task :install do
   sh %{gem build starling.gemspec}
   sh %{sudo gem install starling-*.gem}
@@ -9,7 +11,7 @@ end
 
 Spec::Rake::SpecTask.new do |t|
   t.ruby_opts = ['-rtest/unit']
-  t.spec_files = FileList['test/test_*.rb']
+  t.spec_files = FileList['spec/*_spec.rb']
   t.fail_on_error = true
 end
   
@@ -17,5 +19,4 @@ Rake::RDocTask.new do |rd|
   rd.main = "README.rdoc"
   rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
   rd.rdoc_dir = 'doc'
-#  rd.options = spec.rdoc_options
 end

data/etc/sample-config.yml

@@ -0,0 +1,9 @@
+starling:
+  port: 22122
+  pid_file: /tmp/starling/starling.pid
+  queue_path: /tmp/starling/spool
+  timeout: 0
+  syslog_channel: starling-tampopo
+  log_level: DEBUG
+  daemonize: true
+

data/etc/starling.redhat

@@ -1,63 +1,66 @@
 #!/bin/bash
 #
+# Make sure the /var/run/starling, /var/log/starling and /var/spool/starling
+# all exist and are owned by starling:starling
+#
 # starling    This shell script takes care of starting and stopping
 #                    the starling server
 # chkconfig: 345 98 98
 # description: The starling queue server
+# processname: starling
+# pidfile: /var/run/starling/starling.pid
+# logfile: /var/log/starling/starling.log
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+# Source networking configuration.
+[ -f /etc/sysconfig/network ] && . /etc/sysconfig/network
+
+DUSER=starling
+DGROUP=starling
+LOGFILE=/var/log/starling/starling.log
+SPOOLDIR=/var/spool/starling
+PORT=22122
+LISTEN=0.0.0.0
+PIDFILE=/var/run/starling/starling.pid
 
-#determine where the 'pidof' executable is located
-if [ -e /bin/pidof ]; then
-  PIDOF="/bin/pidof"
-elif [ -e /sbin/pidof ]; then
-  PIDOF="/sbin/pidof"
-elif [ -e /usr/local/bin/pidof ]; then
-  PIDOF="/usr/local/bin/pidof"
-elif [ -e /bin/pgrep ]; then
-  PIDOF="/bin/pgrep"
-elif [ -e /usr/bin/pgrep ]; then
-  PIDOF="/usr/bin/pgrep"
-elif [ -e /usr/local/bin/pgrep ]; then
-  PIDOF="/usr/local/bin/pgrep"
-else
-  echo "Could not find pidof or pgrep"
-fi
-
-PROGDIR="/usr/bin"
-PROGNAME="starling"
-OPTIONS="-u nobody -g nobody -L /var/log/starling.log -q /var/spool/starling"
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=starling
+INSTALL_DIR=/usr/local/bin
+DAEMON=$INSTALL_DIR/$NAME
+OPTS="-h $LISTEN -p $PORT -d -q $SPOOLDIR -P $PIDFILE -L $LOGFILE -u $DUSER -g $DGROUP"
 
 start() {
-  pid=`$PIDOF $PROGNAME`
-  if [ "$pid" != "" ]; then
-    echo "$PROGDIR$PROGNAME already running: $pid"
-  else
-    echo "Starting $PROGDIR$PROGNAME"
-    cd $PROGDIR
-    nohup $PROGDIR$PROGNAME $OPTIONS &
-  fi
+  echo -n $"Starting starling: "
+
+  daemon --pidfile=$PIDFILE $DAEMON $OPTS
+  echo
 }
 
 stop() {
-  pid=`$PIDOF $PROGNAME`
-  if [ "$pid" != "" ]; then
-    echo "Stopping $PROGDIR$PROGNAME: $pid"
-    kill $pid
-  else
-    echo "$PROGDIR$PROGNAME not running"
-  fi
+  echo -n $"Stopping starling: "
+
+  killproc -p $PIDFILE starling
+  echo
 }
 
 case "$1" in
-start)
-  start
-;;
-stop)
-  stop
-;;
-restart)
-  stop
-  sleep 3
-  start
-;;
-
+  start)
+    start
+  ;;
+  stop)
+    stop
+  ;;
+  restart)
+    stop
+    sleep 3
+    start
+  ;;
+  status)
+    status -p $PIDFILE starling
+  ;;
+  *)
+    echo $"Usage: $0 {start|stop|restart|status}"
+    exit 1
 esac

data/lib/starling.rb

@@ -3,18 +3,60 @@ require 'memcache'
 class Starling < MemCache
 
   WAIT_TIME = 0.25
+  alias_method :_original_get, :get
+  alias_method :_original_delete, :delete
 
   ##
   # fetch an item from a queue.
 
   def get(*args)
     loop do
-      response = super(*args)
+      response = _original_get(*args)
       return response unless response.nil?
       sleep WAIT_TIME
     end
   end
+  
+  ##
+  # will return the next item or nil
+
+  def fetch(*args)
+    _original_get(*args)
+  end
 
+  ##
+  # Delete the key (queue) from all Starling servers. This is necessary
+  # because the random way a server is chosen in #get_server_for_key
+  # implies that the queue could easily be spread across the entire
+  # Starling cluster.
+
+  def delete(key, expiry = 0)
+    with_servers do
+      _original_delete(key, expiry)
+    end
+  end
+
+  ##
+  # Provides a way to work with a specific list of servers by
+  # forcing all calls to #get_server_for_key to use a specific
+  # server, and changing that server each time that the call
+  # yields to the block provided.  This helps work around the
+  # normally random nature of the #get_server_for_key method.
+  #
+  # Acquires the mutex for the entire duration of the call
+  # since unrelated calls to #get_server_for_key might be
+  # adversely affected by the non_random result.
+  def with_servers(my_servers = @servers.dup)
+    return unless block_given?
+    with_lock do
+      my_servers.each do |server|
+        @force_server = server
+        yield
+      end
+      @force_server = nil
+    end
+  end
+  
   ##
   # insert +value+ into +queue+.
   #
@@ -90,6 +132,7 @@ class Starling < MemCache
     raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
     raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
     raise MemCacheError, "No servers available" if @servers.empty?
+    return @force_server if @force_server
 
     bukkits = @buckets.dup
     bukkits.nitems.times do |try|
@@ -102,3 +145,24 @@ class Starling < MemCache
     raise MemCacheError, "No servers available (all dead)"
   end
 end
+
+
+class MemCache
+
+ protected
+
+  ##
+  # Ensure that everything within the given block is executed
+  # within the locked mutex if this client is multithreaded.
+  # If the client isn't multithreaded, the block is simply executed.
+  def with_lock
+    return unless block_given?
+    begin
+      @mutex.lock if @multithread
+      yield
+    ensure
+      @mutex.unlock if @multithread
+    end
+  end
+
+end

data/lib/starling/handler.rb

@@ -22,33 +22,37 @@ module StarlingServer
     SET_RESPONSE_SUCCESS  = "STORED\r\n".freeze
     SET_RESPONSE_FAILURE  = "NOT STORED\r\n".freeze
     SET_CLIENT_DATA_ERROR = "CLIENT_ERROR bad data chunk\r\nERROR\r\n".freeze
+    
+    # DELETE Responses
+    DELETE_COMMAND = /\Adelete (.{1,250}) ([0-9]+)\r\n/m
+    DELETE_RESPONSE = "END\r\n".freeze
 
     # STAT Response
     STATS_COMMAND = /\Astats\r\n/m
-    STATS_RESPONSE = "STAT pid %d
-STAT uptime %d
-STAT time %d
-STAT version %s
-STAT rusage_user %0.6f
-STAT rusage_system %0.6f
-STAT curr_items %d
-STAT total_items %d
-STAT bytes %d
-STAT curr_connections %d
-STAT total_connections %d
-STAT cmd_get %d
-STAT cmd_set %d
-STAT get_hits %d
-STAT get_misses %d
-STAT bytes_read %d
-STAT bytes_written %d
-STAT limit_maxbytes %d
+    STATS_RESPONSE = "STAT pid %d\r
+STAT uptime %d\r
+STAT time %d\r
+STAT version %s\r
+STAT rusage_user %0.6f\r
+STAT rusage_system %0.6f\r
+STAT curr_items %d\r
+STAT total_items %d\r
+STAT bytes %d\r
+STAT curr_connections %d\r
+STAT total_connections %d\r
+STAT cmd_get %d\r
+STAT cmd_set %d\r
+STAT get_hits %d\r
+STAT get_misses %d\r
+STAT bytes_read %d\r
+STAT bytes_written %d\r
+STAT limit_maxbytes %d\r
 %sEND\r\n".freeze
-    QUEUE_STATS_RESPONSE = "STAT queue_%s_items %d
-STAT queue_%s_total_items %d
-STAT queue_%s_logsize %d
-STAT queue_%s_expired_items %d
-STAT queue_%s_age %d\n".freeze
+    QUEUE_STATS_RESPONSE = "STAT queue_%s_items %d\r
+STAT queue_%s_total_items %d\r
+STAT queue_%s_logsize %d\r
+STAT queue_%s_expired_items %d\r
+STAT queue_%s_age %d\r\n".freeze
 
     SHUTDOWN_COMMAND = /\Ashutdown\r\n/m
 
@@ -108,7 +112,6 @@ STAT queue_%s_age %d\n".freeze
         @data_buf = data
         return
       end
-
       case data
       when SET_COMMAND
         @server.stats[:set_requests] += 1
@@ -121,6 +124,8 @@ STAT queue_%s_age %d\n".freeze
       when SHUTDOWN_COMMAND
         # no point in responding, they'll never get it.
         Runner::shutdown
+      when DELETE_COMMAND
+        delete $1
       else
         logger.warn "Unknown command: #{data}."
         respond ERR_UNKNOWN_COMMAND
@@ -136,6 +141,12 @@ STAT queue_%s_age %d\n".freeze
     end
 
   private
+  
+    def delete(queue)
+      @queue_collection.delete(queue)
+      respond DELETE_RESPONSE
+    end
+  
     def respond(str, *args)
       response = sprintf(str, *args)
       @server.stats[:bytes_written] += response.length

data/lib/starling/persistent_queue.rb

@@ -83,6 +83,11 @@ module StarlingServer
       @trx = nil
       @not_trx.close
     end
+    
+    def purge
+      close
+      File.delete(log_path)
+    end
 
     private
 

data/lib/starling/queue_collection.rb

@@ -15,7 +15,7 @@ module StarlingServer
 
     def initialize(path)
       unless File.directory?(path) && File.writable?(path)
-        raise InaccessibleQueuePath.new("#{path} must exist and be writable by #{Etc.getpwuid(Process.uid).name}.")
+        raise InaccessibleQueuePath.new("'#{path}' must exist and be read-writable by #{Etc.getpwuid(Process.uid).name}.")
       end
 
       @shutdown_mutex = Mutex.new
@@ -59,6 +59,12 @@ module StarlingServer
       @stats[:current_bytes] -= result.size
       result
     end
+    
+    def delete(key)
+      queue = @queues.delete(key)
+      return if queue.nil?
+      queue.purge
+    end
 
     ##
     # Returns all active queues.

data/lib/starling/server.rb

@@ -2,7 +2,6 @@ require 'socket'
 require 'logger'
 require 'rubygems'
 require 'eventmachine'
-require 'analyzer_tools/syslog_logger'
 
 here = File.dirname(__FILE__)
 
@@ -11,7 +10,7 @@ require File.join(here, 'handler')
 
 module StarlingServer
 
-  VERSION = "0.9.7.9"
+  VERSION = "0.9.8"
   
   class Base
     attr_reader :logger
@@ -68,14 +67,27 @@ module StarlingServer
     def run
       @stats[:start_time] = Time.now
 
-      @@logger = case @opts[:logger]
-                 when IO, String; Logger.new(@opts[:logger])
-                 when Logger; @opts[:logger]
-                 else; Logger.new(STDERR)
-                 end
-      @@logger = SyslogLogger.new(@opts[:syslog_channel]) if @opts[:syslog_channel]
+      if @opts[:syslog_channel]
+        begin
+          require 'syslog_logger'
+          @@logger = SyslogLogger.new(@opts[:syslog_channel])
+        rescue LoadError
+          # SyslogLogger isn't available, so we're just going to use Logger
+        end
+      end
 
-      @opts[:queue] = QueueCollection.new(@opts[:path])
+      @@logger ||= case @opts[:logger]
+                   when IO, String; Logger.new(@opts[:logger])
+                   when Logger; @opts[:logger]
+                   else; Logger.new(STDERR)
+                   end
+
+      begin
+        @opts[:queue] = QueueCollection.new(@opts[:path])
+      rescue InaccessibleQueuePath => e
+        puts "Error: #{e.message}"
+        exit 1
+      end
       @@logger.level = @opts[:log_level] || Logger::ERROR
 
       @@logger.info "Starling STARTUP on #{@opts[:host]}:#{@opts[:port]}"

data/lib/starling/server_runner.rb

@@ -40,6 +40,11 @@ module StarlingServer
         when "queue_path" then key = "path"
         when "log_file" then key = "logger"
         end
+
+        if %w(logger path pid_file).include?(key)
+          value = File.expand_path(value)
+        end
+
         options[key.to_sym] = value
 
         if options[:log_level].instance_of?(String)
@@ -51,11 +56,11 @@ module StarlingServer
     def parse_options
       self.options = { :host => '127.0.0.1',
                        :port => 22122,
-                       :path => File.join(%w( / tmp starling spool )),
+                       :path => File.join('', 'var', 'spool', 'starling'),
                        :log_level => Logger::INFO,
                        :daemonize => false,
                        :timeout => 0,
-                       :pid_file => File.join(%w( / tmp starling starling.pid )) }
+                       :pid_file => File.join('', 'var', 'run', 'starling.pid') }
 
       OptionParser.new do |opts|
         opts.summary_width = 25
@@ -76,7 +81,7 @@ module StarlingServer
         opts.on("-q", "--queue_path PATH",
                 :REQUIRED,
                 "Path to store Starling queue logs", "(default: #{options[:path]})") do |queue_path|
-          options[:path] = queue_path
+          options[:path] = File.expand_path(queue_path)
         end
 
         opts.separator ""; opts.separator "Network:"
@@ -96,7 +101,7 @@ module StarlingServer
         end
 
         opts.on("-PFILE", "--pid FILENAME", "save PID in FILENAME when using -d option.", "(default: #{options[:pid_file]})") do |pid_file|
-          options[:pid_file] = pid_file
+          options[:pid_file] = File.expand_path(pid_file)
         end
 
         opts.on("-u", "--user USER", "User to run as") do |user|
@@ -110,11 +115,16 @@ module StarlingServer
         opts.separator ""; opts.separator "Logging:"
 
         opts.on("-L", "--log [FILE]", "Path to print debugging information.") do |log_path|
-          options[:logger] = log_path
+          options[:logger] = File.expand_path(log_path)
         end
 
-        opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
-          options[:syslog_channel] = channel
+        begin
+          require 'syslog_logger'
+
+          opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
+            options[:syslog_channel] = channel
+          end
+        rescue LoadError
         end
 
         opts.on("-v", "Increase logging verbosity (may be used multiple times).") do
@@ -216,7 +226,7 @@ module StarlingServer
       safefork and exit
 
       unless sess_id = Process.setsid
-        raise "Couldn't detache from controlling terminal."
+        raise "Couldn't detach from controlling terminal."
       end
 
       trap 'SIGHUP', 'IGNORE'

data/{test/test_starling_server.rb → spec/starling_server_spec.rb}

@@ -4,6 +4,7 @@ require 'rubygems'
 require 'fileutils'
 require 'memcache'
 require 'digest/md5'
+require 'starling'
 
 require 'starling/server'
 
@@ -22,7 +23,7 @@ def safely_fork(&block)
   while blocking
     sleep 0.1
   end
-  
+
   pid
 end
 
@@ -41,12 +42,17 @@ describe "StarlingServer" do
                                         :path => @tmp_path,
                                         :logger => Logger.new(STDERR),
                                         :log_level => Logger::FATAL)
-      Signal.trap("INT") { server.stop }
+      Signal.trap("INT") {
+        server.stop
+        exit
+      }
+
       Process.kill("USR1", Process.ppid)
       server.run
     end
 
     @client = MemCache.new('127.0.0.1:22133')
+
   end
   
   it "should test if temp_path exists and is writeable" do
@@ -62,6 +68,14 @@ describe "StarlingServer" do
     @client.get('test_set_and_get_one_entry').should eql(v)
   end
   
+  it "should respond to delete" do
+    @client.delete("my_queue").should eql("END\r\n")   
+    starling_client = Starling.new('127.0.0.1:22133')
+    starling_client.set('my_queue', 50)
+    starling_client.available_queues.size.should eql(1)
+    starling_client.delete("my_queue")
+    starling_client.available_queues.size.should eql(0)
+  end
   
   it "should expire entries" do
     v = rand((2**32)-1)
@@ -143,45 +157,42 @@ describe "StarlingServer" do
     # handle more than 1024 connections.
     
     unless IO::popen("uname").read.chomp == "Linux"
-      raise "(Skipping epoll test: not on Linux)"
-      skip = true
+      pending "skipping epoll test: not on Linux"
     end
+
     fd_limit = IO::popen("bash -c 'ulimit -n'").read.chomp.to_i
     unless fd_limit > 1024
-      raise "(Skipping epoll test: 'ulimit -n' = #{fd_limit}, need > 1024)"
-      skip = true
+      pending "skipping epoll test: 'ulimit -n' = #{fd_limit}, need > 1024"
     end
     
-    unless skip
-      v = rand(2**32 - 1)
-      @client.set('test_epoll', v)
-
-      # we can't open 1024 connections to memcache from within this process,
-      # because we will hit ruby's 1024 fd limit ourselves!
-      pid1 = safely_fork do
-        unused_sockets = []
-        600.times do
-          unused_sockets << TCPSocket.new("127.0.0.1", 22133)
-        end
-        Process.kill("USR1", Process.ppid)
-        sleep 90
+    v = rand(2**32 - 1)
+    @client.set('test_epoll', v)
+
+    # we can't open 1024 connections to memcache from within this process,
+    # because we will hit ruby's 1024 fd limit ourselves!
+    pid1 = safely_fork do
+      unused_sockets = []
+      600.times do
+        unused_sockets << TCPSocket.new("127.0.0.1", 22133)
       end
-      pid2 = safely_fork do
-        unused_sockets = []
-        600.times do
-          unused_sockets << TCPSocket.new("127.0.0.1", 22133)
-        end
-        Process.kill("USR1", Process.ppid)
-        sleep 90
+      Process.kill("USR1", Process.ppid)
+      sleep 90
+    end
+    pid2 = safely_fork do
+      unused_sockets = []
+      600.times do
+        unused_sockets << TCPSocket.new("127.0.0.1", 22133)
       end
+      Process.kill("USR1", Process.ppid)
+      sleep 90
+    end
 
-      begin
-        client = MemCache.new('127.0.0.1:22133')
-        client.get('test_epoll').should eql(v)
-      ensure
-        Process.kill("TERM", pid1)
-        Process.kill("TERM", pid2)
-      end
+    begin
+      client = MemCache.new('127.0.0.1:22133')
+      client.get('test_epoll').should eql(v)
+    ensure
+      Process.kill("TERM", pid1)
+      Process.kill("TERM", pid2)
     end
   end
   
@@ -202,4 +213,4 @@ describe "StarlingServer" do
     @client.reset
     FileUtils.rm(Dir.glob(File.join(@tmp_path, '*')))
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: starling-starling
 version: !ruby/object:Gem::Version 
-  version: 0.9.7.10
+  version: 0.9.8.200810061510
 platform: ruby
 authors: 
 - Blaine Cook
@@ -13,20 +13,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-08-13 00:00:00 -07:00
+date: 2008-10-09 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: fiveruns-memcache-client
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: SyslogLogger
+  name: memcache-client
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -70,6 +61,7 @@ files:
 - lib/starling.rb
 - etc/starling.redhat
 - etc/starling.ubuntu
+- etc/sample-config.yml
 has_rdoc: true
 homepage: http://github.com/starling/starling/
 post_install_message: 
@@ -105,4 +97,4 @@ signing_key:
 specification_version: 2
 summary: Starling is a lightweight, transactional, distributed queue server
 test_files: 
-- test/test_starling_server.rb
+- spec/starling_server_spec.rb