starling 0.9.8 → 0.10.0

data/CHANGELOG

@@ -1,3 +1,17 @@
+== 0.10.1 (future)
+* Fixed a potential privilege escalation issue <jdamato@alumni.cmu.edu>
+
+== 0.10.0
+* Solved compatibility issues with the new (and awesome, thanks mperham) Memcache client
+* Fixed the old rspec tests.
+
+== 0.9.9
+* remove dependency on SyslogLogger so that starling works on windows.
+* fix config file loading so relative paths are expanded properly <jacob@jacobatzen.dk>
+* clean up redhat init.d script <gaffneyc@gmail.com>
+* add a 'fetch' command that is non-blocking equivalent to 'get' <aaron.hawkins@shawkvalue.com>
+* implement the 'delete' method to allow queues to be deleted <timshadel@gmail.com>
+
 == 0.9.8
 * add fix to enable relative paths <david@motorator.com>
 * fix tests so they don't run 10 times due to a stupid bug with how the server is forked <seth@mojodna.net>

data/README.rdoc

@@ -11,7 +11,7 @@ that speaks MemCache can take advantage of Starling's queue facilities.
 
 = Installation
 
-  This fork of the Starling source is hosted at GitHub and can be found at:
+  The Starling source is hosted at GitHub and can be found at:
 
   http://github.com/starling/starling/tree/master
 
@@ -44,7 +44,7 @@ that speaks MemCache can take advantage of Starling's queue facilities.
   >> require 'starling'
   => true
   >> starling = Starling.new('127.0.0.1:22122')
-  => MemCache: 1 servers, 1 buckets, ns: nil, ro: false
+  => MemCache: 1 servers, ns: nil, ro: false
   >> starling.set('my_queue', 12345)
   => nil
   >> starling.get('my_queue')
@@ -67,13 +67,13 @@ that speaks MemCache can take advantage of Starling's queue facilities.
 	
   'rdoc'
 
-= Using fiveruns memcache-client
+= Using memcache-client
 
-	memcache-client from fiveruns has a couple of fixed added like supporting failover and retry on failure.
+	memcache-client is now maintained by Mike Perham (http://www.mikeperham.com/)
 	
-	This fork of the memcache-client source is hosted at GitHub and can be found at:
+	He is the current maintainer of the gem which is hosted at GitHub and can be found at:
 
-	http://github.com/fiveruns/memcache-client/tree/master
+  http://github.com/mperham/memcache-client/tree/master
 
 	It can be installed using:
 
@@ -81,10 +81,11 @@ that speaks MemCache can take advantage of Starling's queue facilities.
 	gem sources -a http://gems.github.com/
 
 	# As often as you like
-	sudo gem install fiveruns-memcache-client
+	sudo gem install memcache-client
 
 = Known Issues
 
+* Only tested with Memcache client 1.7.x and with SystemTimer (http://systemtimer.rubyforge.org/)
 * Starling is "slow" as far as messaging systems are concerned. In practice,
   it's fast enough.
 
@@ -99,6 +100,7 @@ that speaks MemCache can take advantage of Starling's queue facilities.
 * AnotherBritt <?>
 * Glenn Rempe <?>
 * Abdul-Rahman Advany <abdulrahman@advany.com>
+* Harm Aarts <harmaarts@gmail.com>
 
 = Copyright
 

data/Rakefile

@@ -1,22 +1,51 @@
 require 'rubygems'
-require 'rake/rdoctask'
-require 'spec/rake/spectask'
-
-task :default => :spec
+require 'rake'
 
-task :install do
-  sh %{gem build starling.gemspec}
-  sh %{sudo gem install starling-*.gem}
+begin
+  require 'jeweler'
+  
+  Jeweler::Tasks.new do |s|
+    s.name = "starling"
+    s.version = "0.10.0"
+    s.authors = ["Blaine Cook", "Chris Wanstrath", "Britt Selvitelle", "Glenn Rempe", "Abdul-Rahman Advany", "Seth Fitzsimmons", "Harm Aarts", "Chris Gaffney"]
+    s.email = ["blaine@twitter.com", "chris@ozmm.org", "abdulrahman@advany.com", "starlingmq@groups.google.com", "harmaarts@gmail.com", "gaffneyc@gmail.com"]
+    s.homepage = "http://github.com/starling/starling/"
+    s.summary = "Starling is a lightweight, transactional, distributed queue server"
+    s.description = s.summary
+    
+    s.executables = ["starling", "starling_top"]
+    s.require_paths = ["lib"]
+    
+    s.has_rdoc = true
+    s.rdoc_options = ["--quiet", "--title", "starling documentation", "--opname", "index.html", "--line-numbers", "--main", "README.rdoc", "--inline-source"]
+    s.extra_rdoc_files = ["README.rdoc", "CHANGELOG", "LICENSE"]
+    
+    s.add_dependency 'memcache-client', [">= 1.7.0"]
+    s.add_dependency 'eventmachine', [">= 0.12.0"]
+  end
+  
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
 end
 
-Spec::Rake::SpecTask.new do |t|
-  t.ruby_opts = ['-rtest/unit']
-  t.spec_files = FileList['spec/*_spec.rb']
-  t.fail_on_error = true
-end
-  
+require 'rake/rdoctask'
 Rake::RDocTask.new do |rd|
   rd.main = "README.rdoc"
   rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
   rd.rdoc_dir = 'doc'
 end
+
+task :default => :spec
+
+begin
+  require 'spec/rake/spectask'
+  
+  Spec::Rake::SpecTask.new do |t|
+    t.ruby_opts = ['-rtest/unit']
+    t.spec_files = FileList['spec/*_spec.rb']
+    t.fail_on_error = true
+  end
+rescue LoadError
+   puts "RSpec (or a dependency) not available. Install it with: sudo gem install rspec"
+end

data/etc/starling.redhat

@@ -1,63 +1,66 @@
 #!/bin/bash
 #
+# Make sure the /var/run/starling, /var/log/starling and /var/spool/starling
+# all exist and are owned by starling:starling
+#
 # starling    This shell script takes care of starting and stopping
 #                    the starling server
 # chkconfig: 345 98 98
 # description: The starling queue server
+# processname: starling
+# pidfile: /var/run/starling/starling.pid
+# logfile: /var/log/starling/starling.log
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+# Source networking configuration.
+[ -f /etc/sysconfig/network ] && . /etc/sysconfig/network
+
+DUSER=starling
+DGROUP=starling
+LOGFILE=/var/log/starling/starling.log
+SPOOLDIR=/var/spool/starling
+PORT=22122
+LISTEN=0.0.0.0
+PIDFILE=/var/run/starling/starling.pid
 
-#determine where the 'pidof' executable is located
-if [ -e /bin/pidof ]; then
-  PIDOF="/bin/pidof"
-elif [ -e /sbin/pidof ]; then
-  PIDOF="/sbin/pidof"
-elif [ -e /usr/local/bin/pidof ]; then
-  PIDOF="/usr/local/bin/pidof"
-elif [ -e /bin/pgrep ]; then
-  PIDOF="/bin/pgrep"
-elif [ -e /usr/bin/pgrep ]; then
-  PIDOF="/usr/bin/pgrep"
-elif [ -e /usr/local/bin/pgrep ]; then
-  PIDOF="/usr/local/bin/pgrep"
-else
-  echo "Could not find pidof or pgrep"
-fi
-
-PROGDIR="/usr/bin"
-PROGNAME="starling"
-OPTIONS="-u nobody -g nobody -L /var/log/starling.log -q /var/spool/starling"
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=starling
+INSTALL_DIR=/usr/local/bin
+DAEMON=$INSTALL_DIR/$NAME
+OPTS="-h $LISTEN -p $PORT -d -q $SPOOLDIR -P $PIDFILE -L $LOGFILE -u $DUSER -g $DGROUP"
 
 start() {
-  pid=`$PIDOF $PROGNAME`
-  if [ "$pid" != "" ]; then
-    echo "$PROGDIR$PROGNAME already running: $pid"
-  else
-    echo "Starting $PROGDIR$PROGNAME"
-    cd $PROGDIR
-    nohup $PROGDIR$PROGNAME $OPTIONS &
-  fi
+  echo -n $"Starting starling: "
+
+  daemon --pidfile=$PIDFILE $DAEMON $OPTS
+  echo
 }
 
 stop() {
-  pid=`$PIDOF $PROGNAME`
-  if [ "$pid" != "" ]; then
-    echo "Stopping $PROGDIR$PROGNAME: $pid"
-    kill $pid
-  else
-    echo "$PROGDIR$PROGNAME not running"
-  fi
+  echo -n $"Stopping starling: "
+
+  killproc -p $PIDFILE starling
+  echo
 }
 
 case "$1" in
-start)
-  start
-;;
-stop)
-  stop
-;;
-restart)
-  stop
-  sleep 3
-  start
-;;
-
+  start)
+    start
+  ;;
+  stop)
+    stop
+  ;;
+  restart)
+    stop
+    sleep 3
+    start
+  ;;
+  status)
+    status -p $PIDFILE starling
+  ;;
+  *)
+    echo $"Usage: $0 {start|stop|restart|status}"
+    exit 1
 esac

data/etc/starling.ubuntu

@@ -9,7 +9,7 @@
 # Description: The Starling distributed, transactional queue server
 ### END INIT INFO
 # Author: Twitter
-# Version: 0.9.7.7
+# Version: 0.10.0
 
 set -e
 

data/lib/starling.rb

@@ -3,18 +3,53 @@ require 'memcache'
 class Starling < MemCache
 
   WAIT_TIME = 0.25
+  alias_method :_original_get, :get
+  alias_method :_original_delete, :delete
 
   ##
   # fetch an item from a queue.
 
   def get(*args)
     loop do
-      response = super(*args)
+      response = _original_get(*args)
       return response unless response.nil?
       sleep WAIT_TIME
     end
   end
+  
+  ##
+  # will return the next item or nil
+
+  def fetch(*args)
+    _original_get(*args)
+  end
+
+  ##
+  # Delete the key (queue) from all Starling servers. This is necessary
+  # because the random way a server is chosen in #get_server_for_key
+  # implies that the queue could easily be spread across the entire
+  # Starling cluster.
+
+  def delete(key, expiry = 0)
+    with_servers do
+      _original_delete(key, expiry)
+    end
+  end
 
+  ##
+  # Provides a way to work with a specific list of servers by
+  # forcing all calls to #get_server_for_key to use a specific
+  # server, and changing that server each time that the call
+  # yields to the block provided.  This helps work around the
+  # normally random nature of the #get_server_for_key method.
+  #
+  def with_servers(my_servers = self.servers.dup)
+    return unless block_given?
+    my_servers.each do |server|
+      yield
+    end
+  end
+  
   ##
   # insert +value+ into +queue+.
   #
@@ -89,16 +124,17 @@ class Starling < MemCache
   def get_server_for_key(key)
     raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
     raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
-    raise MemCacheError, "No servers available" if @servers.empty?
-
-    bukkits = @buckets.dup
-    bukkits.nitems.times do |try|
-      n = rand(bukkits.nitems)
-      server = bukkits[n]
+    raise MemCacheError, "No servers available" if servers.empty?
+ 
+    # Ignores server weights, oh well
+    srvs = servers.dup
+    srvs.size.times do |try|
+      n = rand(srvs.size)
+      server = srvs[n]
       return server if server.alive?
-      bukkits.delete_at(n)
+      srvs.delete_at(n)
     end
-
+  
     raise MemCacheError, "No servers available (all dead)"
   end
-end
+end

data/lib/starling/handler.rb

@@ -22,36 +22,41 @@ module StarlingServer
     SET_RESPONSE_SUCCESS  = "STORED\r\n".freeze
     SET_RESPONSE_FAILURE  = "NOT STORED\r\n".freeze
     SET_CLIENT_DATA_ERROR = "CLIENT_ERROR bad data chunk\r\nERROR\r\n".freeze
+    
+    # DELETE Responses
+    DELETE_COMMAND = /\Adelete (.{1,250}) ([0-9]+)\r\n/m
+    DELETE_RESPONSE = "END\r\n".freeze
 
     # STAT Response
     STATS_COMMAND = /\Astats\r\n/m
-    STATS_RESPONSE = "STAT pid %d
-STAT uptime %d
-STAT time %d
-STAT version %s
-STAT rusage_user %0.6f
-STAT rusage_system %0.6f
-STAT curr_items %d
-STAT total_items %d
-STAT bytes %d
-STAT curr_connections %d
-STAT total_connections %d
-STAT cmd_get %d
-STAT cmd_set %d
-STAT get_hits %d
-STAT get_misses %d
-STAT bytes_read %d
-STAT bytes_written %d
-STAT limit_maxbytes %d
+    STATS_RESPONSE = "STAT pid %d\r
+STAT uptime %d\r
+STAT time %d\r
+STAT version %s\r
+STAT rusage_user %0.6f\r
+STAT rusage_system %0.6f\r
+STAT curr_items %d\r
+STAT total_items %d\r
+STAT bytes %d\r
+STAT curr_connections %d\r
+STAT total_connections %d\r
+STAT cmd_get %d\r
+STAT cmd_set %d\r
+STAT get_hits %d\r
+STAT get_misses %d\r
+STAT bytes_read %d\r
+STAT bytes_written %d\r
+STAT limit_maxbytes %d\r
 %sEND\r\n".freeze
-    QUEUE_STATS_RESPONSE = "STAT queue_%s_items %d
-STAT queue_%s_total_items %d
-STAT queue_%s_logsize %d
-STAT queue_%s_expired_items %d
+    QUEUE_STATS_RESPONSE = "STAT queue_%s_items %d\r
+STAT queue_%s_total_items %d\r
+STAT queue_%s_logsize %d\r
+STAT queue_%s_expired_items %d\r
 STAT queue_%s_age %d\r\n".freeze
 
     SHUTDOWN_COMMAND = /\Ashutdown\r\n/m
 
+    QUIT_COMMAND = /\Aquit\r\n/m
 
     @@next_session_id = 1
 
@@ -108,7 +113,6 @@ STAT queue_%s_age %d\r\n".freeze
         @data_buf = data
         return
       end
-
       case data
       when SET_COMMAND
         @server.stats[:set_requests] += 1
@@ -121,6 +125,11 @@ STAT queue_%s_age %d\r\n".freeze
       when SHUTDOWN_COMMAND
         # no point in responding, they'll never get it.
         Runner::shutdown
+      when DELETE_COMMAND
+        delete $1
+      when QUIT_COMMAND
+        # ignore the command, client is closing connection.
+        return nil
       else
         logger.warn "Unknown command: #{data}."
         respond ERR_UNKNOWN_COMMAND
@@ -136,6 +145,12 @@ STAT queue_%s_age %d\r\n".freeze
     end
 
   private
+  
+    def delete(queue)
+      @queue_collection.delete(queue)
+      respond DELETE_RESPONSE
+    end
+  
     def respond(str, *args)
       response = sprintf(str, *args)
       @server.stats[:bytes_written] += response.length

data/lib/starling/persistent_queue.rb

@@ -83,6 +83,11 @@ module StarlingServer
       @trx = nil
       @not_trx.close
     end
+    
+    def purge
+      close
+      File.delete(log_path)
+    end
 
     private
 

data/lib/starling/queue_collection.rb

@@ -59,6 +59,12 @@ module StarlingServer
       @stats[:current_bytes] -= result.size
       result
     end
+    
+    def delete(key)
+      queue = @queues.delete(key)
+      return if queue.nil?
+      queue.purge
+    end
 
     ##
     # Returns all active queues.

data/lib/starling/server.rb

@@ -2,7 +2,6 @@ require 'socket'
 require 'logger'
 require 'rubygems'
 require 'eventmachine'
-require 'analyzer_tools/syslog_logger'
 
 here = File.dirname(__FILE__)
 
@@ -11,7 +10,7 @@ require File.join(here, 'handler')
 
 module StarlingServer
 
-  VERSION = "0.9.8"
+  VERSION = "0.10.0"
   
   class Base
     attr_reader :logger
@@ -68,12 +67,20 @@ module StarlingServer
     def run
       @stats[:start_time] = Time.now
 
-      @@logger = case @opts[:logger]
-                 when IO, String; Logger.new(@opts[:logger])
-                 when Logger; @opts[:logger]
-                 else; Logger.new(STDERR)
-                 end
-      @@logger = SyslogLogger.new(@opts[:syslog_channel]) if @opts[:syslog_channel]
+      if @opts[:syslog_channel]
+        begin
+          require 'syslog_logger'
+          @@logger = SyslogLogger.new(@opts[:syslog_channel])
+        rescue LoadError
+          # SyslogLogger isn't available, so we're just going to use Logger
+        end
+      end
+
+      @@logger ||= case @opts[:logger]
+                   when IO, String; Logger.new(@opts[:logger])
+                   when Logger; @opts[:logger]
+                   else; Logger.new(STDERR)
+                   end
 
       begin
         @opts[:queue] = QueueCollection.new(@opts[:path])

data/lib/starling/server_runner.rb

@@ -1,6 +1,9 @@
 require File.join(File.dirname(__FILE__), 'server')
+require 'erb'
+require 'fileutils'
 require 'optparse'
 require 'yaml'
+require 'fileutils'
 
 module StarlingServer
   class Runner
@@ -34,12 +37,29 @@ module StarlingServer
     end
 
     def load_config_file(filename)
-      YAML.load(File.open(filename))['starling'].each do |key, value|
+      config = YAML.load(ERB.new(File.read(filename)).result)
+
+      unless config.is_a?(Hash)
+        STDERR.puts "Config file does not contain a hash: #{filename}, exiting."
+        exit(1)
+      end
+
+      if config['starling'].nil?
+        STDERR.puts "Missing starling section in config file: #{filename}, exiting."
+        exit(1)
+      end
+
+      config['starling'].each do |key, value|
         # alias some keys
         case key
         when "queue_path" then key = "path"
         when "log_file" then key = "logger"
         end
+
+        if %w(logger path pid_file).include?(key)
+          value = File.expand_path(value)
+        end
+
         options[key.to_sym] = value
 
         if options[:log_level].instance_of?(String)
@@ -113,8 +133,13 @@ module StarlingServer
           options[:logger] = File.expand_path(log_path)
         end
 
-        opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
-          options[:syslog_channel] = channel
+        begin
+          require 'syslog_logger'
+
+          opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
+            options[:syslog_channel] = channel
+          end
+        rescue LoadError
         end
 
         opts.on("-v", "Increase logging verbosity (may be used multiple times).") do
@@ -157,8 +182,8 @@ module StarlingServer
     end
 
     def drop_privileges
-      Process.egid = options[:group] if options[:group]
-      Process.euid = options[:user] if options[:user]
+      Process::Sys.setgid(options[:group]) if options[:group]
+      Process::Sys.setuid(options[:user]) if options[:user]
     end
 
     def shutdown

data/spec/starling_server_spec.rb

@@ -4,6 +4,7 @@ require 'rubygems'
 require 'fileutils'
 require 'memcache'
 require 'digest/md5'
+require 'starling'
 
 require 'starling/server'
 
@@ -51,6 +52,7 @@ describe "StarlingServer" do
     end
 
     @client = MemCache.new('127.0.0.1:22133')
+
   end
   
   it "should test if temp_path exists and is writeable" do
@@ -66,6 +68,15 @@ describe "StarlingServer" do
     @client.get('test_set_and_get_one_entry').should eql(v)
   end
   
+  it "should respond to delete" do
+    @client.delete("my_queue").should eql("END\r\n")   
+    starling_client = Starling.new('127.0.0.1:22133')
+    starling_client.set('my_queue', 50)
+    starling_client.available_queues.size.should eql(1)
+    starling_client.delete("my_queue")
+    starling_client.available_queues.size.should eql(0)
+  end
+  
   it "should expire entries" do
     v = rand((2**32)-1)
     @client.get('test_set_with_expiry').should be_nil
@@ -113,8 +124,8 @@ describe "StarlingServer" do
 
   it "should output statistics per server" do
     stats = @client.stats
-    assert_kind_of Hash, stats
-    assert stats.has_key?('127.0.0.1:22133')
+    stats.kind_of? Hash
+    stats.has_key?('127.0.0.1:22133').should be_true
     
     server_stats = stats['127.0.0.1:22133']
     
@@ -129,8 +140,10 @@ describe "StarlingServer" do
   end
 
   it "should return valid response with unkown command" do
-    response = @client.add('blah', 1)
-    response.should eql("CLIENT_ERROR bad command line format\r\n")
+    #why is this an unknown command?
+    lambda {
+      response = @client.add('blah', 1)
+    }.should raise_error(MemCache::MemCacheError)
   end
 
   it "should disconnect and reconnect again" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: starling
 version: !ruby/object:Gem::Version 
-  version: 0.9.8
+  version: 0.10.0
 platform: ruby
 authors: 
 - Blaine Cook
@@ -9,11 +9,14 @@ authors:
 - Britt Selvitelle
 - Glenn Rempe
 - Abdul-Rahman Advany
+- Seth Fitzsimmons
+- Harm Aarts
+- Chris Gaffney
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2008-08-26 00:00:00 -07:00
+date: 2010-01-20 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -24,17 +27,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: "0"
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: SyslogLogger
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+        version: 1.7.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: eventmachine
@@ -51,6 +44,9 @@ email:
 - blaine@twitter.com
 - chris@ozmm.org
 - abdulrahman@advany.com
+- starlingmq@groups.google.com
+- harmaarts@gmail.com
+- gaffneyc@gmail.com
 executables: 
 - starling
 - starling_top
@@ -61,21 +57,26 @@ extra_rdoc_files:
 - CHANGELOG
 - LICENSE
 files: 
-- README.rdoc
-- LICENSE
+- bin/starling
+- bin/starling_top
 - CHANGELOG
-- Rakefile
+- etc/sample-config.yml
+- etc/starling.redhat
+- etc/starling.ubuntu
 - lib/starling/handler.rb
 - lib/starling/persistent_queue.rb
 - lib/starling/queue_collection.rb
-- lib/starling/server_runner.rb
 - lib/starling/server.rb
+- lib/starling/server_runner.rb
 - lib/starling.rb
-- etc/starling.redhat
-- etc/starling.ubuntu
-- etc/sample-config.yml
+- LICENSE
+- Rakefile
+- README.rdoc
+- spec/starling_server_spec.rb
 has_rdoc: true
 homepage: http://github.com/starling/starling/
+licenses: []
+
 post_install_message: 
 rdoc_options: 
 - --quiet
@@ -104,9 +105,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.2.0
+rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: Starling is a lightweight, transactional, distributed queue server
-test_files: 
-- spec/starling_server_spec.rb
+test_files: []
+