fiveruns-starling 0.9.7.5

data/CHANGELOG

@@ -0,0 +1,27 @@
+== 0.9.7.5 2008-05-04
+* added worker class, using starling client you can now run them in background
+ - handles creation of threadpool
+ - handles fetching of messages and passing these to the threads
+ - handles pushing of processed messages to starling again if needed
+
+== 0.9.7 2008-05-03
+* merged branch of AnotherBritt and Glenn Rempe by Abdul-Rahman Advany <abdulrahman@advany.com>
+
+==  0.9.6 2008-04-30
+* logging of message lifecycle as :age by AnotherBritt <?>
+* added some extra logging options by AnotherBritt <?>
+* added some test for epoll by AnotherBritt <?>
+
+== 0.9.5.4 2008-04-28
+
+* Bumped version number by Glenn Rempe <glenn.rempe@gmail.com>
+* Purged all old RubyForge config.  Now GitHub friendly by Glenn Rempe <glenn.rempe@gmail.com>
+* New gemspec for GitHub gem auto-build-serve by Glenn Rempe <glenn.rempe@gmail.com>
+
+== 0.9.4 2008-01-31 ==
+* Evented code added using EventMachine by Chris Wanstrath <chris@ozmm.org>
+
+== 2007-11-02
+
+* Initial release
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2007 FIXME full name
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,93 @@
+= Name
+
+Starling - a light weight server for reliable distributed message passing.
+
+= Description
+
+Starling is a powerful but simple messaging server that enables reliable
+distributed queuing with an absolutely minimal overhead. It speaks the
+MemCache protocol for maximum cross-platform compatibility. Any language
+that speaks MemCache can take advantage of Starling's queue facilities.
+
+
+= Installation
+
+  This fork of the Starling source is hosted at GitHub and can be found at:
+
+  http://github.com/advany/starling/tree/master
+
+  The original source was to be found at RubyForge but no longer exists there.
+
+  GitHub serves gems prefixed by a username to differentiate different forks.
+  This project can be installed with:
+
+  # THIS COMMAND ONE TIME ONLY
+  gem sources -a http://gems.github.com/
+
+  # As often as you like
+  sudo gem install advany-starling
+
+  See http://gems.github.com/ if you want more info about GitHub and gems.
+
+= Quick Start Usage
+
+  # View the Starling help and usage message
+  starling --help
+
+  # In a console window start the Starling server.  By default
+  # it runs verbosely in the foreground, listening on 127.0.0.1:22122
+  # and stores its files under /tmp/starling:
+  starling
+
+  # In a new console test the put and get of messages on a queue:
+
+  irb
+  >> require 'starling'
+  => true
+  >> starling = Starling.new('127.0.0.1:22122')
+  => MemCache: 1 servers, 1 buckets, ns: nil, ro: false
+  >> starling.set('my_queue', 12345)
+  => nil
+  >> starling.get('my_queue')
+  => 12345
+
+  # You can do a simple loop over a queue with something like:
+  >> loop { puts starling.get('my_queue'); sleep 1 }
+  12345
+  nil
+  nil
+  ...
+
+  For more information run the following in a new console:
+  
+  'gem server'
+
+  This will start a gem server on http://localhost:8808/ which you can view in your
+  browser to see the RDocs for the gem. Or generate rdocs by running the following
+  in a new console:
+	
+  'rdoc'
+
+= Known Issues
+
+* Starling is "slow" as far as messaging systems are concerned. In practice,
+  it's fast enough.
+
+== TODO
+
+* Implement memcached in stead of memcache as a client interface
+* Improve client structure (with workers - simplify and add defaults)
+* Fix running starling_client as deamonized process (can't seem to fork inside a fork)
+
+= Authors
+
+* Blaine Cook <romeda@gmail.com>
+* Chris Wanstrath <chris@ozmm.org>
+* AnotherBritt <?>
+* Glenn Rempe <?>
+* Abdul-Rahman Advany <abdulrahman@advany.com>
+
+= Copyright
+
+Starling - a light-weight server for reliable distributed message passing.
+Copyright 2007-2008 Blaine Cook <blaine@twitter.com>, Twitter Inc.

data/Rakefile

@@ -0,0 +1,21 @@
+require 'rubygems'
+require 'rake/rdoctask'
+require 'spec/rake/spectask'
+
+task :install do
+  sh %{gem build starling.gemspec}
+  sh %{sudo gem install starling-*.gem}
+end
+
+Spec::Rake::SpecTask.new do |t|
+  t.ruby_opts = ['-rtest/unit']
+  t.spec_files = FileList['test/test_*.rb']
+  t.fail_on_error = true
+end
+  
+Rake::RDocTask.new do |rd|
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+  rd.rdoc_dir = 'doc'
+#  rd.options = spec.rdoc_options
+end

data/bin/starling

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'starling/server_runner'
+StarlingServer::Runner.run
+

data/bin/starling_client

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+# $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'starling/client_runner'
+StarlingClient::Runner.run

data/bin/starling_top

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'ostruct'
+require 'socket'
+
+@options = OpenStruct.new
+@options.hostname = 'localhost'
+@options.port = 11211
+
+op = OptionParser.new do |opts|
+	opts.banner = "STARLING TOP\nUsage: startop [options]"
+	opts.separator "General Options:"
+	opts.on("-h HOSTNAME", "--hostname=HOSTNAME", "Hostname [default: localhost]") do |h|
+		@options.hostname = h
+	end
+	opts.on("-p PORT", "--port=PORT", Integer, "Port [default: 11211]") do |p|
+		@options.port = p
+	end
+	opts.on_tail("--help", "Show this message") do
+		puts opts
+		exit
+	end
+end
+op.parse!
+
+
+def stats_data
+  data = ''
+  sock = TCPSocket.new(@options.hostname, @options.port)
+  sock.print("stats\r\n")
+  sock.flush
+  # memcached does not close the socket once it is done writing
+  # the stats data.  We need to read line by line until we detect
+  # the END line and then stop/close on our side.
+  stats = sock.gets
+  while true
+    data += stats
+    break if stats.strip == 'END'
+    stats = sock.gets
+  end
+  sock.close
+  data
+end
+
+def parse(stats_data)
+  stats = []
+  stats_data.each_line do |line|
+    stats << "#{$1}: #{$2}" if line =~ /STAT (\w+) (\S+)/
+  end
+  stats.sort
+end
+
+stats = parse(stats_data)
+stats.each do |stat|
+	puts stat
+end

data/etc/starling.redhat

@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# starling    This shell script takes care of starting and stopping
+#                    the starling server
+# chkconfig: 345 98 98
+# description: The starling queue server
+
+#determine where the 'pidof' executable is located
+if [ -e /bin/pidof ]; then
+  PIDOF="/bin/pidof"
+elif [ -e /sbin/pidof ]; then
+  PIDOF="/sbin/pidof"
+elif [ -e /usr/local/bin/pidof ]; then
+  PIDOF="/usr/local/bin/pidof"
+elif [ -e /bin/pgrep ]; then
+  PIDOF="/bin/pgrep"
+elif [ -e /usr/bin/pgrep ]; then
+  PIDOF="/usr/bin/pgrep"
+elif [ -e /usr/local/bin/pgrep ]; then
+  PIDOF="/usr/local/bin/pgrep"
+else
+  echo "Could not find pidof or pgrep"
+fi
+
+PROGDIR="/usr/bin"
+PROGNAME="starling"
+OPTIONS="-u nobody -g nobody -L /var/log/starling.log -q /var/spool/starling"
+
+start() {
+  pid=`$PIDOF $PROGNAME`
+  if [ "$pid" != "" ]; then
+    echo "$PROGDIR$PROGNAME already running: $pid"
+  else
+    echo "Starting $PROGDIR$PROGNAME"
+    cd $PROGDIR
+    nohup $PROGDIR$PROGNAME $OPTIONS &
+  fi
+}
+
+stop() {
+  pid=`$PIDOF $PROGNAME`
+  if [ "$pid" != "" ]; then
+    echo "Stopping $PROGDIR$PROGNAME: $pid"
+    kill $pid
+  else
+    echo "$PROGDIR$PROGNAME not running"
+  fi
+}
+
+case "$1" in
+start)
+  start
+;;
+stop)
+  stop
+;;
+restart)
+  stop
+  sleep 3
+  start
+;;
+
+esac

data/etc/starling.ubuntu

@@ -0,0 +1,71 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides: starling
+# Required-Start: $local_fs $remote_fs
+# Required-Stop: $local_fs $remote_fs
+# Default-Start: 2 3 4 5
+# Default-Stop: S 0 1 6
+# Short-Description: Starling queue server
+# Description: The Starling distributed, transactional queue server
+### END INIT INFO
+# Author: Twitter
+# Version: 0.9.7.5
+
+set -e
+
+DUSER=starling
+DGROUP=starling
+LOGFILE=/var/log/starling/starling.log
+SPOOLDIR=/var/spool/starling
+PORT=22122
+LISTEN=0.0.0.0
+PIDFILE=/var/run/starling/starling.pid
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=starling
+DESC="Starling"
+INSTALL_DIR=/usr/local/bin
+DAEMON=$INSTALL_DIR/$NAME
+SCRIPTNAME=/etc/init.d/$NAME
+OPTS="-h $LISTEN -p $PORT -d -q $SPOOLDIR -P $PIDFILE -L $LOGFILE"
+
+. /lib/lsb/init-functions
+
+
+# Gracefully exit if the package has been removed.
+test -x $DAEMON || exit 0
+
+d_start() {
+	log_begin_msg "Starting Starling Server..."
+		start-stop-daemon -c $DUSER:$DGROUP --start --quiet --pidfile $PIDFILE --exec $DAEMON \
+		-- $OPTS || log_end_msg 1
+		log_end_msg 0
+}
+
+d_stop() {
+	log_begin_msg "Stopping Starling Server..."
+		start-stop-daemon -c $DUSER:$DGROUP --stop --quiet --pidfile $PIDFILE \
+		|| log_end_msg 1
+		log_end_msg 0
+}
+
+case "$1" in
+start)
+d_start
+;;
+stop)
+d_stop
+;;
+restart|force-reload|reload)
+d_stop
+sleep 2
+d_start
+;;
+*)
+echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2
+exit 3
+;;
+esac
+
+exit 0
+

data/lib/starling.rb

@@ -0,0 +1,104 @@
+require 'memcache'
+
+class Starling < MemCache
+
+  WAIT_TIME = 0.25
+
+  ##
+  # fetch an item from a queue.
+
+  def get(*args)
+    loop do
+      response = super(*args)
+      return response unless response.nil?
+      sleep WAIT_TIME
+    end
+  end
+
+  ##
+  # insert +value+ into +queue+.
+  #
+  # +expiry+ is expressed as a UNIX timestamp
+  #
+  # If +raw+ is true, +value+ will not be Marshalled. If +raw+ = :yaml, +value+
+  # will be serialized with YAML, instead.
+
+  def set(queue, value, expiry = 0, raw = false)
+    retries = 0
+    begin
+      if raw == :yaml
+        value = YAML.dump(value)
+        raw = true
+      end
+
+      super(queue, value, expiry, raw)
+    rescue MemCache::MemCacheError => e
+      retries += 1
+      sleep WAIT_TIME
+      retry unless retries > 3
+      raise e
+    end
+  end
+
+  ##
+  # returns the number of items in +queue+. If +queue+ is +:all+, a hash of all
+  # queue sizes will be returned.
+
+  def sizeof(queue, statistics = nil)
+    statistics ||= stats
+
+    if queue == :all
+      queue_sizes = {}
+      available_queues(statistics).each do |queue|
+        queue_sizes[queue] = sizeof(queue, statistics)
+      end
+      return queue_sizes
+    end
+
+    statistics.inject(0) { |m,(k,v)| m + v["queue_#{queue}_items"].to_i }
+  end
+
+  ##
+  # returns a list of available (currently allocated) queues.
+
+  def available_queues(statistics = nil)
+    statistics ||= stats
+
+    statistics.map { |k,v|
+      v.keys
+    }.flatten.uniq.grep(/^queue_(.*)_items/).map { |v|
+      v.gsub(/^queue_/, '').gsub(/_items$/, '')
+    }.reject { |v|
+      v =~ /_total$/ || v =~ /_expired$/
+    }
+  end
+
+  ##
+  # iterator to flush +queue+. Each element will be passed to the provided
+  # +block+
+
+  def flush(queue)
+    sizeof(queue).times do
+      v = get(queue)
+      yield v if block_given?
+    end
+  end
+
+  private
+
+  def get_server_for_key(key)
+    raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
+    raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
+    raise MemCacheError, "No servers available" if @servers.empty?
+
+    bukkits = @buckets.dup
+    bukkits.nitems.times do |try|
+      n = rand(bukkits.nitems)
+      server = bukkits[n]
+      return server if server.alive?
+      bukkits.delete_at(n)
+    end
+
+    raise MemCacheError, "No servers available (all dead)"
+  end
+end