kim-toms-starling 1.0.0

data/CHANGELOG

@@ -0,0 +1,60 @@
+== 0.10.0
+* Solved compatibility issues with the new (and awesome, thanks mperham) Memcache client
+* Fixed the old rspec tests.
+
+== 0.9.9
+* remove dependency on SyslogLogger so that starling works on windows.
+* fix config file loading so relative paths are expanded properly <jacob@jacobatzen.dk>
+* clean up redhat init.d script <gaffneyc@gmail.com>
+* add a 'fetch' command that is non-blocking equivalent to 'get' <aaron.hawkins@shawkvalue.com>
+* implement the 'delete' method to allow queues to be deleted <timshadel@gmail.com>
+
+== 0.9.8
+* add fix to enable relative paths <david@motorator.com>
+* fix tests so they don't run 10 times due to a stupid bug with how the server is forked <seth@mojodna.net>
+* fix some other tests <romeda@gmail.com>
+* fix some error messages <romeda@gmail.com>
+* probably some other things <romeda@gmail.com>
+
+== 0.9.7.9
+* properly complain if the spool directory isn't writable <seth@mojodna.net>
+* assume group and user privileges in a working order <seth@mojodna.net>
+* support string user / group names in addition to uid/gids <seth@mojodna.net>
+
+== 0.9.7.7
+* added init.d scripts for redhat and ubuntu by Mike Perham <mperham@gmail.com>
+* fixed dependencies for SyslogLogger, eventmachine and memcache-client by Mike Perham <mperham@gmail.com>
+* added starling_top script to monitor starling server by Mike Perham <mperham@gmail.com>
+* fixed starling_top to use 22122 as port by Abdul-Rahman Advany <abdulrahman@advany.com>
+
+== 0.9.7.6 2008-06-24
+* removed client code (will be added to different project) by Abdul-Rahman Advany <abdulrahman@advany.com>
+
+== 0.9.7.5 2008-05-04
+* added worker class, using starling client you can now run them in background by Abdul-Rahman Advany <abdulrahman@advany.com>
+ - handles creation of threadpool
+ - handles fetching of messages and passing these to the threads
+ - handles pushing of processed messages to starling again if needed
+
+== 0.9.7 2008-05-03
+* merged branch of AnotherBritt and Glenn Rempe by Abdul-Rahman Advany <abdulrahman@advany.com>
+* rspeced tests by Abdul-Rahman Advany <abdulrahman@advany.com>
+
+==  0.9.6 2008-04-30
+* logging of message lifecycle as :age by AnotherBritt <?>
+* added some extra logging options by AnotherBritt <?>
+* added some test for epoll by AnotherBritt <?>
+
+== 0.9.5.4 2008-04-28
+
+* Bumped version number by Glenn Rempe <glenn.rempe@gmail.com>
+* Purged all old RubyForge config.  Now GitHub friendly by Glenn Rempe <glenn.rempe@gmail.com>
+* New gemspec for GitHub gem auto-build-serve by Glenn Rempe <glenn.rempe@gmail.com>
+
+== 0.9.4 2008-01-31 ==
+* Evented code added using EventMachine by Chris Wanstrath <chris@ozmm.org>
+
+== 2007-11-02
+
+* Initial release
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2007 FIXME full name
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,108 @@
+= Name
+
+Starling - a light weight server for reliable distributed message passing.
+
+= Description
+
+Starling is a powerful but simple messaging server that enables reliable
+distributed queuing with an absolutely minimal overhead. It speaks the
+MemCache protocol for maximum cross-platform compatibility. Any language
+that speaks MemCache can take advantage of Starling's queue facilities.
+
+= Installation
+
+  The Starling source is hosted at GitHub and can be found at:
+
+  http://github.com/starling/starling/tree/master
+
+  The original source was to be found at RubyForge but no longer exists there.
+
+  GitHub serves gems prefixed by a username to differentiate different forks.
+  This project can be installed with:
+
+  # THIS COMMAND ONE TIME ONLY
+  gem sources -a http://gems.github.com/
+
+  # As often as you like
+  sudo gem install starling-starling
+
+  See http://gems.github.com/ if you want more info about GitHub and gems.
+
+= Quick Start Usage
+
+  # View the Starling help and usage message
+  starling --help
+
+  # In a console window start the Starling server.  By default
+  # it runs verbosely in the foreground, listening on 127.0.0.1:22122
+  # and stores its files under /tmp/starling:
+  starling
+
+  # In a new console test the put and get of messages on a queue:
+
+  irb
+  >> require 'starling'
+  => true
+  >> starling = Starling.new('127.0.0.1:22122')
+  => MemCache: 1 servers, ns: nil, ro: false
+  >> starling.set('my_queue', 12345)
+  => nil
+  >> starling.get('my_queue')
+  => 12345
+
+  # You can do a simple loop over a queue with something like:
+  >> loop { puts starling.get('my_queue'); sleep 1 }
+  12345
+  nil
+  nil
+  ...
+
+  For more information run the following in a new console:
+  
+  'gem server'
+
+  This will start a gem server on http://localhost:8808/ which you can view in your
+  browser to see the RDocs for the gem. Or generate rdocs by running the following
+  in a new console:
+	
+  'rdoc'
+
+= Using memcache-client
+
+	memcache-client is now maintained by Mike Perham (http://www.mikeperham.com/)
+	
+	He is the current maintainer of the gem which is hosted at GitHub and can be found at:
+
+  http://github.com/mperham/memcache-client/tree/master
+
+	It can be installed using:
+
+	# THIS COMMAND ONE TIME ONLY
+	gem sources -a http://gems.github.com/
+
+	# As often as you like
+	sudo gem install memcache-client
+
+= Known Issues
+
+* Only tested with Memcache client 1.7.x and with SystemTimer (http://systemtimer.rubyforge.org/)
+* Starling is "slow" as far as messaging systems are concerned. In practice,
+  it's fast enough.
+
+== TODO
+
+* Implement memcached instead of memcache as a client interface (to make it faster)
+
+= Authors
+
+* Blaine Cook <romeda@gmail.com>
+* Chris Wanstrath <chris@ozmm.org>
+* AnotherBritt <?>
+* Glenn Rempe <?>
+* Abdul-Rahman Advany <abdulrahman@advany.com>
+* Harm Aarts <harmaarts@gmail.com>
+
+= Copyright
+
+Starling - a light-weight server for reliable distributed message passing.
+Copyright 2007-2008 Blaine Cook <blaine@twitter.com>, Twitter Inc.

data/Rakefile

@@ -0,0 +1,76 @@
+require 'rubygems'
+require 'rake/rdoctask'
+require 'spec/rake/spectask'
+require File.join("lib", "starling", "server")
+
+task :default => :spec
+
+task :install do
+  sh %{gem build starling.gemspec}
+  sh %{sudo gem install starling-*.gem}
+end
+
+desc "Run specs"
+task :spec do
+  Dir.glob('spec/*_spec.rb').each do |f|
+    puts f
+    system "spec #{f}"
+  end
+end
+
+Rake::RDocTask.new do |rd|
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+  rd.rdoc_dir = 'doc'
+end
+
+def get_files(rev, path)
+  result = []
+  `git-ls-tree #{rev}`.each_line do |line|
+    columns = line.split(/\s/)
+    case columns[1]
+    when 'blob'
+      next if columns[3] == '.gitignore' #Not part of gem
+      result << "#{path}/#{columns[3]}"
+    when 'tree'
+      result << get_files(columns[2], "#{path}/#{columns[3]}")
+    end
+  end
+  result
+end
+
+desc "Generate a gemspec"
+task :gemspec do
+  excess_files = Dir.glob('./spec/*') # remove spec files from generated gem
+
+  gemspec =<<-EOF
+# this file is automatically generated
+  Gem::Specification.new do |s|
+    s.name = "starling"
+    s.version = "#{StarlingServer::VERSION}"
+    s.authors = ["Blaine Cook", "Chris Wanstrath", "Britt Selvitelle", "Glenn Rempe", "Abdul-Rahman Advany", "Seth Fitzsimmons", "Haarm Arts", "Chris Gaffney", "Kim Toms"]
+    s.email = ["blaine@twitter.com", "chris@ozmm.org", "abdulrahman@advany.com", "starlingmq@groups.google.com", "harmaarts@gmail.com", "gaffneyc@gmail.com", "kim.toms@gmail.com"]
+    s.homepage = "http://github.com/starling/starling/"
+    s.summary = "Starling is a lightweight, transactional, distributed queue server"
+    s.description = s.summary
+    
+    s.files = #{(get_files('HEAD', '.').flatten - excess_files).inspect}
+    
+    s.executables = ["starling", "starling_top"]
+    s.require_paths = ["lib"]
+
+    s.has_rdoc = true
+    s.rdoc_options = ["--quiet", "--title", "starling documentation", "--opname", "index.html", "--line-numbers", "--main", "README.rdoc", "--inline-source"]
+    s.extra_rdoc_files = ["README.rdoc", "CHANGELOG", "LICENSE"]
+    
+    s.add_dependency 'fiveruns-memcache-client'
+    s.add_dependency 'eventmachine', [">= 0.12.0"]
+  end
+  EOF
+
+  open("starling.gemspec", "w") do |f|
+    f << gemspec
+  end
+
+  puts "gemspec successfully created."
+end

data/bin/starling

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'starling/server_runner'
+
+StarlingServer::Runner.run
+

data/bin/starling_top

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'ostruct'
+require 'socket'
+
+@options = OpenStruct.new
+@options.hostname = 'localhost'
+@options.port = 22122
+
+op = OptionParser.new do |opts|
+	opts.banner = "STARLING TOP\nUsage: startop [options]"
+	opts.separator "General Options:"
+	opts.on("-h HOSTNAME", "--hostname=HOSTNAME", "Hostname [default: localhost]") do |h|
+		@options.hostname = h
+	end
+	opts.on("-p PORT", "--port=PORT", Integer, "Port [default: 22122]") do |p|
+		@options.port = p
+	end
+	opts.on_tail("--help", "Show this message") do
+		puts opts
+		exit
+	end
+end
+op.parse!
+
+
+def stats_data
+  data = ''
+  sock = TCPSocket.new(@options.hostname, @options.port)
+  sock.print("stats\r\n")
+  sock.flush
+  # memcached does not close the socket once it is done writing
+  # the stats data.  We need to read line by line until we detect
+  # the END line and then stop/close on our side.
+  stats = sock.gets
+  while true
+    data += stats
+    break if stats.strip == 'END'
+    stats = sock.gets
+  end
+  sock.close
+  data
+end
+
+def parse(stats_data)
+  stats = []
+  stats_data.each_line do |line|
+    stats << "#{$1}: #{$2}" if line =~ /STAT (\w+) (\S+)/
+  end
+  stats.sort
+end
+
+stats = parse(stats_data)
+stats.each do |stat|
+	puts stat
+end

data/etc/sample-config.yml

@@ -0,0 +1,9 @@
+starling:
+  port: 22122
+  pid_file: /tmp/starling/starling.pid
+  queue_path: /tmp/starling/spool
+  timeout: 0
+  syslog_channel: starling-tampopo
+  log_level: DEBUG
+  daemonize: true
+

data/etc/starling.redhat

@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Make sure the /var/run/starling, /var/log/starling and /var/spool/starling
+# all exist and are owned by starling:starling
+#
+# starling    This shell script takes care of starting and stopping
+#                    the starling server
+# chkconfig: 345 98 98
+# description: The starling queue server
+# processname: starling
+# pidfile: /var/run/starling/starling.pid
+# logfile: /var/log/starling/starling.log
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+# Source networking configuration.
+[ -f /etc/sysconfig/network ] && . /etc/sysconfig/network
+
+DUSER=starling
+DGROUP=starling
+LOGFILE=/var/log/starling/starling.log
+SPOOLDIR=/var/spool/starling
+PORT=22122
+LISTEN=0.0.0.0
+PIDFILE=/var/run/starling/starling.pid
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=starling
+INSTALL_DIR=/usr/local/bin
+DAEMON=$INSTALL_DIR/$NAME
+OPTS="-h $LISTEN -p $PORT -d -q $SPOOLDIR -P $PIDFILE -L $LOGFILE -u $DUSER -g $DGROUP"
+
+start() {
+  echo -n $"Starting starling: "
+
+  daemon --pidfile=$PIDFILE $DAEMON $OPTS
+  echo
+}
+
+stop() {
+  echo -n $"Stopping starling: "
+
+  killproc -p $PIDFILE starling
+  echo
+}
+
+case "$1" in
+  start)
+    start
+  ;;
+  stop)
+    stop
+  ;;
+  restart)
+    stop
+    sleep 3
+    start
+  ;;
+  status)
+    status -p $PIDFILE starling
+  ;;
+  *)
+    echo $"Usage: $0 {start|stop|restart|status}"
+    exit 1
+esac

data/etc/starling.ubuntu

@@ -0,0 +1,71 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides: starling
+# Required-Start: $local_fs $remote_fs
+# Required-Stop: $local_fs $remote_fs
+# Default-Start: 2 3 4 5
+# Default-Stop: S 0 1 6
+# Short-Description: Starling queue server
+# Description: The Starling distributed, transactional queue server
+### END INIT INFO
+# Author: Twitter
+# Version: 0.10.0
+
+set -e
+
+DUSER=starling
+DGROUP=starling
+LOGFILE=/var/log/starling/starling.log
+SPOOLDIR=/var/spool/starling
+PORT=22122
+LISTEN=0.0.0.0
+PIDFILE=/var/run/starling/starling.pid
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=starling
+DESC="Starling"
+INSTALL_DIR=/usr/local/bin
+DAEMON=$INSTALL_DIR/$NAME
+SCRIPTNAME=/etc/init.d/$NAME
+OPTS="-h $LISTEN -p $PORT -d -q $SPOOLDIR -P $PIDFILE -L $LOGFILE"
+
+. /lib/lsb/init-functions
+
+
+# Gracefully exit if the package has been removed.
+test -x $DAEMON || exit 0
+
+d_start() {
+	log_begin_msg "Starting Starling Server..."
+		start-stop-daemon -c $DUSER:$DGROUP --start --quiet --pidfile $PIDFILE --exec $DAEMON \
+		-- $OPTS || log_end_msg 1
+		log_end_msg 0
+}
+
+d_stop() {
+	log_begin_msg "Stopping Starling Server..."
+		start-stop-daemon -c $DUSER:$DGROUP --stop --quiet --pidfile $PIDFILE \
+		|| log_end_msg 1
+		log_end_msg 0
+}
+
+case "$1" in
+start)
+d_start
+;;
+stop)
+d_stop
+;;
+restart|force-reload|reload)
+d_stop
+sleep 2
+d_start
+;;
+*)
+echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2
+exit 3
+;;
+esac
+
+exit 0
+