advany-starling 0.9.7.7
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +37 -0
- data/LICENSE +20 -0
- data/README.rdoc +106 -0
- data/Rakefile +21 -0
- data/bin/starling +6 -0
- data/bin/starling_top +57 -0
- data/etc/starling.redhat +63 -0
- data/etc/starling.ubuntu +71 -0
- data/lib/starling.rb +104 -0
- data/lib/starling/handler.rb +222 -0
- data/lib/starling/persistent_queue.rb +151 -0
- data/lib/starling/queue_collection.rb +141 -0
- data/lib/starling/server.rb +113 -0
- data/lib/starling/server_runner.rb +292 -0
- data/test/test_starling_server.rb +205 -0
- metadata +108 -0
data/CHANGELOG
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
== 0.9.7.7
|
2
|
+
* added init.d scripts for redhat and ubuntu by Mike Perham <mperham@gmail.com>
|
3
|
+
* fixed dependencies for SyslogLogger, eventmachine and memcache-client by Mike Perham <mperham@gmail.com>
|
4
|
+
* added starling_top script to monitor starling server by Mike Perham <mperham@gmail.com>
|
5
|
+
* fixed starling_top to use 22122 as port by Abdul-Rahman Advany <abdulrahman@advany.com>
|
6
|
+
|
7
|
+
== 0.9.7.6 2008-06-24
|
8
|
+
* removed client code (will be added to different project) by Abdul-Rahman Advany <abdulrahman@advany.com>
|
9
|
+
|
10
|
+
== 0.9.7.5 2008-05-04
|
11
|
+
* added worker class, using starling client you can now run them in background by Abdul-Rahman Advany <abdulrahman@advany.com>
|
12
|
+
- handles creation of threadpool
|
13
|
+
- handles fetching of messages and passing these to the threads
|
14
|
+
- handles pushing of processed messages to starling again if needed
|
15
|
+
|
16
|
+
== 0.9.7 2008-05-03
|
17
|
+
* merged branch of AnotherBritt and Glenn Rempe by Abdul-Rahman Advany <abdulrahman@advany.com>
|
18
|
+
* rspeced tests by Abdul-Rahman Advany <abdulrahman@advany.com>
|
19
|
+
|
20
|
+
== 0.9.6 2008-04-30
|
21
|
+
* logging of message lifecycle as :age by AnotherBritt <?>
|
22
|
+
* added some extra logging options by AnotherBritt <?>
|
23
|
+
* added some test for epoll by AnotherBritt <?>
|
24
|
+
|
25
|
+
== 0.9.5.4 2008-04-28
|
26
|
+
|
27
|
+
* Bumped version number by Glenn Rempe <glenn.rempe@gmail.com>
|
28
|
+
* Purged all old RubyForge config. Now GitHub friendly by Glenn Rempe <glenn.rempe@gmail.com>
|
29
|
+
* New gemspec for GitHub gem auto-build-serve by Glenn Rempe <glenn.rempe@gmail.com>
|
30
|
+
|
31
|
+
== 0.9.4 2008-01-31 ==
|
32
|
+
* Evented code added using EventMachine by Chris Wanstrath <chris@ozmm.org>
|
33
|
+
|
34
|
+
== 2007-11-02
|
35
|
+
|
36
|
+
* Initial release
|
37
|
+
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2007 FIXME full name
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
= Name
|
2
|
+
|
3
|
+
Starling - a light weight server for reliable distributed message passing.
|
4
|
+
|
5
|
+
= Description
|
6
|
+
|
7
|
+
Starling is a powerful but simple messaging server that enables reliable
|
8
|
+
distributed queuing with an absolutely minimal overhead. It speaks the
|
9
|
+
MemCache protocol for maximum cross-platform compatibility. Any language
|
10
|
+
that speaks MemCache can take advantage of Starling's queue facilities.
|
11
|
+
|
12
|
+
= Installation
|
13
|
+
|
14
|
+
This fork of the Starling source is hosted at GitHub and can be found at:
|
15
|
+
|
16
|
+
http://github.com/starling/starling/tree/master
|
17
|
+
|
18
|
+
The original source was to be found at RubyForge but no longer exists there.
|
19
|
+
|
20
|
+
GitHub serves gems prefixed by a username to differentiate different forks.
|
21
|
+
This project can be installed with:
|
22
|
+
|
23
|
+
# THIS COMMAND ONE TIME ONLY
|
24
|
+
gem sources -a http://gems.github.com/
|
25
|
+
|
26
|
+
# As often as you like
|
27
|
+
sudo gem install starling-starling
|
28
|
+
|
29
|
+
See http://gems.github.com/ if you want more info about GitHub and gems.
|
30
|
+
|
31
|
+
= Quick Start Usage
|
32
|
+
|
33
|
+
# View the Starling help and usage message
|
34
|
+
starling --help
|
35
|
+
|
36
|
+
# In a console window start the Starling server. By default
|
37
|
+
# it runs verbosely in the foreground, listening on 127.0.0.1:22122
|
38
|
+
# and stores its files under /tmp/starling:
|
39
|
+
starling
|
40
|
+
|
41
|
+
# In a new console test the put and get of messages on a queue:
|
42
|
+
|
43
|
+
irb
|
44
|
+
>> require 'starling'
|
45
|
+
=> true
|
46
|
+
>> starling = Starling.new('127.0.0.1:22122')
|
47
|
+
=> MemCache: 1 servers, 1 buckets, ns: nil, ro: false
|
48
|
+
>> starling.set('my_queue', 12345)
|
49
|
+
=> nil
|
50
|
+
>> starling.get('my_queue')
|
51
|
+
=> 12345
|
52
|
+
|
53
|
+
# You can do a simple loop over a queue with something like:
|
54
|
+
>> loop { puts starling.get('my_queue'); sleep 1 }
|
55
|
+
12345
|
56
|
+
nil
|
57
|
+
nil
|
58
|
+
...
|
59
|
+
|
60
|
+
For more information run the following in a new console:
|
61
|
+
|
62
|
+
'gem server'
|
63
|
+
|
64
|
+
This will start a gem server on http://localhost:8808/ which you can view in your
|
65
|
+
browser to see the RDocs for the gem. Or generate rdocs by running the following
|
66
|
+
in a new console:
|
67
|
+
|
68
|
+
'rdoc'
|
69
|
+
|
70
|
+
= Using fiveruns memcache-client
|
71
|
+
|
72
|
+
memcache-client from fiveruns has a couple of fixed added like supporting failover and retry on failure.
|
73
|
+
|
74
|
+
This fork of the memcache-client source is hosted at GitHub and can be found at:
|
75
|
+
|
76
|
+
http://github.com/fiveruns/memcache-client/tree/master
|
77
|
+
|
78
|
+
It can be installed using:
|
79
|
+
|
80
|
+
# THIS COMMAND ONE TIME ONLY
|
81
|
+
gem sources -a http://gems.github.com/
|
82
|
+
|
83
|
+
# As often as you like
|
84
|
+
sudo gem install fiveruns-memcache-client
|
85
|
+
|
86
|
+
= Known Issues
|
87
|
+
|
88
|
+
* Starling is "slow" as far as messaging systems are concerned. In practice,
|
89
|
+
it's fast enough.
|
90
|
+
|
91
|
+
== TODO
|
92
|
+
|
93
|
+
* Implement memcached instead of memcache as a client interface (to make it faster)
|
94
|
+
|
95
|
+
= Authors
|
96
|
+
|
97
|
+
* Blaine Cook <romeda@gmail.com>
|
98
|
+
* Chris Wanstrath <chris@ozmm.org>
|
99
|
+
* AnotherBritt <?>
|
100
|
+
* Glenn Rempe <?>
|
101
|
+
* Abdul-Rahman Advany <abdulrahman@advany.com>
|
102
|
+
|
103
|
+
= Copyright
|
104
|
+
|
105
|
+
Starling - a light-weight server for reliable distributed message passing.
|
106
|
+
Copyright 2007-2008 Blaine Cook <blaine@twitter.com>, Twitter Inc.
|
data/Rakefile
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake/rdoctask'
|
3
|
+
require 'spec/rake/spectask'
|
4
|
+
|
5
|
+
task :install do
|
6
|
+
sh %{gem build starling.gemspec}
|
7
|
+
sh %{sudo gem install starling-*.gem}
|
8
|
+
end
|
9
|
+
|
10
|
+
Spec::Rake::SpecTask.new do |t|
|
11
|
+
t.ruby_opts = ['-rtest/unit']
|
12
|
+
t.spec_files = FileList['test/test_*.rb']
|
13
|
+
t.fail_on_error = true
|
14
|
+
end
|
15
|
+
|
16
|
+
Rake::RDocTask.new do |rd|
|
17
|
+
rd.main = "README.rdoc"
|
18
|
+
rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
|
19
|
+
rd.rdoc_dir = 'doc'
|
20
|
+
# rd.options = spec.rdoc_options
|
21
|
+
end
|
data/bin/starling
ADDED
data/bin/starling_top
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'ostruct'
|
5
|
+
require 'socket'
|
6
|
+
|
7
|
+
@options = OpenStruct.new
|
8
|
+
@options.hostname = 'localhost'
|
9
|
+
@options.port = 22122
|
10
|
+
|
11
|
+
op = OptionParser.new do |opts|
|
12
|
+
opts.banner = "STARLING TOP\nUsage: startop [options]"
|
13
|
+
opts.separator "General Options:"
|
14
|
+
opts.on("-h HOSTNAME", "--hostname=HOSTNAME", "Hostname [default: localhost]") do |h|
|
15
|
+
@options.hostname = h
|
16
|
+
end
|
17
|
+
opts.on("-p PORT", "--port=PORT", Integer, "Port [default: 22122]") do |p|
|
18
|
+
@options.port = p
|
19
|
+
end
|
20
|
+
opts.on_tail("--help", "Show this message") do
|
21
|
+
puts opts
|
22
|
+
exit
|
23
|
+
end
|
24
|
+
end
|
25
|
+
op.parse!
|
26
|
+
|
27
|
+
|
28
|
+
def stats_data
|
29
|
+
data = ''
|
30
|
+
sock = TCPSocket.new(@options.hostname, @options.port)
|
31
|
+
sock.print("stats\r\n")
|
32
|
+
sock.flush
|
33
|
+
# memcached does not close the socket once it is done writing
|
34
|
+
# the stats data. We need to read line by line until we detect
|
35
|
+
# the END line and then stop/close on our side.
|
36
|
+
stats = sock.gets
|
37
|
+
while true
|
38
|
+
data += stats
|
39
|
+
break if stats.strip == 'END'
|
40
|
+
stats = sock.gets
|
41
|
+
end
|
42
|
+
sock.close
|
43
|
+
data
|
44
|
+
end
|
45
|
+
|
46
|
+
def parse(stats_data)
|
47
|
+
stats = []
|
48
|
+
stats_data.each_line do |line|
|
49
|
+
stats << "#{$1}: #{$2}" if line =~ /STAT (\w+) (\S+)/
|
50
|
+
end
|
51
|
+
stats.sort
|
52
|
+
end
|
53
|
+
|
54
|
+
stats = parse(stats_data)
|
55
|
+
stats.each do |stat|
|
56
|
+
puts stat
|
57
|
+
end
|
data/etc/starling.redhat
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
#
|
3
|
+
# starling This shell script takes care of starting and stopping
|
4
|
+
# the starling server
|
5
|
+
# chkconfig: 345 98 98
|
6
|
+
# description: The starling queue server
|
7
|
+
|
8
|
+
#determine where the 'pidof' executable is located
|
9
|
+
if [ -e /bin/pidof ]; then
|
10
|
+
PIDOF="/bin/pidof"
|
11
|
+
elif [ -e /sbin/pidof ]; then
|
12
|
+
PIDOF="/sbin/pidof"
|
13
|
+
elif [ -e /usr/local/bin/pidof ]; then
|
14
|
+
PIDOF="/usr/local/bin/pidof"
|
15
|
+
elif [ -e /bin/pgrep ]; then
|
16
|
+
PIDOF="/bin/pgrep"
|
17
|
+
elif [ -e /usr/bin/pgrep ]; then
|
18
|
+
PIDOF="/usr/bin/pgrep"
|
19
|
+
elif [ -e /usr/local/bin/pgrep ]; then
|
20
|
+
PIDOF="/usr/local/bin/pgrep"
|
21
|
+
else
|
22
|
+
echo "Could not find pidof or pgrep"
|
23
|
+
fi
|
24
|
+
|
25
|
+
PROGDIR="/usr/bin"
|
26
|
+
PROGNAME="starling"
|
27
|
+
OPTIONS="-u nobody -g nobody -L /var/log/starling.log -q /var/spool/starling"
|
28
|
+
|
29
|
+
start() {
|
30
|
+
pid=`$PIDOF $PROGNAME`
|
31
|
+
if [ "$pid" != "" ]; then
|
32
|
+
echo "$PROGDIR$PROGNAME already running: $pid"
|
33
|
+
else
|
34
|
+
echo "Starting $PROGDIR$PROGNAME"
|
35
|
+
cd $PROGDIR
|
36
|
+
nohup $PROGDIR$PROGNAME $OPTIONS &
|
37
|
+
fi
|
38
|
+
}
|
39
|
+
|
40
|
+
stop() {
|
41
|
+
pid=`$PIDOF $PROGNAME`
|
42
|
+
if [ "$pid" != "" ]; then
|
43
|
+
echo "Stopping $PROGDIR$PROGNAME: $pid"
|
44
|
+
kill $pid
|
45
|
+
else
|
46
|
+
echo "$PROGDIR$PROGNAME not running"
|
47
|
+
fi
|
48
|
+
}
|
49
|
+
|
50
|
+
case "$1" in
|
51
|
+
start)
|
52
|
+
start
|
53
|
+
;;
|
54
|
+
stop)
|
55
|
+
stop
|
56
|
+
;;
|
57
|
+
restart)
|
58
|
+
stop
|
59
|
+
sleep 3
|
60
|
+
start
|
61
|
+
;;
|
62
|
+
|
63
|
+
esac
|
data/etc/starling.ubuntu
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
#! /bin/sh
|
2
|
+
### BEGIN INIT INFO
|
3
|
+
# Provides: starling
|
4
|
+
# Required-Start: $local_fs $remote_fs
|
5
|
+
# Required-Stop: $local_fs $remote_fs
|
6
|
+
# Default-Start: 2 3 4 5
|
7
|
+
# Default-Stop: S 0 1 6
|
8
|
+
# Short-Description: Starling queue server
|
9
|
+
# Description: The Starling distributed, transactional queue server
|
10
|
+
### END INIT INFO
|
11
|
+
# Author: Twitter
|
12
|
+
# Version: 0.9.7.7
|
13
|
+
|
14
|
+
set -e
|
15
|
+
|
16
|
+
DUSER=starling
|
17
|
+
DGROUP=starling
|
18
|
+
LOGFILE=/var/log/starling/starling.log
|
19
|
+
SPOOLDIR=/var/spool/starling
|
20
|
+
PORT=22122
|
21
|
+
LISTEN=0.0.0.0
|
22
|
+
PIDFILE=/var/run/starling/starling.pid
|
23
|
+
|
24
|
+
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
25
|
+
NAME=starling
|
26
|
+
DESC="Starling"
|
27
|
+
INSTALL_DIR=/usr/local/bin
|
28
|
+
DAEMON=$INSTALL_DIR/$NAME
|
29
|
+
SCRIPTNAME=/etc/init.d/$NAME
|
30
|
+
OPTS="-h $LISTEN -p $PORT -d -q $SPOOLDIR -P $PIDFILE -L $LOGFILE"
|
31
|
+
|
32
|
+
. /lib/lsb/init-functions
|
33
|
+
|
34
|
+
|
35
|
+
# Gracefully exit if the package has been removed.
|
36
|
+
test -x $DAEMON || exit 0
|
37
|
+
|
38
|
+
d_start() {
|
39
|
+
log_begin_msg "Starting Starling Server..."
|
40
|
+
start-stop-daemon -c $DUSER:$DGROUP --start --quiet --pidfile $PIDFILE --exec $DAEMON \
|
41
|
+
-- $OPTS || log_end_msg 1
|
42
|
+
log_end_msg 0
|
43
|
+
}
|
44
|
+
|
45
|
+
d_stop() {
|
46
|
+
log_begin_msg "Stopping Starling Server..."
|
47
|
+
start-stop-daemon -c $DUSER:$DGROUP --stop --quiet --pidfile $PIDFILE \
|
48
|
+
|| log_end_msg 1
|
49
|
+
log_end_msg 0
|
50
|
+
}
|
51
|
+
|
52
|
+
case "$1" in
|
53
|
+
start)
|
54
|
+
d_start
|
55
|
+
;;
|
56
|
+
stop)
|
57
|
+
d_stop
|
58
|
+
;;
|
59
|
+
restart|force-reload|reload)
|
60
|
+
d_stop
|
61
|
+
sleep 2
|
62
|
+
d_start
|
63
|
+
;;
|
64
|
+
*)
|
65
|
+
echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2
|
66
|
+
exit 3
|
67
|
+
;;
|
68
|
+
esac
|
69
|
+
|
70
|
+
exit 0
|
71
|
+
|
data/lib/starling.rb
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
require 'memcache'
|
2
|
+
|
3
|
+
class Starling < MemCache
|
4
|
+
|
5
|
+
WAIT_TIME = 0.25
|
6
|
+
|
7
|
+
##
|
8
|
+
# fetch an item from a queue.
|
9
|
+
|
10
|
+
def get(*args)
|
11
|
+
loop do
|
12
|
+
response = super(*args)
|
13
|
+
return response unless response.nil?
|
14
|
+
sleep WAIT_TIME
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
##
|
19
|
+
# insert +value+ into +queue+.
|
20
|
+
#
|
21
|
+
# +expiry+ is expressed as a UNIX timestamp
|
22
|
+
#
|
23
|
+
# If +raw+ is true, +value+ will not be Marshalled. If +raw+ = :yaml, +value+
|
24
|
+
# will be serialized with YAML, instead.
|
25
|
+
|
26
|
+
def set(queue, value, expiry = 0, raw = false)
|
27
|
+
retries = 0
|
28
|
+
begin
|
29
|
+
if raw == :yaml
|
30
|
+
value = YAML.dump(value)
|
31
|
+
raw = true
|
32
|
+
end
|
33
|
+
|
34
|
+
super(queue, value, expiry, raw)
|
35
|
+
rescue MemCache::MemCacheError => e
|
36
|
+
retries += 1
|
37
|
+
sleep WAIT_TIME
|
38
|
+
retry unless retries > 3
|
39
|
+
raise e
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
##
|
44
|
+
# returns the number of items in +queue+. If +queue+ is +:all+, a hash of all
|
45
|
+
# queue sizes will be returned.
|
46
|
+
|
47
|
+
def sizeof(queue, statistics = nil)
|
48
|
+
statistics ||= stats
|
49
|
+
|
50
|
+
if queue == :all
|
51
|
+
queue_sizes = {}
|
52
|
+
available_queues(statistics).each do |queue|
|
53
|
+
queue_sizes[queue] = sizeof(queue, statistics)
|
54
|
+
end
|
55
|
+
return queue_sizes
|
56
|
+
end
|
57
|
+
|
58
|
+
statistics.inject(0) { |m,(k,v)| m + v["queue_#{queue}_items"].to_i }
|
59
|
+
end
|
60
|
+
|
61
|
+
##
|
62
|
+
# returns a list of available (currently allocated) queues.
|
63
|
+
|
64
|
+
def available_queues(statistics = nil)
|
65
|
+
statistics ||= stats
|
66
|
+
|
67
|
+
statistics.map { |k,v|
|
68
|
+
v.keys
|
69
|
+
}.flatten.uniq.grep(/^queue_(.*)_items/).map { |v|
|
70
|
+
v.gsub(/^queue_/, '').gsub(/_items$/, '')
|
71
|
+
}.reject { |v|
|
72
|
+
v =~ /_total$/ || v =~ /_expired$/
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
##
|
77
|
+
# iterator to flush +queue+. Each element will be passed to the provided
|
78
|
+
# +block+
|
79
|
+
|
80
|
+
def flush(queue)
|
81
|
+
sizeof(queue).times do
|
82
|
+
v = get(queue)
|
83
|
+
yield v if block_given?
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
private
|
88
|
+
|
89
|
+
def get_server_for_key(key)
|
90
|
+
raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
|
91
|
+
raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
|
92
|
+
raise MemCacheError, "No servers available" if @servers.empty?
|
93
|
+
|
94
|
+
bukkits = @buckets.dup
|
95
|
+
bukkits.nitems.times do |try|
|
96
|
+
n = rand(bukkits.nitems)
|
97
|
+
server = bukkits[n]
|
98
|
+
return server if server.alive?
|
99
|
+
bukkits.delete_at(n)
|
100
|
+
end
|
101
|
+
|
102
|
+
raise MemCacheError, "No servers available (all dead)"
|
103
|
+
end
|
104
|
+
end
|