starling 0.9.3
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +3 -0
- data/License.txt +20 -0
- data/Manifest.txt +29 -0
- data/README.txt +42 -0
- data/Rakefile +4 -0
- data/bin/starling +4 -0
- data/config/hoe.rb +71 -0
- data/config/requirements.rb +15 -0
- data/lib/starling.rb +104 -0
- data/lib/starling/handler.rb +181 -0
- data/lib/starling/persistent_queue.rb +156 -0
- data/lib/starling/queue_collection.rb +140 -0
- data/lib/starling/runner.rb +253 -0
- data/lib/starling/server.rb +208 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/script/txt2html +74 -0
- data/setup.rb +1585 -0
- data/tasks/deployment.rake +34 -0
- data/tasks/environment.rake +7 -0
- data/tasks/website.rake +17 -0
- data/test/test_helper.rb +2 -0
- data/test/test_persistent_queue.rb +17 -0
- data/test/test_starling.rb +111 -0
- data/website/index.html +0 -0
- data/website/index.txt +66 -0
- data/website/javascripts/rounded_corners_lite.inc.js +285 -0
- data/website/stylesheets/screen.css +138 -0
- data/website/template.rhtml +48 -0
- metadata +88 -0
data/History.txt
ADDED
data/License.txt
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2007 Blaine Cook, Twitter, Inc.
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Manifest.txt
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
History.txt
|
2
|
+
License.txt
|
3
|
+
Manifest.txt
|
4
|
+
README.txt
|
5
|
+
Rakefile
|
6
|
+
bin/starling
|
7
|
+
config/hoe.rb
|
8
|
+
config/requirements.rb
|
9
|
+
lib/starling.rb
|
10
|
+
lib/starling/handler.rb
|
11
|
+
lib/starling/persistent_queue.rb
|
12
|
+
lib/starling/queue_collection.rb
|
13
|
+
lib/starling/runner.rb
|
14
|
+
lib/starling/server.rb
|
15
|
+
script/destroy
|
16
|
+
script/generate
|
17
|
+
script/txt2html
|
18
|
+
setup.rb
|
19
|
+
tasks/deployment.rake
|
20
|
+
tasks/environment.rake
|
21
|
+
tasks/website.rake
|
22
|
+
test/test_helper.rb
|
23
|
+
test/test_persistent_queue.rb
|
24
|
+
test/test_starling.rb
|
25
|
+
website/index.html
|
26
|
+
website/index.txt
|
27
|
+
website/javascripts/rounded_corners_lite.inc.js
|
28
|
+
website/stylesheets/screen.css
|
29
|
+
website/template.rhtml
|
data/README.txt
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
= Name
|
2
|
+
|
3
|
+
Starling - a light weight server for reliable distributed message passing.
|
4
|
+
|
5
|
+
= Synopsis
|
6
|
+
|
7
|
+
# Start the Starling server as a daemonized process:
|
8
|
+
starling -h 192.168.1.1 -d
|
9
|
+
|
10
|
+
# Put messages onto a queue:
|
11
|
+
require 'memcache'
|
12
|
+
starling = MemCache.new('192.168.1.1:22122')
|
13
|
+
starling.set('my_queue', 12345)
|
14
|
+
|
15
|
+
# Get messages from the queue:
|
16
|
+
require 'memcache'
|
17
|
+
starling = MemCache.new('192.168.1.1:22122')
|
18
|
+
loop { puts starling.get('my_queue') }
|
19
|
+
|
20
|
+
# See the Starling documentation for more information.
|
21
|
+
|
22
|
+
= Description
|
23
|
+
|
24
|
+
Starling is a powerful but simple messaging server that enables reliable
|
25
|
+
distributed queuing with an absolutely minimal overhead. It speaks the
|
26
|
+
MemCache protocol for maximum cross-platform compatibility. Any language
|
27
|
+
that speaks MemCache can take advantage of Starling's queue facilities.
|
28
|
+
|
29
|
+
= Known Issues
|
30
|
+
|
31
|
+
* Starling is "slow" as far as messaging systems are concerned. In practice,
|
32
|
+
it's fast enough. If you'd like to help make it faster please do. Starting
|
33
|
+
points would be to use an event-driven interface, and get rid of threading.
|
34
|
+
|
35
|
+
= Authors
|
36
|
+
|
37
|
+
Blaine Cook <romeda@gmail.com>
|
38
|
+
|
39
|
+
= Copyright
|
40
|
+
|
41
|
+
Starling - a light-weight server for reliable distributed message passing.
|
42
|
+
Copyright 2007 Blaine Cook <blaine@twitter.com>, Twitter Inc.
|
data/Rakefile
ADDED
data/bin/starling
ADDED
data/config/hoe.rb
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'starling/server'
|
2
|
+
|
3
|
+
AUTHOR = 'Blaine Cook' # can also be an array of Authors
|
4
|
+
EMAIL = "blaine@twitter.com"
|
5
|
+
DESCRIPTION = "Starling is a lightweight, transactional, distributed queue server"
|
6
|
+
GEM_NAME = 'starling' # what ppl will type to install your gem
|
7
|
+
RUBYFORGE_PROJECT = 'starling' # The unix name for your project
|
8
|
+
HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
|
9
|
+
DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
|
10
|
+
|
11
|
+
@config_file = "~/.rubyforge/user-config.yml"
|
12
|
+
@config = nil
|
13
|
+
RUBYFORGE_USERNAME = "unknown"
|
14
|
+
def rubyforge_username
|
15
|
+
unless @config
|
16
|
+
begin
|
17
|
+
@config = YAML.load(File.read(File.expand_path(@config_file)))
|
18
|
+
rescue
|
19
|
+
puts <<-EOS
|
20
|
+
ERROR: No rubyforge config file found: #{@config_file}
|
21
|
+
Run 'rubyforge setup' to prepare your env for access to Rubyforge
|
22
|
+
- See http://newgem.rubyforge.org/rubyforge.html for more details
|
23
|
+
EOS
|
24
|
+
exit
|
25
|
+
end
|
26
|
+
end
|
27
|
+
RUBYFORGE_USERNAME.replace @config["username"]
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
REV = nil
|
32
|
+
# UNCOMMENT IF REQUIRED:
|
33
|
+
# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
|
34
|
+
VERS = StarlingServer::VERSION + (REV ? ".#{REV}" : "")
|
35
|
+
RDOC_OPTS = ['--quiet', '--title', 'starling documentation',
|
36
|
+
"--opname", "index.html",
|
37
|
+
"--line-numbers",
|
38
|
+
"--main", "README",
|
39
|
+
"--inline-source"]
|
40
|
+
|
41
|
+
class Hoe
|
42
|
+
def extra_deps
|
43
|
+
@extra_deps.reject! { |x| Array(x).first == 'hoe' }
|
44
|
+
@extra_deps
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Generate all the Rake tasks
|
49
|
+
# Run 'rake -T' to see list of generated tasks (from gem root directory)
|
50
|
+
hoe = Hoe.new(GEM_NAME, VERS) do |p|
|
51
|
+
p.author = AUTHOR
|
52
|
+
p.description = DESCRIPTION
|
53
|
+
p.email = EMAIL
|
54
|
+
p.summary = DESCRIPTION
|
55
|
+
p.url = HOMEPATH
|
56
|
+
p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
|
57
|
+
p.test_globs = ["test/**/test_*.rb"]
|
58
|
+
p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean.
|
59
|
+
|
60
|
+
# == Optional
|
61
|
+
p.changes = p.paragraphs_of("History.txt", 0..1).join("\\n\\n")
|
62
|
+
#p.extra_deps = [] # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
|
63
|
+
|
64
|
+
#p.spec_extras = {} # A hash of extra values to set in the gemspec.
|
65
|
+
|
66
|
+
end
|
67
|
+
|
68
|
+
CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
|
69
|
+
PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
|
70
|
+
hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
|
71
|
+
hoe.rsync_args = '-av --delete --ignore-errors'
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
include FileUtils
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
%w[rake hoe newgem rubigen].each do |req_gem|
|
6
|
+
begin
|
7
|
+
require req_gem
|
8
|
+
rescue LoadError
|
9
|
+
puts "This Rakefile requires the '#{req_gem}' RubyGem."
|
10
|
+
puts "Installation: gem install #{req_gem} -y"
|
11
|
+
exit
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
|
data/lib/starling.rb
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
require 'memcache'
|
2
|
+
|
3
|
+
class Starling < MemCache
|
4
|
+
|
5
|
+
WAIT_TIME = 0.25
|
6
|
+
|
7
|
+
##
|
8
|
+
# fetch an item from a queue.
|
9
|
+
|
10
|
+
def get(*args)
|
11
|
+
loop do
|
12
|
+
response = super(*args)
|
13
|
+
return response unless response.nil?
|
14
|
+
sleep WAIT_TIME
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
##
|
19
|
+
# insert +value+ into +queue+.
|
20
|
+
#
|
21
|
+
# +expiry+ is expressed as a UNIX timestamp
|
22
|
+
#
|
23
|
+
# If +raw+ is true, +value+ will not be Marshalled. If +raw+ = :yaml, +value+
|
24
|
+
# will be serialized with YAML, instead.
|
25
|
+
|
26
|
+
def set(queue, value, expiry = 0, raw = false)
|
27
|
+
retries = 0
|
28
|
+
begin
|
29
|
+
if raw == :yaml
|
30
|
+
value = YAML.dump(value)
|
31
|
+
raw = true
|
32
|
+
end
|
33
|
+
|
34
|
+
super(queue, value, expiry, raw)
|
35
|
+
rescue MemCache::MemCacheError => e
|
36
|
+
retries += 1
|
37
|
+
sleep WAIT_TIME
|
38
|
+
retry unless retries > 3
|
39
|
+
raise e
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
##
|
44
|
+
# returns the number of items in +queue+. If +queue+ is +:all+, a hash of all
|
45
|
+
# queue sizes will be returned.
|
46
|
+
|
47
|
+
def sizeof(queue, statistics = nil)
|
48
|
+
statistics ||= stats
|
49
|
+
|
50
|
+
if queue == :all
|
51
|
+
queue_sizes = {}
|
52
|
+
available_queues(statistics).each do |queue|
|
53
|
+
queue_sizes[queue] = sizeof(queue, statistics)
|
54
|
+
end
|
55
|
+
return queue_sizes
|
56
|
+
end
|
57
|
+
|
58
|
+
statistics.inject(0) { |m,(k,v)| m + v["queue_#{queue}_items"].to_i }
|
59
|
+
end
|
60
|
+
|
61
|
+
##
|
62
|
+
# returns a list of available (currently allocated) queues.
|
63
|
+
|
64
|
+
def available_queues(statistics = nil)
|
65
|
+
statistics ||= stats
|
66
|
+
|
67
|
+
statistics.map { |k,v|
|
68
|
+
v.keys
|
69
|
+
}.flatten.uniq.grep(/^queue_(.*)_items/).map { |v|
|
70
|
+
v.gsub(/^queue_/, '').gsub(/_items$/, '')
|
71
|
+
}.reject { |v|
|
72
|
+
v =~ /_total$/ || v =~ /_expired$/
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
##
|
77
|
+
# iterator to flush +queue+. Each element will be passed to the provided
|
78
|
+
# +block+
|
79
|
+
|
80
|
+
def flush(queue)
|
81
|
+
sizeof(queue).times do
|
82
|
+
v = get(queue)
|
83
|
+
yield v if block_given?
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
private
|
88
|
+
|
89
|
+
def get_server_for_key(key)
|
90
|
+
raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
|
91
|
+
raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
|
92
|
+
raise MemCacheError, "No servers available" if @servers.empty?
|
93
|
+
|
94
|
+
bukkits = @buckets.dup
|
95
|
+
bukkits.nitems.times do |try|
|
96
|
+
n = rand(bukkits.nitems)
|
97
|
+
server = bukkits[n]
|
98
|
+
return server if server.alive?
|
99
|
+
bukkits.delete_at(n)
|
100
|
+
end
|
101
|
+
|
102
|
+
raise MemCacheError, "No servers available (all dead)"
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,181 @@
|
|
1
|
+
module StarlingServer
|
2
|
+
|
3
|
+
##
|
4
|
+
# This is an internal class that's used by Starling::Server to handle the
|
5
|
+
# MemCache protocol and act as an interface between the Server and the
|
6
|
+
# QueueCollection.
|
7
|
+
|
8
|
+
class Handler
|
9
|
+
|
10
|
+
DATA_PACK_FMT = "Ia*".freeze
|
11
|
+
|
12
|
+
# ERROR responses
|
13
|
+
ERR_UNKNOWN_COMMAND = "CLIENT_ERROR bad command line format\r\n".freeze
|
14
|
+
|
15
|
+
# GET Responses
|
16
|
+
GET_COMMAND = /^get (.{1,250})\r\n$/
|
17
|
+
GET_RESPONSE = "VALUE %s %s %s\r\n%s\r\nEND\r\n".freeze
|
18
|
+
GET_RESPONSE_EMPTY = "END\r\n".freeze
|
19
|
+
|
20
|
+
# SET Responses
|
21
|
+
SET_COMMAND = /^set (.{1,250}) ([0-9]+) ([0-9]+) ([0-9]+)\r\n$/
|
22
|
+
SET_RESPONSE_SUCCESS = "STORED\r\n".freeze
|
23
|
+
SET_RESPONSE_FAILURE = "NOT STORED\r\n".freeze
|
24
|
+
SET_CLIENT_DATA_ERROR = "CLIENT_ERROR bad data chunk\r\nERROR\r\n".freeze
|
25
|
+
|
26
|
+
# STAT Response
|
27
|
+
STATS_COMMAND = /stats\r\n$/
|
28
|
+
STATS_RESPONSE = "STAT pid %d
|
29
|
+
STAT uptime %d
|
30
|
+
STAT time %d
|
31
|
+
STAT version %s
|
32
|
+
STAT rusage_user %0.6f
|
33
|
+
STAT rusage_system %0.6f
|
34
|
+
STAT curr_items %d
|
35
|
+
STAT total_items %d
|
36
|
+
STAT bytes %d
|
37
|
+
STAT curr_connections %d
|
38
|
+
STAT total_connections %d
|
39
|
+
STAT cmd_get %d
|
40
|
+
STAT cmd_set %d
|
41
|
+
STAT get_hits %d
|
42
|
+
STAT get_misses %d
|
43
|
+
STAT bytes_read %d
|
44
|
+
STAT bytes_written %d
|
45
|
+
STAT limit_maxbytes %d
|
46
|
+
%sEND\r\n".freeze
|
47
|
+
QUEUE_STATS_RESPONSE = "STAT queue_%s_items %d
|
48
|
+
STAT queue_%s_total_items %d
|
49
|
+
STAT queue_%s_logsize %d
|
50
|
+
STAT queue_%s_expired_items %d\n".freeze
|
51
|
+
|
52
|
+
##
|
53
|
+
# Creates a new handler for the MemCache protocol that communicates with a
|
54
|
+
# given client.
|
55
|
+
|
56
|
+
def initialize(client, server, queue_collection)
|
57
|
+
@client = client
|
58
|
+
@server = server
|
59
|
+
@queue_collection = queue_collection
|
60
|
+
@expiry_stats = Hash.new(0)
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# Process incoming commands from the attached client.
|
65
|
+
|
66
|
+
def run
|
67
|
+
while running? do
|
68
|
+
process_command(@client.readline)
|
69
|
+
Thread.current[:last_activity] = Time.now
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
private
|
74
|
+
|
75
|
+
def running?
|
76
|
+
!Thread.current[:shutdown]
|
77
|
+
end
|
78
|
+
|
79
|
+
def respond(str, *args)
|
80
|
+
response = sprintf(str, *args)
|
81
|
+
@server.stats[:bytes_written] += response.length
|
82
|
+
@client.write response
|
83
|
+
end
|
84
|
+
|
85
|
+
def process_command(command)
|
86
|
+
begin
|
87
|
+
@server.stats[:bytes_read] += command.length
|
88
|
+
case command
|
89
|
+
when SET_COMMAND
|
90
|
+
@server.stats[:set_requests] += 1
|
91
|
+
set($1, $2, $3, $4.to_i)
|
92
|
+
when GET_COMMAND
|
93
|
+
@server.stats[:get_requests] += 1
|
94
|
+
get($1)
|
95
|
+
when STATS_COMMAND
|
96
|
+
stats
|
97
|
+
else
|
98
|
+
logger.warn "Unknown command: #{command[0,4]}.\nFull command was #{command}."
|
99
|
+
respond ERR_UNKNOWN_COMMAND
|
100
|
+
end
|
101
|
+
rescue => e
|
102
|
+
logger.error "Error handling request: #{e}."
|
103
|
+
logger.debug e.backtrace.join("\n")
|
104
|
+
respond GET_RESPONSE_EMPTY
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def set(key, flags, expiry, len)
|
109
|
+
data = @client.read(len)
|
110
|
+
data_end = @client.read(2)
|
111
|
+
@server.stats[:bytes_read] += len + 2
|
112
|
+
if data_end == "\r\n" && data.size == len
|
113
|
+
internal_data = [expiry.to_i, data].pack(DATA_PACK_FMT)
|
114
|
+
if @queue_collection.put(key, internal_data)
|
115
|
+
respond SET_RESPONSE_SUCCESS
|
116
|
+
else
|
117
|
+
respond SET_RESPONSE_FAILURE
|
118
|
+
end
|
119
|
+
else
|
120
|
+
respond SET_CLIENT_DATA_ERROR
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def get(key)
|
125
|
+
now = Time.now.to_i
|
126
|
+
|
127
|
+
while response = @queue_collection.take(key)
|
128
|
+
expiry, data = response.unpack(DATA_PACK_FMT)
|
129
|
+
|
130
|
+
break if expiry == 0 || expiry >= now
|
131
|
+
|
132
|
+
@expiry_stats[key] += 1
|
133
|
+
expiry, data = nil
|
134
|
+
end
|
135
|
+
|
136
|
+
if data
|
137
|
+
respond GET_RESPONSE, key, 0, data.size, data
|
138
|
+
else
|
139
|
+
respond GET_RESPONSE_EMPTY
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def stats
|
144
|
+
respond STATS_RESPONSE,
|
145
|
+
Process.pid, # pid
|
146
|
+
Time.now - @server.stats(:start_time), # uptime
|
147
|
+
Time.now.to_i, # time
|
148
|
+
StarlingServer::VERSION, # version
|
149
|
+
Process.times.utime, # rusage_user
|
150
|
+
Process.times.stime, # rusage_system
|
151
|
+
@queue_collection.stats(:current_size), # curr_items
|
152
|
+
@queue_collection.stats(:total_items), # total_items
|
153
|
+
@queue_collection.stats(:current_bytes), # bytes
|
154
|
+
@server.stats(:connections), # curr_connections
|
155
|
+
@server.stats(:total_connections), # total_connections
|
156
|
+
@server.stats(:get_requests), # get count
|
157
|
+
@server.stats(:set_requests), # set count
|
158
|
+
@queue_collection.stats(:get_hits),
|
159
|
+
@queue_collection.stats(:get_misses),
|
160
|
+
@server.stats(:bytes_read), # total bytes read
|
161
|
+
@server.stats(:bytes_written), # total bytes written
|
162
|
+
0, # limit_maxbytes
|
163
|
+
queue_stats
|
164
|
+
end
|
165
|
+
|
166
|
+
def queue_stats
|
167
|
+
@queue_collection.queues.inject("") do |m,(k,v)|
|
168
|
+
m + sprintf(QUEUE_STATS_RESPONSE,
|
169
|
+
k, v.length,
|
170
|
+
k, v.total_items,
|
171
|
+
k, v.logsize,
|
172
|
+
k, @expiry_stats[k])
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
def logger
|
177
|
+
@server.logger
|
178
|
+
end
|
179
|
+
|
180
|
+
end
|
181
|
+
end
|