sparrow 0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +5 -0
- data/Manifest.txt +13 -0
- data/README.txt +87 -0
- data/Rakefile +18 -0
- data/bin/sparrow +3 -0
- data/lib/sparrow.rb +53 -0
- data/lib/sparrow/queue.rb +41 -0
- data/lib/sparrow/queues/disk.rb +105 -0
- data/lib/sparrow/queues/memory.rb +35 -0
- data/lib/sparrow/queues/sqlite.rb +56 -0
- data/lib/sparrow/runner.rb +166 -0
- data/lib/sparrow/server.rb +201 -0
- data/lib/sparrow/utils.rb +71 -0
- metadata +78 -0
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
History.txt
|
2
|
+
Manifest.txt
|
3
|
+
README.txt
|
4
|
+
Rakefile
|
5
|
+
bin/sparrow
|
6
|
+
lib/sparrow.rb
|
7
|
+
lib/sparrow/queue.rb
|
8
|
+
lib/sparrow/queues/disk.rb
|
9
|
+
lib/sparrow/queues/memory.rb
|
10
|
+
lib/sparrow/queues/sqlite.rb
|
11
|
+
lib/sparrow/runner.rb
|
12
|
+
lib/sparrow/server.rb
|
13
|
+
lib/sparrow/utils.rb
|
data/README.txt
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
Sparrow
|
2
|
+
by Alex MacCaw
|
3
|
+
http://code.google.com/p/sparrow/
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached.
|
8
|
+
# That means you can use Sparrow with any memcached client library (Ruby or otherwise).
|
9
|
+
#
|
10
|
+
# Basic tests shows that Sparrow processes messages at a rate of 850-900 per second.
|
11
|
+
# The load Sparrow can cope with increases exponentially as you add to the cluster.
|
12
|
+
# Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance.
|
13
|
+
#
|
14
|
+
# Sparrow comes with built in support for daemonization and clustering.
|
15
|
+
# Also included are example libraries and clients. For example:
|
16
|
+
#
|
17
|
+
# require 'memcache'
|
18
|
+
# m = MemCache.new('127.0.0.1:11212')
|
19
|
+
# m['queue_name'] = '1' # Publish to queue
|
20
|
+
# m['queue_name'] #=> 1 Pull next msg from queue
|
21
|
+
# m['queue_name'] #=> nil
|
22
|
+
# m.delete('queue_name) # Delete queue
|
23
|
+
#
|
24
|
+
# # or using the included client:
|
25
|
+
#
|
26
|
+
# class MyQueue < MQ3::Queue
|
27
|
+
# def on_message
|
28
|
+
# logger.info "Received msg with args: #{args.inspect}"
|
29
|
+
# end
|
30
|
+
# end
|
31
|
+
#
|
32
|
+
# MyQueue.servers = [
|
33
|
+
# MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1})
|
34
|
+
# ]
|
35
|
+
# MyQueue.publish('test msg')
|
36
|
+
# MyQueue.run
|
37
|
+
#
|
38
|
+
# Messages are deleted as soon as they're read and the order you add messages to the queue probably won't
|
39
|
+
# be the same order when they're removed.
|
40
|
+
#
|
41
|
+
# Additional memcached commands that are supported are:
|
42
|
+
# flush_all # Deletes all queues
|
43
|
+
# version
|
44
|
+
# quit
|
45
|
+
# The memcached commands 'add', and 'replace' just call 'set'.
|
46
|
+
#
|
47
|
+
# Call sparrow with --help for usage options
|
48
|
+
#
|
49
|
+
# The daemonization won't work on Windows.
|
50
|
+
#
|
51
|
+
# Check out the code:
|
52
|
+
# svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow
|
53
|
+
#
|
54
|
+
# Sparrow was inspired by Twitter's Starling
|
55
|
+
|
56
|
+
== REQUIREMENTS:
|
57
|
+
|
58
|
+
* eventmachine
|
59
|
+
|
60
|
+
== INSTALL:
|
61
|
+
|
62
|
+
* sudo gem install sparrow
|
63
|
+
|
64
|
+
== LICENSE:
|
65
|
+
|
66
|
+
(The MIT License)
|
67
|
+
|
68
|
+
Copyright (c) 2008 FIX
|
69
|
+
|
70
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
71
|
+
a copy of this software and associated documentation files (the
|
72
|
+
'Software'), to deal in the Software without restriction, including
|
73
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
74
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
75
|
+
permit persons to whom the Software is furnished to do so, subject to
|
76
|
+
the following conditions:
|
77
|
+
|
78
|
+
The above copyright notice and this permission notice shall be
|
79
|
+
included in all copies or substantial portions of the Software.
|
80
|
+
|
81
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
82
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
83
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
84
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
85
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
86
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
87
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'hoe'
|
5
|
+
require './lib/sparrow.rb'
|
6
|
+
|
7
|
+
Hoe.new('sparrow', Sparrow::VERSION) do |p|
|
8
|
+
p.rubyforge_name = 'Sparrow'
|
9
|
+
p.author = 'Alex MacCAw'
|
10
|
+
p.email = 'info@eribium.org'
|
11
|
+
p.summary = 'Simple file based messagine queue using the memcache protocol'
|
12
|
+
# p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
|
13
|
+
p.url = 'http://code.google.com/p/sparrow'
|
14
|
+
p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
|
15
|
+
p.extra_deps << ['eventmachine', '>=0.10.0']
|
16
|
+
end
|
17
|
+
|
18
|
+
# vim: syntax=Ruby
|
data/bin/sparrow
ADDED
data/lib/sparrow.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
$:.unshift(File.dirname(__FILE__))
|
4
|
+
require 'sparrow/utils'
|
5
|
+
|
6
|
+
module Sparrow
|
7
|
+
|
8
|
+
class SparrowError < StandardError #:nodoc:
|
9
|
+
end
|
10
|
+
|
11
|
+
VERSION = '0.2'
|
12
|
+
|
13
|
+
@@options = {}
|
14
|
+
|
15
|
+
class << self
|
16
|
+
def options
|
17
|
+
@@options
|
18
|
+
end
|
19
|
+
|
20
|
+
def options=(val)
|
21
|
+
@@options = val
|
22
|
+
end
|
23
|
+
|
24
|
+
def logger
|
25
|
+
return @@logger if defined?(@@loggger)
|
26
|
+
FileUtils.mkdir_p(File.dirname(log_path))
|
27
|
+
@@logger = Logger.new(log_path)
|
28
|
+
@@logger.level = Logger::INFO if options[:debug] == false
|
29
|
+
@@logger
|
30
|
+
rescue
|
31
|
+
@@logger = Logger.new(STDOUT)
|
32
|
+
end
|
33
|
+
|
34
|
+
def base_dir
|
35
|
+
options[:base_dir] || File.join(%w( / var spool sparrow ))
|
36
|
+
end
|
37
|
+
|
38
|
+
def log_path
|
39
|
+
options[:log_path] || File.join(%w( / var run sparrow.log ))
|
40
|
+
end
|
41
|
+
|
42
|
+
def pid_path
|
43
|
+
options[:pid_path] || File.join(%w( / var run sparrow.#{options[:port]}.pid ))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
require 'sparrow/server'
|
49
|
+
require 'sparrow/queues/sqlite' rescue LoadError nil
|
50
|
+
require 'sparrow/queues/memory'
|
51
|
+
require 'sparrow/queues/disk'
|
52
|
+
require 'sparrow/queue'
|
53
|
+
require 'sparrow/runner'
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
module Sparrow
|
3
|
+
class Queue
|
4
|
+
|
5
|
+
cattr_accessor :queues
|
6
|
+
self.queues = {}
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def get_queue(queue_name)
|
10
|
+
@@queues[queue_name] ||= case Sparrow.options[:type]
|
11
|
+
when 'memory': Sparrow::Queues::Memory.new(queue_name)
|
12
|
+
when 'sqlite': Sparrow::Queues::Sqlite.new(queue_name)
|
13
|
+
else
|
14
|
+
Sparrow::Queues::Disk.new(queue_name)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def next_message(queue_name)
|
19
|
+
self.get_queue(queue_name).pop
|
20
|
+
end
|
21
|
+
|
22
|
+
def add_message(queue_name, value)
|
23
|
+
self.get_queue(queue_name).push(value)
|
24
|
+
end
|
25
|
+
|
26
|
+
def delete(queue_name)
|
27
|
+
queue = self.get_queue(queue_name)
|
28
|
+
queue.clear
|
29
|
+
@@queues.delete(queue_name)
|
30
|
+
true
|
31
|
+
end
|
32
|
+
|
33
|
+
def delete_all
|
34
|
+
@@queues = {}
|
35
|
+
FileUtils.rm_rf base_dir
|
36
|
+
FileUtils.mkdir_p base_dir
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
module Sparrow
|
3
|
+
module Queues
|
4
|
+
class Disk
|
5
|
+
include Sparrow::Miscel
|
6
|
+
|
7
|
+
TRX_CMD_PUSH = "\000".freeze
|
8
|
+
TRX_CMD_POP = "\001".freeze
|
9
|
+
|
10
|
+
TRX_PUSH = "\000%s%s".freeze
|
11
|
+
TRX_POP = "\001".freeze
|
12
|
+
|
13
|
+
attr_accessor :queue_name
|
14
|
+
attr_accessor :trxr
|
15
|
+
attr_accessor :trxw
|
16
|
+
attr_accessor :count_all
|
17
|
+
|
18
|
+
def initialize(queue_name)
|
19
|
+
self.queue_name = queue_name
|
20
|
+
self.count_all = 0
|
21
|
+
open_queue
|
22
|
+
end
|
23
|
+
|
24
|
+
def push(value)
|
25
|
+
value = value.to_s
|
26
|
+
size = [value.size].pack("I")
|
27
|
+
data = sprintf(TRX_PUSH, size, value)
|
28
|
+
trxw.seek(0, IO::SEEK_END)
|
29
|
+
trxw.write data
|
30
|
+
trxw.fsync
|
31
|
+
rotate_queue if trxw.pos > max_log_size
|
32
|
+
self.count_all += 1
|
33
|
+
value
|
34
|
+
end
|
35
|
+
|
36
|
+
def pop
|
37
|
+
while !trxr.eof?
|
38
|
+
s_pos = trxr.pos
|
39
|
+
cmd = trxr.read(1)
|
40
|
+
if cmd != TRX_CMD_POP and cmd != TRX_CMD_PUSH
|
41
|
+
logger.fatal 'Corrupt queue'
|
42
|
+
return
|
43
|
+
end
|
44
|
+
raw_size = trxr.read(4)
|
45
|
+
size = raw_size.unpack("I").first
|
46
|
+
value = trxr.read(size)
|
47
|
+
next if cmd == TRX_CMD_POP
|
48
|
+
e_pos = trxr.pos
|
49
|
+
trxr.seek(s_pos, IO::SEEK_SET)
|
50
|
+
trxr.write(TRX_POP)
|
51
|
+
trxr.fsync
|
52
|
+
trxr.pos = e_pos
|
53
|
+
next unless value
|
54
|
+
return value
|
55
|
+
end
|
56
|
+
|
57
|
+
if trxr.path == queue_path
|
58
|
+
File.truncate(trxr.path, 0)
|
59
|
+
else
|
60
|
+
FileUtils.rm_rf trxr.path
|
61
|
+
end
|
62
|
+
open_reader
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
def clear
|
67
|
+
dirs = Dir.glob(queue_path) | Dir.glob(queue_path + '.*')
|
68
|
+
FileUtils.rm_rf(dirs) unless dirs.empty?
|
69
|
+
end
|
70
|
+
|
71
|
+
def count
|
72
|
+
end
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def queue_path
|
77
|
+
File.join(base_dir, queue_name)
|
78
|
+
end
|
79
|
+
|
80
|
+
def rotate_queue
|
81
|
+
File.rename(queue_path, File.join(base_dir, "#{queue_name}.#{Time.now.to_i}"))
|
82
|
+
open_writer
|
83
|
+
end
|
84
|
+
|
85
|
+
def open_writer
|
86
|
+
self.trxw = File.open(queue_path, 'a+')
|
87
|
+
end
|
88
|
+
|
89
|
+
def open_reader
|
90
|
+
old_queue = Dir.glob(queue_path + '.*').first
|
91
|
+
self.trxr = File.open(old_queue||queue_path, 'r+')
|
92
|
+
end
|
93
|
+
|
94
|
+
def open_queue
|
95
|
+
open_writer
|
96
|
+
open_reader
|
97
|
+
end
|
98
|
+
|
99
|
+
def max_log_size
|
100
|
+
@max_log_size ||= (options[:log_size] || 16) * (1024**2) # 16mb
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Sparrow
|
2
|
+
module Queues
|
3
|
+
class Memory
|
4
|
+
include Sparrow::Miscel
|
5
|
+
|
6
|
+
attr_accessor :queue_name
|
7
|
+
attr_accessor :queue_data
|
8
|
+
attr_accessor :count_all
|
9
|
+
|
10
|
+
def initialize(queue_name)
|
11
|
+
self.queue_name = queue_name
|
12
|
+
self.queue_data = []
|
13
|
+
self.count_all = 0
|
14
|
+
end
|
15
|
+
|
16
|
+
def pop
|
17
|
+
queue_data.shift
|
18
|
+
end
|
19
|
+
|
20
|
+
def push(value)
|
21
|
+
self.count_all += 1
|
22
|
+
queue_data.push(value)
|
23
|
+
end
|
24
|
+
|
25
|
+
def clear
|
26
|
+
self.queue_data = []
|
27
|
+
end
|
28
|
+
|
29
|
+
def count
|
30
|
+
queue_data.length
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'sqlite3'
|
2
|
+
module Sparrow
|
3
|
+
module Queues
|
4
|
+
class Sqlite
|
5
|
+
include Sparrow::Miscel
|
6
|
+
|
7
|
+
attr_accessor :queue_name
|
8
|
+
attr_accessor :db
|
9
|
+
attr_accessor :count_all
|
10
|
+
|
11
|
+
def initialize(queue_name)
|
12
|
+
self.queue_name = queue_name
|
13
|
+
self.count_all = 0
|
14
|
+
db_exists = File.exists?(db_path)
|
15
|
+
self.db = SQLite3::Database.new( db_path )
|
16
|
+
if !db_exists
|
17
|
+
self.db.execute_batch <<-SQL
|
18
|
+
CREATE TABLE queues (
|
19
|
+
id INTEGER PRIMARY KEY,
|
20
|
+
data VARCHAR(255)
|
21
|
+
);
|
22
|
+
PRAGMA default_synchronous=OFF;
|
23
|
+
PRAGMA count_changes=OFF;
|
24
|
+
SQL
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def push(value)
|
29
|
+
self.count_all += 1
|
30
|
+
db.execute("INSERT INTO queues (data) VALUES (?);", value)
|
31
|
+
value
|
32
|
+
end
|
33
|
+
|
34
|
+
def pop
|
35
|
+
id, value = db.get_first_row("SELECT * FROM queues LIMIT 1;")
|
36
|
+
db.execute("DELETE FROM queues WHERE id = ?", id)
|
37
|
+
value
|
38
|
+
end
|
39
|
+
|
40
|
+
def count
|
41
|
+
db.get_first_value("SELECT COUNT FROM queues")
|
42
|
+
end
|
43
|
+
|
44
|
+
def clear
|
45
|
+
db.execute("DELETE FROM queues")
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def db_path
|
51
|
+
File.join(base_dir, queue_name)
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,166 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'eventmachine'
|
3
|
+
|
4
|
+
module Sparrow
|
5
|
+
class Runner
|
6
|
+
include Sparrow::Miscel
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def run
|
10
|
+
self.new
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
self.options = {
|
16
|
+
:host => "0.0.0.0",
|
17
|
+
:port => 11212,
|
18
|
+
:debug => false,
|
19
|
+
:base_dir => base_dir,
|
20
|
+
:pid_path => pid_path,
|
21
|
+
:log_path => log_path,
|
22
|
+
:type => 'disk'
|
23
|
+
}
|
24
|
+
|
25
|
+
parse_options
|
26
|
+
|
27
|
+
if options.include?(:kill)
|
28
|
+
kill_pid(options[:kill] || '*')
|
29
|
+
end
|
30
|
+
|
31
|
+
if !options[:daemonize]
|
32
|
+
start
|
33
|
+
else
|
34
|
+
daemonize
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def start
|
39
|
+
puts "Starting Sparrow server on port: #{options[:port]}..."
|
40
|
+
|
41
|
+
trap("INT") {
|
42
|
+
stop
|
43
|
+
exit
|
44
|
+
}
|
45
|
+
trap("TERM"){
|
46
|
+
stop
|
47
|
+
exit
|
48
|
+
}
|
49
|
+
|
50
|
+
EventMachine::run {
|
51
|
+
EventMachine::start_server(options[:host], options[:port].to_i, Sparrow::Server)
|
52
|
+
}
|
53
|
+
end
|
54
|
+
|
55
|
+
def stop
|
56
|
+
puts "Stopping Eventmachine Server"
|
57
|
+
EventMachine::stop
|
58
|
+
end
|
59
|
+
|
60
|
+
def parse_options
|
61
|
+
OptionParser.new do |opts|
|
62
|
+
opts.summary_width = 25
|
63
|
+
opts.banner = "Sparrow (#{VERSION})\n\n",
|
64
|
+
"Usage: sparrow [-b path] [-t type] [-h host] [-p port] [-P file]\n",
|
65
|
+
" [-d] [-k port] [-l file] [-e]\n",
|
66
|
+
" sparrow --help\n",
|
67
|
+
" sparrow --version\n"
|
68
|
+
|
69
|
+
opts.separator ""
|
70
|
+
opts.separator ""; opts.separator "Configuration:"
|
71
|
+
|
72
|
+
opts.on("-b", "--base PATH", String, "Path to queue data store.", "(default: #{options[:base_dir]})") do |v|
|
73
|
+
options[:base_dir] = File.expand_path(v)
|
74
|
+
end
|
75
|
+
|
76
|
+
opts.on("-t", "--type QUEUE_TYPE", String, "Type of queue (disk/memory/sqlite).", "(default: #{options[:type]})") do |v|
|
77
|
+
options[:type] = v
|
78
|
+
end
|
79
|
+
|
80
|
+
opts.separator ""; opts.separator "Network:"
|
81
|
+
|
82
|
+
opts.on("-h", "--host HOST", String, "Specify host", "(default: #{options[:host]})") do |v|
|
83
|
+
options[:host] = v
|
84
|
+
end
|
85
|
+
|
86
|
+
opts.on("-p", "--port PORT", Integer, "Specify port", "(default: #{options[:port]})") do |v|
|
87
|
+
options[:port] = v
|
88
|
+
end
|
89
|
+
|
90
|
+
opts.separator ""; opts.separator "Daemonization:"
|
91
|
+
|
92
|
+
opts.on("-P", "--pid FILE", String, "save PID in FILE when using -d option.", "(default: #{options[:pid_path]})") do |v|
|
93
|
+
options[:pid_path] = File.expand_path(v)
|
94
|
+
end
|
95
|
+
|
96
|
+
opts.on("-d", "--daemon", "Daemonize mode") do |v|
|
97
|
+
options[:daemonize] = v
|
98
|
+
end
|
99
|
+
|
100
|
+
opts.on("-k", "--kill PORT", String, "Kill specified running daemons - leave blank to kill all.") do |v|
|
101
|
+
options[:kill] = v
|
102
|
+
end
|
103
|
+
|
104
|
+
opts.separator ""; opts.separator "Logging:"
|
105
|
+
|
106
|
+
opts.on("-l", "--log [FILE]", String, "Path to print debugging information.") do |v|
|
107
|
+
options[:log_path] = File.expand_path(v)
|
108
|
+
end
|
109
|
+
|
110
|
+
opts.on("-e", "--debug", "Run in debug mode", "(default: #{options[:debug]})") do |v|
|
111
|
+
options[:debug] = v
|
112
|
+
end
|
113
|
+
|
114
|
+
opts.separator ""; opts.separator "Miscellaneous:"
|
115
|
+
|
116
|
+
opts.on_tail("-?", "--help", "Display this usage information.") do
|
117
|
+
puts "#{opts}\n"
|
118
|
+
exit
|
119
|
+
end
|
120
|
+
|
121
|
+
opts.on_tail("-v", "--version", "Display version") do |v|
|
122
|
+
puts "Sparrow #{VERSION}"
|
123
|
+
exit
|
124
|
+
end
|
125
|
+
end.parse!
|
126
|
+
options
|
127
|
+
end
|
128
|
+
|
129
|
+
private
|
130
|
+
|
131
|
+
def store_pid(pid)
|
132
|
+
FileUtils.mkdir_p(File.dirname(pid_path))
|
133
|
+
File.open(pid_path, 'w'){|f| f.write("#{pid}\n")}
|
134
|
+
end
|
135
|
+
|
136
|
+
def kill_pid(k)
|
137
|
+
Dir[options[:pid_path]||File.join(File.dirname(pid_dir), "sparrow.#{k}.pid")].each do |f|
|
138
|
+
begin
|
139
|
+
puts f
|
140
|
+
pid = IO.read(f).chomp.to_i
|
141
|
+
FileUtils.rm f
|
142
|
+
Process.kill(9, pid)
|
143
|
+
puts "killed PID: #{pid}"
|
144
|
+
rescue => e
|
145
|
+
puts "Failed to kill! #{k}: #{e}"
|
146
|
+
end
|
147
|
+
end
|
148
|
+
exit
|
149
|
+
end
|
150
|
+
|
151
|
+
def daemonize
|
152
|
+
fork do
|
153
|
+
Process.setsid
|
154
|
+
exit if fork
|
155
|
+
store_pid(Process.pid)
|
156
|
+
# Dir.chdir "/" # Mucks up logs
|
157
|
+
File.umask 0000
|
158
|
+
STDIN.reopen "/dev/null"
|
159
|
+
STDOUT.reopen "/dev/null", "a"
|
160
|
+
STDERR.reopen STDOUT
|
161
|
+
start
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
end
|
166
|
+
end
|
@@ -0,0 +1,201 @@
|
|
1
|
+
module Sparrow
|
2
|
+
module Server
|
3
|
+
include Sparrow::Miscel
|
4
|
+
|
5
|
+
class NoMoreMessages < SparrowError #:nodoc:
|
6
|
+
end
|
7
|
+
|
8
|
+
class ClientError < SparrowError #:nodoc:
|
9
|
+
end
|
10
|
+
|
11
|
+
class StatementInvalid < ClientError #:nodoc:
|
12
|
+
end
|
13
|
+
|
14
|
+
class InvalidBodyLength < ClientError #:nodoc:
|
15
|
+
end
|
16
|
+
|
17
|
+
CR = "\r\n"
|
18
|
+
ERROR = "ERROR"
|
19
|
+
OK = "OK"
|
20
|
+
EOF = "END"
|
21
|
+
|
22
|
+
CLIENT_ERROR = "CLIENT_ERROR"
|
23
|
+
SERVER_ERROR = "SERVER_ERROR"
|
24
|
+
|
25
|
+
STORED = "STORED"
|
26
|
+
NOT_STORED = "NOT_STORED"
|
27
|
+
|
28
|
+
DELETED = "DELETED"
|
29
|
+
NOT_FOUND = "NOT_FOUND"
|
30
|
+
|
31
|
+
VALUE = "VALUE"
|
32
|
+
|
33
|
+
VERSION = "VERSION"
|
34
|
+
|
35
|
+
SET_REGEX = /\ASET\s/i
|
36
|
+
ADD_REGEX = /\AADD\s/i
|
37
|
+
REPLACE_REGEX = /\AREPLACE\s/i
|
38
|
+
DELETE_REGEX = /\ADELETE\s/i
|
39
|
+
GET_REGEX = /\AGET\s/i
|
40
|
+
QUIT_REGEX = /\AQUIT/i
|
41
|
+
FLUSH_ALL_REGEX = /\AFLUSH_ALL/i
|
42
|
+
VERSION_REGEX = /\AVERSION/i
|
43
|
+
|
44
|
+
def post_init
|
45
|
+
@current_queue = nil
|
46
|
+
@expecting_body = false
|
47
|
+
@expected_bytes = 0
|
48
|
+
@current_flag = nil
|
49
|
+
@buffer = ''
|
50
|
+
logger.debug "New client"
|
51
|
+
end
|
52
|
+
|
53
|
+
def receive_data(data)
|
54
|
+
logger.debug "Receiving data: #{data}"
|
55
|
+
@buffer << data
|
56
|
+
@buffer = process_whole_messages(@buffer)
|
57
|
+
end
|
58
|
+
|
59
|
+
# process any whole messages in the buffer,
|
60
|
+
# and return the new contents of the buffer
|
61
|
+
def process_whole_messages(data)
|
62
|
+
return data if data !~ /\r\n/i # only process if data contains a CR
|
63
|
+
messages = data.split(CR)
|
64
|
+
if data =~ /\r\n$/i
|
65
|
+
data = ''
|
66
|
+
else
|
67
|
+
# remove the last message from the list (because it is incomplete) before processing
|
68
|
+
data = messages.pop
|
69
|
+
end
|
70
|
+
messages.each {|message| process_message(message) }
|
71
|
+
return data
|
72
|
+
end
|
73
|
+
|
74
|
+
def process_message ln
|
75
|
+
@data = ln
|
76
|
+
if ln =~ SET_REGEX
|
77
|
+
set_command
|
78
|
+
elsif ln =~ ADD_REGEX
|
79
|
+
add_command
|
80
|
+
elsif ln =~ REPLACE_REGEX
|
81
|
+
replace_command
|
82
|
+
elsif ln =~ GET_REGEX
|
83
|
+
get_command
|
84
|
+
elsif ln =~ DELETE_REGEX
|
85
|
+
delete_command
|
86
|
+
elsif ln =~ QUIT_REGEX
|
87
|
+
quit_command
|
88
|
+
elsif ln =~ VERSION_REGEX
|
89
|
+
version_command
|
90
|
+
elsif ln =~ FLUSH_ALL_REGEX
|
91
|
+
flush_all_command
|
92
|
+
elsif @expecting_body
|
93
|
+
process_body
|
94
|
+
else
|
95
|
+
raise StatementInvalid
|
96
|
+
end
|
97
|
+
@data = nil
|
98
|
+
@split_args = nil
|
99
|
+
|
100
|
+
rescue ClientError => e
|
101
|
+
logger.error e
|
102
|
+
publish CLIENT_ERROR, e
|
103
|
+
publish ERROR
|
104
|
+
rescue => e
|
105
|
+
debugger
|
106
|
+
logger.error e
|
107
|
+
publish SERVER_ERROR, e
|
108
|
+
end
|
109
|
+
|
110
|
+
def publish *args
|
111
|
+
send_data args.join(' ') + CR
|
112
|
+
end
|
113
|
+
|
114
|
+
# Storage commands
|
115
|
+
|
116
|
+
# <command name> <key> <flags> <exptime> <bytes>\r\n
|
117
|
+
def set_command
|
118
|
+
@current_queue = args[1]
|
119
|
+
@current_flag = args[2] || 0
|
120
|
+
raise ClientError unless @current_queue
|
121
|
+
@expected_bytes = args[4].to_i || 0
|
122
|
+
@expecting_body = true
|
123
|
+
end
|
124
|
+
alias add_command set_command
|
125
|
+
alias replace_command set_command
|
126
|
+
|
127
|
+
def process_body
|
128
|
+
if @data.length != @expected_bytes
|
129
|
+
raise InvalidBodyLength
|
130
|
+
end
|
131
|
+
@data << @current_flag
|
132
|
+
logger.debug "Adding message to queue - #{@current_queue}"
|
133
|
+
Sparrow::Queue.add_message(@current_queue, @data)
|
134
|
+
@expected_bytes = 0
|
135
|
+
@current_queue = nil
|
136
|
+
@expecting_body = false
|
137
|
+
publish STORED
|
138
|
+
end
|
139
|
+
|
140
|
+
# Retrieval commands
|
141
|
+
|
142
|
+
# GET <key>*r\n
|
143
|
+
def get_command
|
144
|
+
args.shift # get rid of the command
|
145
|
+
raise ClientError if args.empty?
|
146
|
+
rsp = []
|
147
|
+
args.each do |queue|
|
148
|
+
begin
|
149
|
+
logger.debug "Getting message from queue - #{queue}"
|
150
|
+
msg = Sparrow::Queue.next_message(queue)
|
151
|
+
next unless msg
|
152
|
+
rescue NoMoreMessages
|
153
|
+
next
|
154
|
+
end
|
155
|
+
flag = msg[-1..-1]
|
156
|
+
msg = msg[0..-2]
|
157
|
+
rsp << [VALUE, queue, flag, msg.length].join(' ')
|
158
|
+
rsp << msg
|
159
|
+
end
|
160
|
+
rsp << EOF
|
161
|
+
send_data(rsp.join(CR) + CR)
|
162
|
+
end
|
163
|
+
|
164
|
+
# Other commands
|
165
|
+
|
166
|
+
# DELETE <key> <time>\r\n
|
167
|
+
def delete_command
|
168
|
+
if Sparrow::Queue.delete(!args[1])
|
169
|
+
logger.info "Deleting queue - #{args[1]}"
|
170
|
+
publish DELETED
|
171
|
+
else
|
172
|
+
publish NOT_FOUND
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
# FLUSH_ALL
|
177
|
+
def flush_all_command
|
178
|
+
logger.info "Flushing all queues"
|
179
|
+
Sparrow::Queue.delete_all
|
180
|
+
publish OK
|
181
|
+
end
|
182
|
+
|
183
|
+
# VERSION
|
184
|
+
def version_command
|
185
|
+
publish VERSION, Sparrow::Version
|
186
|
+
end
|
187
|
+
|
188
|
+
# QUIT
|
189
|
+
def quit_command
|
190
|
+
logger.debug "Closing connection"
|
191
|
+
close_connection
|
192
|
+
end
|
193
|
+
|
194
|
+
private
|
195
|
+
|
196
|
+
def args
|
197
|
+
@split_args ||= @data.split(' ')
|
198
|
+
end
|
199
|
+
|
200
|
+
end
|
201
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
class Class # :nodoc:
|
2
|
+
def cattr_reader(*syms)
|
3
|
+
syms.flatten.each do |sym|
|
4
|
+
next if sym.is_a?(Hash)
|
5
|
+
class_eval(<<-EOS, __FILE__, __LINE__)
|
6
|
+
unless defined? @@#{sym}
|
7
|
+
@@#{sym} = nil
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.#{sym}
|
11
|
+
@@#{sym}
|
12
|
+
end
|
13
|
+
|
14
|
+
def #{sym}
|
15
|
+
@@#{sym}
|
16
|
+
end
|
17
|
+
EOS
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def cattr_writer(*syms)
|
22
|
+
syms.flatten.each do |sym|
|
23
|
+
class_eval(<<-EOS, __FILE__, __LINE__)
|
24
|
+
unless defined? @@#{sym}
|
25
|
+
@@#{sym} = nil
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.#{sym}=(obj)
|
29
|
+
@@#{sym} = obj
|
30
|
+
end
|
31
|
+
|
32
|
+
def #{sym}=(obj)
|
33
|
+
@@#{sym} = obj
|
34
|
+
end
|
35
|
+
EOS
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def cattr_accessor(*syms)
|
40
|
+
cattr_reader(*syms)
|
41
|
+
cattr_writer(*syms)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
module Sparrow
|
46
|
+
module Miscel
|
47
|
+
def options
|
48
|
+
Sparrow.options
|
49
|
+
end
|
50
|
+
|
51
|
+
def options=(ob)
|
52
|
+
Sparrow.options = ob
|
53
|
+
end
|
54
|
+
|
55
|
+
def base_dir
|
56
|
+
Sparrow.base_dir
|
57
|
+
end
|
58
|
+
|
59
|
+
def log_path
|
60
|
+
Sparrow.log_path
|
61
|
+
end
|
62
|
+
|
63
|
+
def pid_path
|
64
|
+
Sparrow.log_path
|
65
|
+
end
|
66
|
+
|
67
|
+
def logger
|
68
|
+
Sparrow.logger
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
metadata
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.9.4
|
3
|
+
specification_version: 1
|
4
|
+
name: sparrow
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: "0.2"
|
7
|
+
date: 2008-01-26 00:00:00 +00:00
|
8
|
+
summary: Simple file based messagine queue using the memcache protocol
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: info@eribium.org
|
12
|
+
homepage: http://code.google.com/p/sparrow
|
13
|
+
rubyforge_project: Sparrow
|
14
|
+
description: The author was too lazy to write a description
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: true
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">"
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.0.0
|
24
|
+
version:
|
25
|
+
platform: ruby
|
26
|
+
signing_key:
|
27
|
+
cert_chain:
|
28
|
+
post_install_message:
|
29
|
+
authors:
|
30
|
+
- Alex MacCAw
|
31
|
+
files:
|
32
|
+
- History.txt
|
33
|
+
- Manifest.txt
|
34
|
+
- README.txt
|
35
|
+
- Rakefile
|
36
|
+
- bin/sparrow
|
37
|
+
- lib/sparrow.rb
|
38
|
+
- lib/sparrow/queue.rb
|
39
|
+
- lib/sparrow/queues/disk.rb
|
40
|
+
- lib/sparrow/queues/memory.rb
|
41
|
+
- lib/sparrow/queues/sqlite.rb
|
42
|
+
- lib/sparrow/runner.rb
|
43
|
+
- lib/sparrow/server.rb
|
44
|
+
- lib/sparrow/utils.rb
|
45
|
+
test_files: []
|
46
|
+
|
47
|
+
rdoc_options:
|
48
|
+
- --main
|
49
|
+
- README.txt
|
50
|
+
extra_rdoc_files:
|
51
|
+
- History.txt
|
52
|
+
- Manifest.txt
|
53
|
+
- README.txt
|
54
|
+
executables:
|
55
|
+
- sparrow
|
56
|
+
extensions: []
|
57
|
+
|
58
|
+
requirements: []
|
59
|
+
|
60
|
+
dependencies:
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: eventmachine
|
63
|
+
version_requirement:
|
64
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.10.0
|
69
|
+
version:
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: hoe
|
72
|
+
version_requirement:
|
73
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 1.4.0
|
78
|
+
version:
|