sparrow 0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +5 -0
- data/Manifest.txt +13 -0
- data/README.txt +87 -0
- data/Rakefile +18 -0
- data/bin/sparrow +3 -0
- data/lib/sparrow.rb +53 -0
- data/lib/sparrow/queue.rb +41 -0
- data/lib/sparrow/queues/disk.rb +105 -0
- data/lib/sparrow/queues/memory.rb +35 -0
- data/lib/sparrow/queues/sqlite.rb +56 -0
- data/lib/sparrow/runner.rb +166 -0
- data/lib/sparrow/server.rb +201 -0
- data/lib/sparrow/utils.rb +71 -0
- metadata +78 -0
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
History.txt
|
2
|
+
Manifest.txt
|
3
|
+
README.txt
|
4
|
+
Rakefile
|
5
|
+
bin/sparrow
|
6
|
+
lib/sparrow.rb
|
7
|
+
lib/sparrow/queue.rb
|
8
|
+
lib/sparrow/queues/disk.rb
|
9
|
+
lib/sparrow/queues/memory.rb
|
10
|
+
lib/sparrow/queues/sqlite.rb
|
11
|
+
lib/sparrow/runner.rb
|
12
|
+
lib/sparrow/server.rb
|
13
|
+
lib/sparrow/utils.rb
|
data/README.txt
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
Sparrow
|
2
|
+
by Alex MacCaw
|
3
|
+
http://code.google.com/p/sparrow/
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached.
|
8
|
+
# That means you can use Sparrow with any memcached client library (Ruby or otherwise).
|
9
|
+
#
|
10
|
+
# Basic tests shows that Sparrow processes messages at a rate of 850-900 per second.
|
11
|
+
# The load Sparrow can cope with increases exponentially as you add to the cluster.
|
12
|
+
# Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance.
|
13
|
+
#
|
14
|
+
# Sparrow comes with built in support for daemonization and clustering.
|
15
|
+
# Also included are example libraries and clients. For example:
|
16
|
+
#
|
17
|
+
# require 'memcache'
|
18
|
+
# m = MemCache.new('127.0.0.1:11212')
|
19
|
+
# m['queue_name'] = '1' # Publish to queue
|
20
|
+
# m['queue_name'] #=> 1 Pull next msg from queue
|
21
|
+
# m['queue_name'] #=> nil
|
22
|
+
# m.delete('queue_name) # Delete queue
|
23
|
+
#
|
24
|
+
# # or using the included client:
|
25
|
+
#
|
26
|
+
# class MyQueue < MQ3::Queue
|
27
|
+
# def on_message
|
28
|
+
# logger.info "Received msg with args: #{args.inspect}"
|
29
|
+
# end
|
30
|
+
# end
|
31
|
+
#
|
32
|
+
# MyQueue.servers = [
|
33
|
+
# MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1})
|
34
|
+
# ]
|
35
|
+
# MyQueue.publish('test msg')
|
36
|
+
# MyQueue.run
|
37
|
+
#
|
38
|
+
# Messages are deleted as soon as they're read and the order you add messages to the queue probably won't
|
39
|
+
# be the same order when they're removed.
|
40
|
+
#
|
41
|
+
# Additional memcached commands that are supported are:
|
42
|
+
# flush_all # Deletes all queues
|
43
|
+
# version
|
44
|
+
# quit
|
45
|
+
# The memcached commands 'add', and 'replace' just call 'set'.
|
46
|
+
#
|
47
|
+
# Call sparrow with --help for usage options
|
48
|
+
#
|
49
|
+
# The daemonization won't work on Windows.
|
50
|
+
#
|
51
|
+
# Check out the code:
|
52
|
+
# svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow
|
53
|
+
#
|
54
|
+
# Sparrow was inspired by Twitter's Starling
|
55
|
+
|
56
|
+
== REQUIREMENTS:
|
57
|
+
|
58
|
+
* eventmachine
|
59
|
+
|
60
|
+
== INSTALL:
|
61
|
+
|
62
|
+
* sudo gem install sparrow
|
63
|
+
|
64
|
+
== LICENSE:
|
65
|
+
|
66
|
+
(The MIT License)
|
67
|
+
|
68
|
+
Copyright (c) 2008 FIX
|
69
|
+
|
70
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
71
|
+
a copy of this software and associated documentation files (the
|
72
|
+
'Software'), to deal in the Software without restriction, including
|
73
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
74
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
75
|
+
permit persons to whom the Software is furnished to do so, subject to
|
76
|
+
the following conditions:
|
77
|
+
|
78
|
+
The above copyright notice and this permission notice shall be
|
79
|
+
included in all copies or substantial portions of the Software.
|
80
|
+
|
81
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
82
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
83
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
84
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
85
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
86
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
87
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'hoe'
|
5
|
+
require './lib/sparrow.rb'
|
6
|
+
|
7
|
+
Hoe.new('sparrow', Sparrow::VERSION) do |p|
|
8
|
+
p.rubyforge_name = 'Sparrow'
|
9
|
+
p.author = 'Alex MacCAw'
|
10
|
+
p.email = 'info@eribium.org'
|
11
|
+
p.summary = 'Simple file based messagine queue using the memcache protocol'
|
12
|
+
# p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
|
13
|
+
p.url = 'http://code.google.com/p/sparrow'
|
14
|
+
p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
|
15
|
+
p.extra_deps << ['eventmachine', '>=0.10.0']
|
16
|
+
end
|
17
|
+
|
18
|
+
# vim: syntax=Ruby
|
data/bin/sparrow
ADDED
data/lib/sparrow.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
$:.unshift(File.dirname(__FILE__))
|
4
|
+
require 'sparrow/utils'
|
5
|
+
|
6
|
+
module Sparrow
|
7
|
+
|
8
|
+
class SparrowError < StandardError #:nodoc:
|
9
|
+
end
|
10
|
+
|
11
|
+
VERSION = '0.2'
|
12
|
+
|
13
|
+
@@options = {}
|
14
|
+
|
15
|
+
class << self
|
16
|
+
def options
|
17
|
+
@@options
|
18
|
+
end
|
19
|
+
|
20
|
+
def options=(val)
|
21
|
+
@@options = val
|
22
|
+
end
|
23
|
+
|
24
|
+
def logger
|
25
|
+
return @@logger if defined?(@@loggger)
|
26
|
+
FileUtils.mkdir_p(File.dirname(log_path))
|
27
|
+
@@logger = Logger.new(log_path)
|
28
|
+
@@logger.level = Logger::INFO if options[:debug] == false
|
29
|
+
@@logger
|
30
|
+
rescue
|
31
|
+
@@logger = Logger.new(STDOUT)
|
32
|
+
end
|
33
|
+
|
34
|
+
def base_dir
|
35
|
+
options[:base_dir] || File.join(%w( / var spool sparrow ))
|
36
|
+
end
|
37
|
+
|
38
|
+
def log_path
|
39
|
+
options[:log_path] || File.join(%w( / var run sparrow.log ))
|
40
|
+
end
|
41
|
+
|
42
|
+
def pid_path
|
43
|
+
options[:pid_path] || File.join(%w( / var run sparrow.#{options[:port]}.pid ))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
require 'sparrow/server'
|
49
|
+
require 'sparrow/queues/sqlite' rescue LoadError nil
|
50
|
+
require 'sparrow/queues/memory'
|
51
|
+
require 'sparrow/queues/disk'
|
52
|
+
require 'sparrow/queue'
|
53
|
+
require 'sparrow/runner'
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
module Sparrow
|
3
|
+
class Queue
|
4
|
+
|
5
|
+
cattr_accessor :queues
|
6
|
+
self.queues = {}
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def get_queue(queue_name)
|
10
|
+
@@queues[queue_name] ||= case Sparrow.options[:type]
|
11
|
+
when 'memory': Sparrow::Queues::Memory.new(queue_name)
|
12
|
+
when 'sqlite': Sparrow::Queues::Sqlite.new(queue_name)
|
13
|
+
else
|
14
|
+
Sparrow::Queues::Disk.new(queue_name)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def next_message(queue_name)
|
19
|
+
self.get_queue(queue_name).pop
|
20
|
+
end
|
21
|
+
|
22
|
+
def add_message(queue_name, value)
|
23
|
+
self.get_queue(queue_name).push(value)
|
24
|
+
end
|
25
|
+
|
26
|
+
def delete(queue_name)
|
27
|
+
queue = self.get_queue(queue_name)
|
28
|
+
queue.clear
|
29
|
+
@@queues.delete(queue_name)
|
30
|
+
true
|
31
|
+
end
|
32
|
+
|
33
|
+
def delete_all
|
34
|
+
@@queues = {}
|
35
|
+
FileUtils.rm_rf base_dir
|
36
|
+
FileUtils.mkdir_p base_dir
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
module Sparrow
|
3
|
+
module Queues
|
4
|
+
class Disk
|
5
|
+
include Sparrow::Miscel
|
6
|
+
|
7
|
+
TRX_CMD_PUSH = "\000".freeze
|
8
|
+
TRX_CMD_POP = "\001".freeze
|
9
|
+
|
10
|
+
TRX_PUSH = "\000%s%s".freeze
|
11
|
+
TRX_POP = "\001".freeze
|
12
|
+
|
13
|
+
attr_accessor :queue_name
|
14
|
+
attr_accessor :trxr
|
15
|
+
attr_accessor :trxw
|
16
|
+
attr_accessor :count_all
|
17
|
+
|
18
|
+
def initialize(queue_name)
|
19
|
+
self.queue_name = queue_name
|
20
|
+
self.count_all = 0
|
21
|
+
open_queue
|
22
|
+
end
|
23
|
+
|
24
|
+
def push(value)
|
25
|
+
value = value.to_s
|
26
|
+
size = [value.size].pack("I")
|
27
|
+
data = sprintf(TRX_PUSH, size, value)
|
28
|
+
trxw.seek(0, IO::SEEK_END)
|
29
|
+
trxw.write data
|
30
|
+
trxw.fsync
|
31
|
+
rotate_queue if trxw.pos > max_log_size
|
32
|
+
self.count_all += 1
|
33
|
+
value
|
34
|
+
end
|
35
|
+
|
36
|
+
def pop
|
37
|
+
while !trxr.eof?
|
38
|
+
s_pos = trxr.pos
|
39
|
+
cmd = trxr.read(1)
|
40
|
+
if cmd != TRX_CMD_POP and cmd != TRX_CMD_PUSH
|
41
|
+
logger.fatal 'Corrupt queue'
|
42
|
+
return
|
43
|
+
end
|
44
|
+
raw_size = trxr.read(4)
|
45
|
+
size = raw_size.unpack("I").first
|
46
|
+
value = trxr.read(size)
|
47
|
+
next if cmd == TRX_CMD_POP
|
48
|
+
e_pos = trxr.pos
|
49
|
+
trxr.seek(s_pos, IO::SEEK_SET)
|
50
|
+
trxr.write(TRX_POP)
|
51
|
+
trxr.fsync
|
52
|
+
trxr.pos = e_pos
|
53
|
+
next unless value
|
54
|
+
return value
|
55
|
+
end
|
56
|
+
|
57
|
+
if trxr.path == queue_path
|
58
|
+
File.truncate(trxr.path, 0)
|
59
|
+
else
|
60
|
+
FileUtils.rm_rf trxr.path
|
61
|
+
end
|
62
|
+
open_reader
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
def clear
|
67
|
+
dirs = Dir.glob(queue_path) | Dir.glob(queue_path + '.*')
|
68
|
+
FileUtils.rm_rf(dirs) unless dirs.empty?
|
69
|
+
end
|
70
|
+
|
71
|
+
def count
|
72
|
+
end
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def queue_path
|
77
|
+
File.join(base_dir, queue_name)
|
78
|
+
end
|
79
|
+
|
80
|
+
def rotate_queue
|
81
|
+
File.rename(queue_path, File.join(base_dir, "#{queue_name}.#{Time.now.to_i}"))
|
82
|
+
open_writer
|
83
|
+
end
|
84
|
+
|
85
|
+
def open_writer
|
86
|
+
self.trxw = File.open(queue_path, 'a+')
|
87
|
+
end
|
88
|
+
|
89
|
+
def open_reader
|
90
|
+
old_queue = Dir.glob(queue_path + '.*').first
|
91
|
+
self.trxr = File.open(old_queue||queue_path, 'r+')
|
92
|
+
end
|
93
|
+
|
94
|
+
def open_queue
|
95
|
+
open_writer
|
96
|
+
open_reader
|
97
|
+
end
|
98
|
+
|
99
|
+
def max_log_size
|
100
|
+
@max_log_size ||= (options[:log_size] || 16) * (1024**2) # 16mb
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Sparrow
|
2
|
+
module Queues
|
3
|
+
class Memory
|
4
|
+
include Sparrow::Miscel
|
5
|
+
|
6
|
+
attr_accessor :queue_name
|
7
|
+
attr_accessor :queue_data
|
8
|
+
attr_accessor :count_all
|
9
|
+
|
10
|
+
def initialize(queue_name)
|
11
|
+
self.queue_name = queue_name
|
12
|
+
self.queue_data = []
|
13
|
+
self.count_all = 0
|
14
|
+
end
|
15
|
+
|
16
|
+
def pop
|
17
|
+
queue_data.shift
|
18
|
+
end
|
19
|
+
|
20
|
+
def push(value)
|
21
|
+
self.count_all += 1
|
22
|
+
queue_data.push(value)
|
23
|
+
end
|
24
|
+
|
25
|
+
def clear
|
26
|
+
self.queue_data = []
|
27
|
+
end
|
28
|
+
|
29
|
+
def count
|
30
|
+
queue_data.length
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'sqlite3'
|
2
|
+
module Sparrow
|
3
|
+
module Queues
|
4
|
+
class Sqlite
|
5
|
+
include Sparrow::Miscel
|
6
|
+
|
7
|
+
attr_accessor :queue_name
|
8
|
+
attr_accessor :db
|
9
|
+
attr_accessor :count_all
|
10
|
+
|
11
|
+
def initialize(queue_name)
|
12
|
+
self.queue_name = queue_name
|
13
|
+
self.count_all = 0
|
14
|
+
db_exists = File.exists?(db_path)
|
15
|
+
self.db = SQLite3::Database.new( db_path )
|
16
|
+
if !db_exists
|
17
|
+
self.db.execute_batch <<-SQL
|
18
|
+
CREATE TABLE queues (
|
19
|
+
id INTEGER PRIMARY KEY,
|
20
|
+
data VARCHAR(255)
|
21
|
+
);
|
22
|
+
PRAGMA default_synchronous=OFF;
|
23
|
+
PRAGMA count_changes=OFF;
|
24
|
+
SQL
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def push(value)
|
29
|
+
self.count_all += 1
|
30
|
+
db.execute("INSERT INTO queues (data) VALUES (?);", value)
|
31
|
+
value
|
32
|
+
end
|
33
|
+
|
34
|
+
def pop
|
35
|
+
id, value = db.get_first_row("SELECT * FROM queues LIMIT 1;")
|
36
|
+
db.execute("DELETE FROM queues WHERE id = ?", id)
|
37
|
+
value
|
38
|
+
end
|
39
|
+
|
40
|
+
def count
|
41
|
+
db.get_first_value("SELECT COUNT FROM queues")
|
42
|
+
end
|
43
|
+
|
44
|
+
def clear
|
45
|
+
db.execute("DELETE FROM queues")
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def db_path
|
51
|
+
File.join(base_dir, queue_name)
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,166 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'eventmachine'
|
3
|
+
|
4
|
+
module Sparrow
|
5
|
+
class Runner
|
6
|
+
include Sparrow::Miscel
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def run
|
10
|
+
self.new
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
self.options = {
|
16
|
+
:host => "0.0.0.0",
|
17
|
+
:port => 11212,
|
18
|
+
:debug => false,
|
19
|
+
:base_dir => base_dir,
|
20
|
+
:pid_path => pid_path,
|
21
|
+
:log_path => log_path,
|
22
|
+
:type => 'disk'
|
23
|
+
}
|
24
|
+
|
25
|
+
parse_options
|
26
|
+
|
27
|
+
if options.include?(:kill)
|
28
|
+
kill_pid(options[:kill] || '*')
|
29
|
+
end
|
30
|
+
|
31
|
+
if !options[:daemonize]
|
32
|
+
start
|
33
|
+
else
|
34
|
+
daemonize
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def start
|
39
|
+
puts "Starting Sparrow server on port: #{options[:port]}..."
|
40
|
+
|
41
|
+
trap("INT") {
|
42
|
+
stop
|
43
|
+
exit
|
44
|
+
}
|
45
|
+
trap("TERM"){
|
46
|
+
stop
|
47
|
+
exit
|
48
|
+
}
|
49
|
+
|
50
|
+
EventMachine::run {
|
51
|
+
EventMachine::start_server(options[:host], options[:port].to_i, Sparrow::Server)
|
52
|
+
}
|
53
|
+
end
|
54
|
+
|
55
|
+
def stop
|
56
|
+
puts "Stopping Eventmachine Server"
|
57
|
+
EventMachine::stop
|
58
|
+
end
|
59
|
+
|
60
|
+
def parse_options
|
61
|
+
OptionParser.new do |opts|
|
62
|
+
opts.summary_width = 25
|
63
|
+
opts.banner = "Sparrow (#{VERSION})\n\n",
|
64
|
+
"Usage: sparrow [-b path] [-t type] [-h host] [-p port] [-P file]\n",
|
65
|
+
" [-d] [-k port] [-l file] [-e]\n",
|
66
|
+
" sparrow --help\n",
|
67
|
+
" sparrow --version\n"
|
68
|
+
|
69
|
+
opts.separator ""
|
70
|
+
opts.separator ""; opts.separator "Configuration:"
|
71
|
+
|
72
|
+
opts.on("-b", "--base PATH", String, "Path to queue data store.", "(default: #{options[:base_dir]})") do |v|
|
73
|
+
options[:base_dir] = File.expand_path(v)
|
74
|
+
end
|
75
|
+
|
76
|
+
opts.on("-t", "--type QUEUE_TYPE", String, "Type of queue (disk/memory/sqlite).", "(default: #{options[:type]})") do |v|
|
77
|
+
options[:type] = v
|
78
|
+
end
|
79
|
+
|
80
|
+
opts.separator ""; opts.separator "Network:"
|
81
|
+
|
82
|
+
opts.on("-h", "--host HOST", String, "Specify host", "(default: #{options[:host]})") do |v|
|
83
|
+
options[:host] = v
|
84
|
+
end
|
85
|
+
|
86
|
+
opts.on("-p", "--port PORT", Integer, "Specify port", "(default: #{options[:port]})") do |v|
|
87
|
+
options[:port] = v
|
88
|
+
end
|
89
|
+
|
90
|
+
opts.separator ""; opts.separator "Daemonization:"
|
91
|
+
|
92
|
+
opts.on("-P", "--pid FILE", String, "save PID in FILE when using -d option.", "(default: #{options[:pid_path]})") do |v|
|
93
|
+
options[:pid_path] = File.expand_path(v)
|
94
|
+
end
|
95
|
+
|
96
|
+
opts.on("-d", "--daemon", "Daemonize mode") do |v|
|
97
|
+
options[:daemonize] = v
|
98
|
+
end
|
99
|
+
|
100
|
+
opts.on("-k", "--kill PORT", String, "Kill specified running daemons - leave blank to kill all.") do |v|
|
101
|
+
options[:kill] = v
|
102
|
+
end
|
103
|
+
|
104
|
+
opts.separator ""; opts.separator "Logging:"
|
105
|
+
|
106
|
+
opts.on("-l", "--log [FILE]", String, "Path to print debugging information.") do |v|
|
107
|
+
options[:log_path] = File.expand_path(v)
|
108
|
+
end
|
109
|
+
|
110
|
+
opts.on("-e", "--debug", "Run in debug mode", "(default: #{options[:debug]})") do |v|
|
111
|
+
options[:debug] = v
|
112
|
+
end
|
113
|
+
|
114
|
+
opts.separator ""; opts.separator "Miscellaneous:"
|
115
|
+
|
116
|
+
opts.on_tail("-?", "--help", "Display this usage information.") do
|
117
|
+
puts "#{opts}\n"
|
118
|
+
exit
|
119
|
+
end
|
120
|
+
|
121
|
+
opts.on_tail("-v", "--version", "Display version") do |v|
|
122
|
+
puts "Sparrow #{VERSION}"
|
123
|
+
exit
|
124
|
+
end
|
125
|
+
end.parse!
|
126
|
+
options
|
127
|
+
end
|
128
|
+
|
129
|
+
private
|
130
|
+
|
131
|
+
def store_pid(pid)
|
132
|
+
FileUtils.mkdir_p(File.dirname(pid_path))
|
133
|
+
File.open(pid_path, 'w'){|f| f.write("#{pid}\n")}
|
134
|
+
end
|
135
|
+
|
136
|
+
def kill_pid(k)
|
137
|
+
Dir[options[:pid_path]||File.join(File.dirname(pid_dir), "sparrow.#{k}.pid")].each do |f|
|
138
|
+
begin
|
139
|
+
puts f
|
140
|
+
pid = IO.read(f).chomp.to_i
|
141
|
+
FileUtils.rm f
|
142
|
+
Process.kill(9, pid)
|
143
|
+
puts "killed PID: #{pid}"
|
144
|
+
rescue => e
|
145
|
+
puts "Failed to kill! #{k}: #{e}"
|
146
|
+
end
|
147
|
+
end
|
148
|
+
exit
|
149
|
+
end
|
150
|
+
|
151
|
+
def daemonize
|
152
|
+
fork do
|
153
|
+
Process.setsid
|
154
|
+
exit if fork
|
155
|
+
store_pid(Process.pid)
|
156
|
+
# Dir.chdir "/" # Mucks up logs
|
157
|
+
File.umask 0000
|
158
|
+
STDIN.reopen "/dev/null"
|
159
|
+
STDOUT.reopen "/dev/null", "a"
|
160
|
+
STDERR.reopen STDOUT
|
161
|
+
start
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
end
|
166
|
+
end
|
@@ -0,0 +1,201 @@
|
|
1
|
+
module Sparrow
|
2
|
+
module Server
|
3
|
+
include Sparrow::Miscel
|
4
|
+
|
5
|
+
class NoMoreMessages < SparrowError #:nodoc:
|
6
|
+
end
|
7
|
+
|
8
|
+
class ClientError < SparrowError #:nodoc:
|
9
|
+
end
|
10
|
+
|
11
|
+
class StatementInvalid < ClientError #:nodoc:
|
12
|
+
end
|
13
|
+
|
14
|
+
class InvalidBodyLength < ClientError #:nodoc:
|
15
|
+
end
|
16
|
+
|
17
|
+
CR = "\r\n"
|
18
|
+
ERROR = "ERROR"
|
19
|
+
OK = "OK"
|
20
|
+
EOF = "END"
|
21
|
+
|
22
|
+
CLIENT_ERROR = "CLIENT_ERROR"
|
23
|
+
SERVER_ERROR = "SERVER_ERROR"
|
24
|
+
|
25
|
+
STORED = "STORED"
|
26
|
+
NOT_STORED = "NOT_STORED"
|
27
|
+
|
28
|
+
DELETED = "DELETED"
|
29
|
+
NOT_FOUND = "NOT_FOUND"
|
30
|
+
|
31
|
+
VALUE = "VALUE"
|
32
|
+
|
33
|
+
VERSION = "VERSION"
|
34
|
+
|
35
|
+
SET_REGEX = /\ASET\s/i
|
36
|
+
ADD_REGEX = /\AADD\s/i
|
37
|
+
REPLACE_REGEX = /\AREPLACE\s/i
|
38
|
+
DELETE_REGEX = /\ADELETE\s/i
|
39
|
+
GET_REGEX = /\AGET\s/i
|
40
|
+
QUIT_REGEX = /\AQUIT/i
|
41
|
+
FLUSH_ALL_REGEX = /\AFLUSH_ALL/i
|
42
|
+
VERSION_REGEX = /\AVERSION/i
|
43
|
+
|
44
|
+
def post_init
|
45
|
+
@current_queue = nil
|
46
|
+
@expecting_body = false
|
47
|
+
@expected_bytes = 0
|
48
|
+
@current_flag = nil
|
49
|
+
@buffer = ''
|
50
|
+
logger.debug "New client"
|
51
|
+
end
|
52
|
+
|
53
|
+
def receive_data(data)
|
54
|
+
logger.debug "Receiving data: #{data}"
|
55
|
+
@buffer << data
|
56
|
+
@buffer = process_whole_messages(@buffer)
|
57
|
+
end
|
58
|
+
|
59
|
+
# process any whole messages in the buffer,
|
60
|
+
# and return the new contents of the buffer
|
61
|
+
def process_whole_messages(data)
|
62
|
+
return data if data !~ /\r\n/i # only process if data contains a CR
|
63
|
+
messages = data.split(CR)
|
64
|
+
if data =~ /\r\n$/i
|
65
|
+
data = ''
|
66
|
+
else
|
67
|
+
# remove the last message from the list (because it is incomplete) before processing
|
68
|
+
data = messages.pop
|
69
|
+
end
|
70
|
+
messages.each {|message| process_message(message) }
|
71
|
+
return data
|
72
|
+
end
|
73
|
+
|
74
|
+
def process_message ln
|
75
|
+
@data = ln
|
76
|
+
if ln =~ SET_REGEX
|
77
|
+
set_command
|
78
|
+
elsif ln =~ ADD_REGEX
|
79
|
+
add_command
|
80
|
+
elsif ln =~ REPLACE_REGEX
|
81
|
+
replace_command
|
82
|
+
elsif ln =~ GET_REGEX
|
83
|
+
get_command
|
84
|
+
elsif ln =~ DELETE_REGEX
|
85
|
+
delete_command
|
86
|
+
elsif ln =~ QUIT_REGEX
|
87
|
+
quit_command
|
88
|
+
elsif ln =~ VERSION_REGEX
|
89
|
+
version_command
|
90
|
+
elsif ln =~ FLUSH_ALL_REGEX
|
91
|
+
flush_all_command
|
92
|
+
elsif @expecting_body
|
93
|
+
process_body
|
94
|
+
else
|
95
|
+
raise StatementInvalid
|
96
|
+
end
|
97
|
+
@data = nil
|
98
|
+
@split_args = nil
|
99
|
+
|
100
|
+
rescue ClientError => e
|
101
|
+
logger.error e
|
102
|
+
publish CLIENT_ERROR, e
|
103
|
+
publish ERROR
|
104
|
+
rescue => e
|
105
|
+
debugger
|
106
|
+
logger.error e
|
107
|
+
publish SERVER_ERROR, e
|
108
|
+
end
|
109
|
+
|
110
|
+
def publish *args
|
111
|
+
send_data args.join(' ') + CR
|
112
|
+
end
|
113
|
+
|
114
|
+
# Storage commands
|
115
|
+
|
116
|
+
# <command name> <key> <flags> <exptime> <bytes>\r\n
|
117
|
+
def set_command
|
118
|
+
@current_queue = args[1]
|
119
|
+
@current_flag = args[2] || 0
|
120
|
+
raise ClientError unless @current_queue
|
121
|
+
@expected_bytes = args[4].to_i || 0
|
122
|
+
@expecting_body = true
|
123
|
+
end
|
124
|
+
alias add_command set_command
|
125
|
+
alias replace_command set_command
|
126
|
+
|
127
|
+
def process_body
|
128
|
+
if @data.length != @expected_bytes
|
129
|
+
raise InvalidBodyLength
|
130
|
+
end
|
131
|
+
@data << @current_flag
|
132
|
+
logger.debug "Adding message to queue - #{@current_queue}"
|
133
|
+
Sparrow::Queue.add_message(@current_queue, @data)
|
134
|
+
@expected_bytes = 0
|
135
|
+
@current_queue = nil
|
136
|
+
@expecting_body = false
|
137
|
+
publish STORED
|
138
|
+
end
|
139
|
+
|
140
|
+
# Retrieval commands
|
141
|
+
|
142
|
+
# GET <key>*r\n
|
143
|
+
def get_command
|
144
|
+
args.shift # get rid of the command
|
145
|
+
raise ClientError if args.empty?
|
146
|
+
rsp = []
|
147
|
+
args.each do |queue|
|
148
|
+
begin
|
149
|
+
logger.debug "Getting message from queue - #{queue}"
|
150
|
+
msg = Sparrow::Queue.next_message(queue)
|
151
|
+
next unless msg
|
152
|
+
rescue NoMoreMessages
|
153
|
+
next
|
154
|
+
end
|
155
|
+
flag = msg[-1..-1]
|
156
|
+
msg = msg[0..-2]
|
157
|
+
rsp << [VALUE, queue, flag, msg.length].join(' ')
|
158
|
+
rsp << msg
|
159
|
+
end
|
160
|
+
rsp << EOF
|
161
|
+
send_data(rsp.join(CR) + CR)
|
162
|
+
end
|
163
|
+
|
164
|
+
# Other commands
|
165
|
+
|
166
|
+
# DELETE <key> <time>\r\n
|
167
|
+
def delete_command
|
168
|
+
if Sparrow::Queue.delete(!args[1])
|
169
|
+
logger.info "Deleting queue - #{args[1]}"
|
170
|
+
publish DELETED
|
171
|
+
else
|
172
|
+
publish NOT_FOUND
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
# FLUSH_ALL
|
177
|
+
def flush_all_command
|
178
|
+
logger.info "Flushing all queues"
|
179
|
+
Sparrow::Queue.delete_all
|
180
|
+
publish OK
|
181
|
+
end
|
182
|
+
|
183
|
+
# VERSION
|
184
|
+
def version_command
|
185
|
+
publish VERSION, Sparrow::Version
|
186
|
+
end
|
187
|
+
|
188
|
+
# QUIT
|
189
|
+
def quit_command
|
190
|
+
logger.debug "Closing connection"
|
191
|
+
close_connection
|
192
|
+
end
|
193
|
+
|
194
|
+
private
|
195
|
+
|
196
|
+
def args
|
197
|
+
@split_args ||= @data.split(' ')
|
198
|
+
end
|
199
|
+
|
200
|
+
end
|
201
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
class Class # :nodoc:
|
2
|
+
def cattr_reader(*syms)
|
3
|
+
syms.flatten.each do |sym|
|
4
|
+
next if sym.is_a?(Hash)
|
5
|
+
class_eval(<<-EOS, __FILE__, __LINE__)
|
6
|
+
unless defined? @@#{sym}
|
7
|
+
@@#{sym} = nil
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.#{sym}
|
11
|
+
@@#{sym}
|
12
|
+
end
|
13
|
+
|
14
|
+
def #{sym}
|
15
|
+
@@#{sym}
|
16
|
+
end
|
17
|
+
EOS
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def cattr_writer(*syms)
|
22
|
+
syms.flatten.each do |sym|
|
23
|
+
class_eval(<<-EOS, __FILE__, __LINE__)
|
24
|
+
unless defined? @@#{sym}
|
25
|
+
@@#{sym} = nil
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.#{sym}=(obj)
|
29
|
+
@@#{sym} = obj
|
30
|
+
end
|
31
|
+
|
32
|
+
def #{sym}=(obj)
|
33
|
+
@@#{sym} = obj
|
34
|
+
end
|
35
|
+
EOS
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def cattr_accessor(*syms)
|
40
|
+
cattr_reader(*syms)
|
41
|
+
cattr_writer(*syms)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
module Sparrow
|
46
|
+
module Miscel
|
47
|
+
def options
|
48
|
+
Sparrow.options
|
49
|
+
end
|
50
|
+
|
51
|
+
def options=(ob)
|
52
|
+
Sparrow.options = ob
|
53
|
+
end
|
54
|
+
|
55
|
+
def base_dir
|
56
|
+
Sparrow.base_dir
|
57
|
+
end
|
58
|
+
|
59
|
+
def log_path
|
60
|
+
Sparrow.log_path
|
61
|
+
end
|
62
|
+
|
63
|
+
def pid_path
|
64
|
+
Sparrow.log_path
|
65
|
+
end
|
66
|
+
|
67
|
+
def logger
|
68
|
+
Sparrow.logger
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
metadata
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.9.4
|
3
|
+
specification_version: 1
|
4
|
+
name: sparrow
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: "0.2"
|
7
|
+
date: 2008-01-26 00:00:00 +00:00
|
8
|
+
summary: Simple file based messagine queue using the memcache protocol
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: info@eribium.org
|
12
|
+
homepage: http://code.google.com/p/sparrow
|
13
|
+
rubyforge_project: Sparrow
|
14
|
+
description: The author was too lazy to write a description
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: true
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">"
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.0.0
|
24
|
+
version:
|
25
|
+
platform: ruby
|
26
|
+
signing_key:
|
27
|
+
cert_chain:
|
28
|
+
post_install_message:
|
29
|
+
authors:
|
30
|
+
- Alex MacCAw
|
31
|
+
files:
|
32
|
+
- History.txt
|
33
|
+
- Manifest.txt
|
34
|
+
- README.txt
|
35
|
+
- Rakefile
|
36
|
+
- bin/sparrow
|
37
|
+
- lib/sparrow.rb
|
38
|
+
- lib/sparrow/queue.rb
|
39
|
+
- lib/sparrow/queues/disk.rb
|
40
|
+
- lib/sparrow/queues/memory.rb
|
41
|
+
- lib/sparrow/queues/sqlite.rb
|
42
|
+
- lib/sparrow/runner.rb
|
43
|
+
- lib/sparrow/server.rb
|
44
|
+
- lib/sparrow/utils.rb
|
45
|
+
test_files: []
|
46
|
+
|
47
|
+
rdoc_options:
|
48
|
+
- --main
|
49
|
+
- README.txt
|
50
|
+
extra_rdoc_files:
|
51
|
+
- History.txt
|
52
|
+
- Manifest.txt
|
53
|
+
- README.txt
|
54
|
+
executables:
|
55
|
+
- sparrow
|
56
|
+
extensions: []
|
57
|
+
|
58
|
+
requirements: []
|
59
|
+
|
60
|
+
dependencies:
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: eventmachine
|
63
|
+
version_requirement:
|
64
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.10.0
|
69
|
+
version:
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: hoe
|
72
|
+
version_requirement:
|
73
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 1.4.0
|
78
|
+
version:
|