sparrow 0.3.1 → 0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Manifest.txt +0 -1
- data/README.txt +2 -0
- data/Rakefile +1 -0
- data/bin/sparrow +2 -1
- data/lib/sparrow.rb +1 -2
- data/lib/sparrow/queue.rb +7 -10
- data/lib/sparrow/queues/memory.rb +29 -4
- data/lib/sparrow/queues/sqlite.rb +3 -2
- data/lib/sparrow/runner.rb +6 -7
- metadata +72 -54
- data/lib/sparrow/queues/disk.rb +0 -109
data/Manifest.txt
CHANGED
data/README.txt
CHANGED
@@ -11,6 +11,8 @@ Sparrow
|
|
11
11
|
# The load Sparrow can cope with increases exponentially as you add to the cluster.
|
12
12
|
# Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance.
|
13
13
|
#
|
14
|
+
# Sparrow is a in-memory queue but will persist the data to disk when receiving a term signal.
|
15
|
+
#
|
14
16
|
# Sparrow comes with built in support for daemonization and clustering.
|
15
17
|
# Also included are example libraries and clients. For example:
|
16
18
|
#
|
data/Rakefile
CHANGED
@@ -13,6 +13,7 @@ Hoe.new('sparrow', Sparrow::VERSION) do |p|
|
|
13
13
|
p.url = 'http://code.google.com/p/sparrow'
|
14
14
|
p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
|
15
15
|
p.extra_deps << ['eventmachine', '>=0.10.0']
|
16
|
+
p.extra_deps << ['sqlite3-ruby', '>=1.2.2']
|
16
17
|
end
|
17
18
|
|
18
19
|
# vim: syntax=Ruby
|
data/bin/sparrow
CHANGED
data/lib/sparrow.rb
CHANGED
@@ -8,7 +8,7 @@ module Sparrow
|
|
8
8
|
class SparrowError < StandardError #:nodoc:
|
9
9
|
end
|
10
10
|
|
11
|
-
VERSION = '0.
|
11
|
+
VERSION = '0.4'
|
12
12
|
|
13
13
|
@@options = {}
|
14
14
|
|
@@ -48,6 +48,5 @@ end
|
|
48
48
|
require 'sparrow/server'
|
49
49
|
require 'sparrow/queues/sqlite' rescue LoadError nil
|
50
50
|
require 'sparrow/queues/memory'
|
51
|
-
require 'sparrow/queues/disk'
|
52
51
|
require 'sparrow/queue'
|
53
52
|
require 'sparrow/runner'
|
data/lib/sparrow/queue.rb
CHANGED
@@ -7,12 +7,7 @@ module Sparrow
|
|
7
7
|
|
8
8
|
class << self
|
9
9
|
def get_queue(queue_name)
|
10
|
-
@@queues[queue_name] ||=
|
11
|
-
when 'memory': Sparrow::Queues::Memory.new(queue_name)
|
12
|
-
when 'sqlite': Sparrow::Queues::Sqlite.new(queue_name)
|
13
|
-
else
|
14
|
-
Sparrow::Queues::Disk.new(queue_name)
|
15
|
-
end
|
10
|
+
@@queues[queue_name] ||= Sparrow::Queues::Memory.new(queue_name)
|
16
11
|
end
|
17
12
|
|
18
13
|
def next_message(queue_name)
|
@@ -31,14 +26,17 @@ module Sparrow
|
|
31
26
|
end
|
32
27
|
|
33
28
|
def delete_all
|
29
|
+
@@queues.each {|name, q| q.clear! }
|
34
30
|
@@queues = {}
|
35
|
-
|
36
|
-
|
31
|
+
end
|
32
|
+
|
33
|
+
def shutdown!
|
34
|
+
@@queues.each {|name, q| q.shutdown! }
|
37
35
|
end
|
38
36
|
|
39
37
|
def get_stats(queue_name)
|
40
38
|
stats = {
|
41
|
-
:type =>
|
39
|
+
:type => 'memory',
|
42
40
|
:total_bytes => (File.size?(Sparrow.base_dir) || 0),
|
43
41
|
:queues => Dir.glob(File.join(Sparrow.base_dir, '*')).collect {|s| File.basename(s) }.join(','),
|
44
42
|
:number_of_queues => queues.keys.length,
|
@@ -53,7 +51,6 @@ module Sparrow
|
|
53
51
|
if queue_name
|
54
52
|
queue = get_queue(queue_name)
|
55
53
|
stats.merge!({
|
56
|
-
:bytes => Dir.glob(File.join(Sparrow.base_dir, queue_name + '**')).inject(0){|a, b| a += (File.size?(b) || 0); a },
|
57
54
|
:total_items => queue.count_push,
|
58
55
|
:curr_items => queue.count
|
59
56
|
})
|
@@ -13,26 +13,51 @@ module Sparrow
|
|
13
13
|
self.queue_data = []
|
14
14
|
self.count_pop = 0
|
15
15
|
self.count_push = 0
|
16
|
+
recover!
|
16
17
|
end
|
17
18
|
|
18
19
|
def pop
|
19
20
|
self.count_pop += 1
|
20
|
-
queue_data.shift
|
21
|
+
self.queue_data.shift
|
21
22
|
end
|
22
23
|
|
23
24
|
def push(value)
|
24
25
|
self.count_push += 1
|
25
|
-
queue_data.push(value)
|
26
|
+
self.queue_data.push(value)
|
26
27
|
end
|
27
28
|
|
28
|
-
def clear
|
29
|
+
def clear!
|
29
30
|
self.queue_data = []
|
31
|
+
self.sqlite.clear!
|
30
32
|
end
|
31
33
|
|
32
34
|
def count
|
33
35
|
queue_data.length
|
34
36
|
end
|
35
|
-
|
37
|
+
|
38
|
+
def to_disk!
|
39
|
+
copy = self.queue_data.dup
|
40
|
+
copy.each do |value|
|
41
|
+
self.sqlite.push(value)
|
42
|
+
end
|
43
|
+
self.queue_data = self.queue_data - copy
|
44
|
+
end
|
45
|
+
|
46
|
+
def shutdown!
|
47
|
+
self.to_disk!
|
48
|
+
end
|
49
|
+
|
50
|
+
def recover!
|
51
|
+
logger.debug "Recovering queue"
|
52
|
+
while msg = self.sqlite.pop
|
53
|
+
self.push(msg)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def sqlite
|
58
|
+
@sqlite ||= Sparrow::Queues::Sqlite.new(self.queue_name)
|
59
|
+
end
|
60
|
+
|
36
61
|
end
|
37
62
|
end
|
38
63
|
end
|
@@ -35,16 +35,17 @@ module Sparrow
|
|
35
35
|
|
36
36
|
def pop
|
37
37
|
id, value = db.get_first_row("SELECT * FROM queues LIMIT 1;")
|
38
|
+
return unless id and value
|
38
39
|
db.execute("DELETE FROM queues WHERE id = ?", id)
|
39
40
|
self.count_pop += 1
|
40
41
|
value
|
41
42
|
end
|
42
43
|
|
43
44
|
def count
|
44
|
-
db.get_first_value("SELECT
|
45
|
+
db.get_first_value("SELECT count(*) FROM queues").to_i
|
45
46
|
end
|
46
47
|
|
47
|
-
def clear
|
48
|
+
def clear!
|
48
49
|
db.execute("DELETE FROM queues")
|
49
50
|
end
|
50
51
|
|
data/lib/sparrow/runner.rb
CHANGED
@@ -22,7 +22,7 @@ module Sparrow
|
|
22
22
|
|
23
23
|
self.options.merge!({
|
24
24
|
:base_dir => base_dir,
|
25
|
-
:pid_dir
|
25
|
+
:pid_dir => pid_dir,
|
26
26
|
:log_path => log_path
|
27
27
|
})
|
28
28
|
|
@@ -64,6 +64,9 @@ module Sparrow
|
|
64
64
|
end
|
65
65
|
|
66
66
|
def stop
|
67
|
+
puts
|
68
|
+
puts "Writing queues to disk..."
|
69
|
+
Sparrow::Queue.shutdown!
|
67
70
|
puts "Stopping Eventmachine Server"
|
68
71
|
EventMachine::stop
|
69
72
|
end
|
@@ -72,7 +75,7 @@ module Sparrow
|
|
72
75
|
OptionParser.new do |opts|
|
73
76
|
opts.summary_width = 25
|
74
77
|
opts.banner = "Sparrow (#{VERSION})\n\n",
|
75
|
-
"Usage: sparrow [-b path] [-
|
78
|
+
"Usage: sparrow [-b path] [-h host] [-p port] [-P file]\n",
|
76
79
|
" [-d] [-k port] [-l file] [-e]\n",
|
77
80
|
" sparrow --help\n",
|
78
81
|
" sparrow --version\n"
|
@@ -84,10 +87,6 @@ module Sparrow
|
|
84
87
|
options[:base_dir] = File.expand_path(v)
|
85
88
|
end
|
86
89
|
|
87
|
-
opts.on("-t", "--type QUEUE_TYPE", String, "Type of queue (disk/memory/sqlite).", "(default: #{options[:type]})") do |v|
|
88
|
-
options[:type] = v
|
89
|
-
end
|
90
|
-
|
91
90
|
opts.separator ""; opts.separator "Network:"
|
92
91
|
|
93
92
|
opts.on("-h", "--host HOST", String, "Specify host", "(default: #{options[:host]})") do |v|
|
@@ -154,7 +153,7 @@ module Sparrow
|
|
154
153
|
puts f
|
155
154
|
pid = IO.read(f).chomp.to_i
|
156
155
|
FileUtils.rm f
|
157
|
-
Process.kill(
|
156
|
+
Process.kill(15, pid) # TERM
|
158
157
|
puts "killed PID: #{pid}"
|
159
158
|
rescue => e
|
160
159
|
puts "Failed to kill! #{k}: #{e}"
|
metadata
CHANGED
@@ -1,33 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
|
-
rubygems_version: 0.9.4
|
3
|
-
specification_version: 1
|
4
2
|
name: sparrow
|
5
3
|
version: !ruby/object:Gem::Version
|
6
|
-
version: 0.
|
7
|
-
date: 2008-02-03 00:00:00 +00:00
|
8
|
-
summary: Simple file based messagine queue using the memcache protocol
|
9
|
-
require_paths:
|
10
|
-
- lib
|
11
|
-
email: info@eribium.org
|
12
|
-
homepage: http://code.google.com/p/sparrow
|
13
|
-
rubyforge_project: Sparrow
|
14
|
-
description: "# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached. # That means you can use Sparrow with any memcached client library (Ruby or otherwise). # # Basic tests shows that Sparrow processes messages at a rate of 850-900 per second. # The load Sparrow can cope with increases exponentially as you add to the cluster. # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance. # # Sparrow comes with built in support for daemonization and clustering. # Also included are example libraries and clients. For example: # # require 'memcache' # m = MemCache.new('127.0.0.1:11212') # m['queue_name'] = '1' # Publish to queue # m['queue_name'] #=> 1 Pull next msg from queue # m['queue_name'] #=> nil # m.delete('queue_name) # Delete queue # # # or using the included client: # # class MyQueue < MQ3::Queue # def on_message # logger.info \"Received msg with args: #{args.inspect}\" # end # end # # MyQueue.servers = [ # MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1}) # ] # MyQueue.publish('test msg') # MyQueue.run # # Messages are deleted as soon as they're read and the order you add messages to the queue probably won't # be the same order when they're removed. # # Additional memcached commands that are supported are: # flush_all # Deletes all queues # version # quit # The memcached commands 'add', and 'replace' just call 'set'. # # Call sparrow with --help for usage options # # The daemonization won't work on Windows. # # Check out the code: # svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow # # Sparrow was inspired by Twitter's Starling"
|
15
|
-
autorequire:
|
16
|
-
default_executable:
|
17
|
-
bindir: bin
|
18
|
-
has_rdoc: true
|
19
|
-
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
-
requirements:
|
21
|
-
- - ">"
|
22
|
-
- !ruby/object:Gem::Version
|
23
|
-
version: 0.0.0
|
24
|
-
version:
|
4
|
+
version: "0.4"
|
25
5
|
platform: ruby
|
26
|
-
signing_key:
|
27
|
-
cert_chain:
|
28
|
-
post_install_message:
|
29
6
|
authors:
|
30
7
|
- Alex MacCAw
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2008-08-02 00:00:00 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: eventmachine
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.10.0
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: sqlite3-ruby
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.2.2
|
34
|
+
version:
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: hoe
|
37
|
+
type: :development
|
38
|
+
version_requirement:
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 1.7.0
|
44
|
+
version:
|
45
|
+
description: "# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached. # That means you can use Sparrow with any memcached client library (Ruby or otherwise). # # Basic tests shows that Sparrow processes messages at a rate of 850-900 per second. # The load Sparrow can cope with increases exponentially as you add to the cluster. # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance. # # Sparrow is a in-memory queue but will persist the data to disk when receiving a term signal. # # Sparrow comes with built in support for daemonization and clustering. # Also included are example libraries and clients. For example: # # require 'memcache' # m = MemCache.new('127.0.0.1:11212') # m['queue_name'] = '1' # Publish to queue # m['queue_name'] #=> 1 Pull next msg from queue # m['queue_name'] #=> nil # m.delete('queue_name) # Delete queue # # # or using the included client: # # class MyQueue < MQ3::Queue # def on_message # logger.info \"Received msg with args: #{args.inspect}\" # end # end # # MyQueue.servers = [ # MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1}) # ] # MyQueue.publish('test msg') # MyQueue.run # # Messages are deleted as soon as they're read and the order you add messages to the queue probably won't # be the same order when they're removed. # # Additional memcached commands that are supported are: # flush_all # Deletes all queues # version # quit # The memcached commands 'add', and 'replace' just call 'set'. # # Call sparrow with --help for usage options # # The daemonization won't work on Windows. # # Check out the code: # svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow # # Sparrow was inspired by Twitter's Starling"
|
46
|
+
email: info@eribium.org
|
47
|
+
executables:
|
48
|
+
- sparrow
|
49
|
+
extensions: []
|
50
|
+
|
51
|
+
extra_rdoc_files:
|
52
|
+
- History.txt
|
53
|
+
- Manifest.txt
|
54
|
+
- README.txt
|
31
55
|
files:
|
32
56
|
- History.txt
|
33
57
|
- Manifest.txt
|
@@ -36,43 +60,37 @@ files:
|
|
36
60
|
- bin/sparrow
|
37
61
|
- lib/sparrow.rb
|
38
62
|
- lib/sparrow/queue.rb
|
39
|
-
- lib/sparrow/queues/disk.rb
|
40
63
|
- lib/sparrow/queues/memory.rb
|
41
64
|
- lib/sparrow/queues/sqlite.rb
|
42
65
|
- lib/sparrow/runner.rb
|
43
66
|
- lib/sparrow/server.rb
|
44
67
|
- lib/sparrow/utils.rb
|
45
|
-
|
46
|
-
|
68
|
+
has_rdoc: true
|
69
|
+
homepage: http://code.google.com/p/sparrow
|
70
|
+
post_install_message:
|
47
71
|
rdoc_options:
|
48
72
|
- --main
|
49
73
|
- README.txt
|
50
|
-
|
51
|
-
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
-
|
56
|
-
|
57
|
-
|
74
|
+
require_paths:
|
75
|
+
- lib
|
76
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: "0"
|
81
|
+
version:
|
82
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: "0"
|
87
|
+
version:
|
58
88
|
requirements: []
|
59
89
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: 0.10.0
|
69
|
-
version:
|
70
|
-
- !ruby/object:Gem::Dependency
|
71
|
-
name: hoe
|
72
|
-
version_requirement:
|
73
|
-
version_requirements: !ruby/object:Gem::Version::Requirement
|
74
|
-
requirements:
|
75
|
-
- - ">="
|
76
|
-
- !ruby/object:Gem::Version
|
77
|
-
version: 1.5.0
|
78
|
-
version:
|
90
|
+
rubyforge_project: Sparrow
|
91
|
+
rubygems_version: 1.2.0
|
92
|
+
signing_key:
|
93
|
+
specification_version: 2
|
94
|
+
summary: Simple file based messagine queue using the memcache protocol
|
95
|
+
test_files: []
|
96
|
+
|
data/lib/sparrow/queues/disk.rb
DELETED
@@ -1,109 +0,0 @@
|
|
1
|
-
require 'fileutils'
|
2
|
-
module Sparrow
|
3
|
-
module Queues
|
4
|
-
class Disk
|
5
|
-
include Sparrow::Miscel
|
6
|
-
|
7
|
-
TRX_CMD_PUSH = "\000".freeze
|
8
|
-
TRX_CMD_POP = "\001".freeze
|
9
|
-
|
10
|
-
TRX_PUSH = "\000%s%s".freeze
|
11
|
-
TRX_POP = "\001".freeze
|
12
|
-
|
13
|
-
attr_accessor :queue_name
|
14
|
-
attr_accessor :trxr
|
15
|
-
attr_accessor :trxw
|
16
|
-
attr_accessor :count_pop
|
17
|
-
attr_accessor :count_push
|
18
|
-
|
19
|
-
def initialize(queue_name)
|
20
|
-
self.queue_name = queue_name
|
21
|
-
self.count_pop = 0
|
22
|
-
self.count_push = 0
|
23
|
-
open_queue
|
24
|
-
end
|
25
|
-
|
26
|
-
def push(value)
|
27
|
-
value = value.to_s
|
28
|
-
size = [value.size].pack("I")
|
29
|
-
data = sprintf(TRX_PUSH, size, value)
|
30
|
-
trxw.seek(0, IO::SEEK_END)
|
31
|
-
trxw.write data
|
32
|
-
# trxw.fsync
|
33
|
-
rotate_queue if trxw.pos > max_log_size
|
34
|
-
self.count_push += 1
|
35
|
-
value
|
36
|
-
end
|
37
|
-
|
38
|
-
def pop
|
39
|
-
while !trxr.eof?
|
40
|
-
s_pos = trxr.pos
|
41
|
-
cmd = trxr.read(1)
|
42
|
-
if cmd != TRX_CMD_POP and cmd != TRX_CMD_PUSH
|
43
|
-
logger.fatal 'Corrupt queue'
|
44
|
-
return
|
45
|
-
end
|
46
|
-
raw_size = trxr.read(4)
|
47
|
-
size = raw_size.unpack("I").first
|
48
|
-
value = trxr.read(size)
|
49
|
-
next if cmd == TRX_CMD_POP
|
50
|
-
e_pos = trxr.pos
|
51
|
-
trxr.seek(s_pos, IO::SEEK_SET)
|
52
|
-
trxr.write(TRX_POP)
|
53
|
-
# trxr.fsync
|
54
|
-
trxr.pos = e_pos
|
55
|
-
next unless value
|
56
|
-
self.count_pop += 1
|
57
|
-
return value
|
58
|
-
end
|
59
|
-
|
60
|
-
if trxr.path == queue_path
|
61
|
-
File.truncate(trxr.path, 0)
|
62
|
-
else
|
63
|
-
FileUtils.rm_rf trxr.path
|
64
|
-
end
|
65
|
-
open_reader
|
66
|
-
nil
|
67
|
-
end
|
68
|
-
|
69
|
-
def clear
|
70
|
-
dirs = Dir.glob(queue_path) | Dir.glob(queue_path + '.*')
|
71
|
-
FileUtils.rm_rf(dirs) unless dirs.empty?
|
72
|
-
end
|
73
|
-
|
74
|
-
def count
|
75
|
-
self.count_push - self.count_pop
|
76
|
-
end
|
77
|
-
|
78
|
-
private
|
79
|
-
|
80
|
-
def queue_path
|
81
|
-
File.join(base_dir, queue_name)
|
82
|
-
end
|
83
|
-
|
84
|
-
def rotate_queue
|
85
|
-
File.rename(queue_path, File.join(base_dir, "#{queue_name}.#{Time.now.to_i}"))
|
86
|
-
open_writer
|
87
|
-
end
|
88
|
-
|
89
|
-
def open_writer
|
90
|
-
self.trxw = File.open(queue_path, 'a+')
|
91
|
-
end
|
92
|
-
|
93
|
-
def open_reader
|
94
|
-
old_queue = Dir.glob(queue_path + '.*').first
|
95
|
-
self.trxr = File.open(old_queue||queue_path, 'r+')
|
96
|
-
end
|
97
|
-
|
98
|
-
def open_queue
|
99
|
-
open_writer
|
100
|
-
open_reader
|
101
|
-
end
|
102
|
-
|
103
|
-
def max_log_size
|
104
|
-
@max_log_size ||= (options[:log_size] || 16) * (1024**2) # 16mb
|
105
|
-
end
|
106
|
-
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|