sparrow 0.3.1 → 0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,7 +5,6 @@ Rakefile
5
5
  bin/sparrow
6
6
  lib/sparrow.rb
7
7
  lib/sparrow/queue.rb
8
- lib/sparrow/queues/disk.rb
9
8
  lib/sparrow/queues/memory.rb
10
9
  lib/sparrow/queues/sqlite.rb
11
10
  lib/sparrow/runner.rb
data/README.txt CHANGED
@@ -11,6 +11,8 @@ Sparrow
11
11
  # The load Sparrow can cope with increases exponentially as you add to the cluster.
12
12
  # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance.
13
13
  #
14
+ # Sparrow is a in-memory queue but will persist the data to disk when receiving a term signal.
15
+ #
14
16
  # Sparrow comes with built in support for daemonization and clustering.
15
17
  # Also included are example libraries and clients. For example:
16
18
  #
data/Rakefile CHANGED
@@ -13,6 +13,7 @@ Hoe.new('sparrow', Sparrow::VERSION) do |p|
13
13
  p.url = 'http://code.google.com/p/sparrow'
14
14
  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
15
15
  p.extra_deps << ['eventmachine', '>=0.10.0']
16
+ p.extra_deps << ['sqlite3-ruby', '>=1.2.2']
16
17
  end
17
18
 
18
19
  # vim: syntax=Ruby
@@ -1,3 +1,4 @@
1
1
  #!/usr/bin/env ruby
2
- require File.join(File.dirname(__FILE__), '..', 'lib', 'sparrow')
2
+ file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
3
+ require File.join(File.dirname(file), '..', 'lib', 'sparrow')
3
4
  Sparrow::Runner.run
@@ -8,7 +8,7 @@ module Sparrow
8
8
  class SparrowError < StandardError #:nodoc:
9
9
  end
10
10
 
11
- VERSION = '0.3.1'
11
+ VERSION = '0.4'
12
12
 
13
13
  @@options = {}
14
14
 
@@ -48,6 +48,5 @@ end
48
48
  require 'sparrow/server'
49
49
  require 'sparrow/queues/sqlite' rescue LoadError nil
50
50
  require 'sparrow/queues/memory'
51
- require 'sparrow/queues/disk'
52
51
  require 'sparrow/queue'
53
52
  require 'sparrow/runner'
@@ -7,12 +7,7 @@ module Sparrow
7
7
 
8
8
  class << self
9
9
  def get_queue(queue_name)
10
- @@queues[queue_name] ||= case Sparrow.options[:type]
11
- when 'memory': Sparrow::Queues::Memory.new(queue_name)
12
- when 'sqlite': Sparrow::Queues::Sqlite.new(queue_name)
13
- else
14
- Sparrow::Queues::Disk.new(queue_name)
15
- end
10
+ @@queues[queue_name] ||= Sparrow::Queues::Memory.new(queue_name)
16
11
  end
17
12
 
18
13
  def next_message(queue_name)
@@ -31,14 +26,17 @@ module Sparrow
31
26
  end
32
27
 
33
28
  def delete_all
29
+ @@queues.each {|name, q| q.clear! }
34
30
  @@queues = {}
35
- FileUtils.rm_rf base_dir
36
- FileUtils.mkdir_p base_dir
31
+ end
32
+
33
+ def shutdown!
34
+ @@queues.each {|name, q| q.shutdown! }
37
35
  end
38
36
 
39
37
  def get_stats(queue_name)
40
38
  stats = {
41
- :type => Sparrow.options[:type],
39
+ :type => 'memory',
42
40
  :total_bytes => (File.size?(Sparrow.base_dir) || 0),
43
41
  :queues => Dir.glob(File.join(Sparrow.base_dir, '*')).collect {|s| File.basename(s) }.join(','),
44
42
  :number_of_queues => queues.keys.length,
@@ -53,7 +51,6 @@ module Sparrow
53
51
  if queue_name
54
52
  queue = get_queue(queue_name)
55
53
  stats.merge!({
56
- :bytes => Dir.glob(File.join(Sparrow.base_dir, queue_name + '**')).inject(0){|a, b| a += (File.size?(b) || 0); a },
57
54
  :total_items => queue.count_push,
58
55
  :curr_items => queue.count
59
56
  })
@@ -13,26 +13,51 @@ module Sparrow
13
13
  self.queue_data = []
14
14
  self.count_pop = 0
15
15
  self.count_push = 0
16
+ recover!
16
17
  end
17
18
 
18
19
  def pop
19
20
  self.count_pop += 1
20
- queue_data.shift
21
+ self.queue_data.shift
21
22
  end
22
23
 
23
24
  def push(value)
24
25
  self.count_push += 1
25
- queue_data.push(value)
26
+ self.queue_data.push(value)
26
27
  end
27
28
 
28
- def clear
29
+ def clear!
29
30
  self.queue_data = []
31
+ self.sqlite.clear!
30
32
  end
31
33
 
32
34
  def count
33
35
  queue_data.length
34
36
  end
35
-
37
+
38
+ def to_disk!
39
+ copy = self.queue_data.dup
40
+ copy.each do |value|
41
+ self.sqlite.push(value)
42
+ end
43
+ self.queue_data = self.queue_data - copy
44
+ end
45
+
46
+ def shutdown!
47
+ self.to_disk!
48
+ end
49
+
50
+ def recover!
51
+ logger.debug "Recovering queue"
52
+ while msg = self.sqlite.pop
53
+ self.push(msg)
54
+ end
55
+ end
56
+
57
+ def sqlite
58
+ @sqlite ||= Sparrow::Queues::Sqlite.new(self.queue_name)
59
+ end
60
+
36
61
  end
37
62
  end
38
63
  end
@@ -35,16 +35,17 @@ module Sparrow
35
35
 
36
36
  def pop
37
37
  id, value = db.get_first_row("SELECT * FROM queues LIMIT 1;")
38
+ return unless id and value
38
39
  db.execute("DELETE FROM queues WHERE id = ?", id)
39
40
  self.count_pop += 1
40
41
  value
41
42
  end
42
43
 
43
44
  def count
44
- db.get_first_value("SELECT COUNT FROM queues")
45
+ db.get_first_value("SELECT count(*) FROM queues").to_i
45
46
  end
46
47
 
47
- def clear
48
+ def clear!
48
49
  db.execute("DELETE FROM queues")
49
50
  end
50
51
 
@@ -22,7 +22,7 @@ module Sparrow
22
22
 
23
23
  self.options.merge!({
24
24
  :base_dir => base_dir,
25
- :pid_dir => pid_dir,
25
+ :pid_dir => pid_dir,
26
26
  :log_path => log_path
27
27
  })
28
28
 
@@ -64,6 +64,9 @@ module Sparrow
64
64
  end
65
65
 
66
66
  def stop
67
+ puts
68
+ puts "Writing queues to disk..."
69
+ Sparrow::Queue.shutdown!
67
70
  puts "Stopping Eventmachine Server"
68
71
  EventMachine::stop
69
72
  end
@@ -72,7 +75,7 @@ module Sparrow
72
75
  OptionParser.new do |opts|
73
76
  opts.summary_width = 25
74
77
  opts.banner = "Sparrow (#{VERSION})\n\n",
75
- "Usage: sparrow [-b path] [-t type] [-h host] [-p port] [-P file]\n",
78
+ "Usage: sparrow [-b path] [-h host] [-p port] [-P file]\n",
76
79
  " [-d] [-k port] [-l file] [-e]\n",
77
80
  " sparrow --help\n",
78
81
  " sparrow --version\n"
@@ -84,10 +87,6 @@ module Sparrow
84
87
  options[:base_dir] = File.expand_path(v)
85
88
  end
86
89
 
87
- opts.on("-t", "--type QUEUE_TYPE", String, "Type of queue (disk/memory/sqlite).", "(default: #{options[:type]})") do |v|
88
- options[:type] = v
89
- end
90
-
91
90
  opts.separator ""; opts.separator "Network:"
92
91
 
93
92
  opts.on("-h", "--host HOST", String, "Specify host", "(default: #{options[:host]})") do |v|
@@ -154,7 +153,7 @@ module Sparrow
154
153
  puts f
155
154
  pid = IO.read(f).chomp.to_i
156
155
  FileUtils.rm f
157
- Process.kill(9, pid)
156
+ Process.kill(15, pid) # TERM
158
157
  puts "killed PID: #{pid}"
159
158
  rescue => e
160
159
  puts "Failed to kill! #{k}: #{e}"
metadata CHANGED
@@ -1,33 +1,57 @@
1
1
  --- !ruby/object:Gem::Specification
2
- rubygems_version: 0.9.4
3
- specification_version: 1
4
2
  name: sparrow
5
3
  version: !ruby/object:Gem::Version
6
- version: 0.3.1
7
- date: 2008-02-03 00:00:00 +00:00
8
- summary: Simple file based messagine queue using the memcache protocol
9
- require_paths:
10
- - lib
11
- email: info@eribium.org
12
- homepage: http://code.google.com/p/sparrow
13
- rubyforge_project: Sparrow
14
- description: "# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached. # That means you can use Sparrow with any memcached client library (Ruby or otherwise). # # Basic tests shows that Sparrow processes messages at a rate of 850-900 per second. # The load Sparrow can cope with increases exponentially as you add to the cluster. # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance. # # Sparrow comes with built in support for daemonization and clustering. # Also included are example libraries and clients. For example: # # require 'memcache' # m = MemCache.new('127.0.0.1:11212') # m['queue_name'] = '1' # Publish to queue # m['queue_name'] #=> 1 Pull next msg from queue # m['queue_name'] #=> nil # m.delete('queue_name) # Delete queue # # # or using the included client: # # class MyQueue < MQ3::Queue # def on_message # logger.info \"Received msg with args: #{args.inspect}\" # end # end # # MyQueue.servers = [ # MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1}) # ] # MyQueue.publish('test msg') # MyQueue.run # # Messages are deleted as soon as they're read and the order you add messages to the queue probably won't # be the same order when they're removed. # # Additional memcached commands that are supported are: # flush_all # Deletes all queues # version # quit # The memcached commands 'add', and 'replace' just call 'set'. # # Call sparrow with --help for usage options # # The daemonization won't work on Windows. # # Check out the code: # svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow # # Sparrow was inspired by Twitter's Starling"
15
- autorequire:
16
- default_executable:
17
- bindir: bin
18
- has_rdoc: true
19
- required_ruby_version: !ruby/object:Gem::Version::Requirement
20
- requirements:
21
- - - ">"
22
- - !ruby/object:Gem::Version
23
- version: 0.0.0
24
- version:
4
+ version: "0.4"
25
5
  platform: ruby
26
- signing_key:
27
- cert_chain:
28
- post_install_message:
29
6
  authors:
30
7
  - Alex MacCAw
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2008-08-02 00:00:00 +01:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: eventmachine
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: 0.10.0
24
+ version:
25
+ - !ruby/object:Gem::Dependency
26
+ name: sqlite3-ruby
27
+ type: :runtime
28
+ version_requirement:
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 1.2.2
34
+ version:
35
+ - !ruby/object:Gem::Dependency
36
+ name: hoe
37
+ type: :development
38
+ version_requirement:
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: 1.7.0
44
+ version:
45
+ description: "# Sparrow is a really fast lightweight queue written in Ruby that speaks memcached. # That means you can use Sparrow with any memcached client library (Ruby or otherwise). # # Basic tests shows that Sparrow processes messages at a rate of 850-900 per second. # The load Sparrow can cope with increases exponentially as you add to the cluster. # Sparrow also takes advantage of eventmachine, which uses a non-blocking io, offering great performance. # # Sparrow is a in-memory queue but will persist the data to disk when receiving a term signal. # # Sparrow comes with built in support for daemonization and clustering. # Also included are example libraries and clients. For example: # # require 'memcache' # m = MemCache.new('127.0.0.1:11212') # m['queue_name'] = '1' # Publish to queue # m['queue_name'] #=> 1 Pull next msg from queue # m['queue_name'] #=> nil # m.delete('queue_name) # Delete queue # # # or using the included client: # # class MyQueue < MQ3::Queue # def on_message # logger.info \"Received msg with args: #{args.inspect}\" # end # end # # MyQueue.servers = [ # MQ3::Protocols::Memcache.new({:host => '127.0.0.1', :port => 11212, :weight => 1}) # ] # MyQueue.publish('test msg') # MyQueue.run # # Messages are deleted as soon as they're read and the order you add messages to the queue probably won't # be the same order when they're removed. # # Additional memcached commands that are supported are: # flush_all # Deletes all queues # version # quit # The memcached commands 'add', and 'replace' just call 'set'. # # Call sparrow with --help for usage options # # The daemonization won't work on Windows. # # Check out the code: # svn checkout http://sparrow.googlecode.com/svn/trunk/ sparrow # # Sparrow was inspired by Twitter's Starling"
46
+ email: info@eribium.org
47
+ executables:
48
+ - sparrow
49
+ extensions: []
50
+
51
+ extra_rdoc_files:
52
+ - History.txt
53
+ - Manifest.txt
54
+ - README.txt
31
55
  files:
32
56
  - History.txt
33
57
  - Manifest.txt
@@ -36,43 +60,37 @@ files:
36
60
  - bin/sparrow
37
61
  - lib/sparrow.rb
38
62
  - lib/sparrow/queue.rb
39
- - lib/sparrow/queues/disk.rb
40
63
  - lib/sparrow/queues/memory.rb
41
64
  - lib/sparrow/queues/sqlite.rb
42
65
  - lib/sparrow/runner.rb
43
66
  - lib/sparrow/server.rb
44
67
  - lib/sparrow/utils.rb
45
- test_files: []
46
-
68
+ has_rdoc: true
69
+ homepage: http://code.google.com/p/sparrow
70
+ post_install_message:
47
71
  rdoc_options:
48
72
  - --main
49
73
  - README.txt
50
- extra_rdoc_files:
51
- - History.txt
52
- - Manifest.txt
53
- - README.txt
54
- executables:
55
- - sparrow
56
- extensions: []
57
-
74
+ require_paths:
75
+ - lib
76
+ required_ruby_version: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ">="
79
+ - !ruby/object:Gem::Version
80
+ version: "0"
81
+ version:
82
+ required_rubygems_version: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ version: "0"
87
+ version:
58
88
  requirements: []
59
89
 
60
- dependencies:
61
- - !ruby/object:Gem::Dependency
62
- name: eventmachine
63
- version_requirement:
64
- version_requirements: !ruby/object:Gem::Version::Requirement
65
- requirements:
66
- - - ">="
67
- - !ruby/object:Gem::Version
68
- version: 0.10.0
69
- version:
70
- - !ruby/object:Gem::Dependency
71
- name: hoe
72
- version_requirement:
73
- version_requirements: !ruby/object:Gem::Version::Requirement
74
- requirements:
75
- - - ">="
76
- - !ruby/object:Gem::Version
77
- version: 1.5.0
78
- version:
90
+ rubyforge_project: Sparrow
91
+ rubygems_version: 1.2.0
92
+ signing_key:
93
+ specification_version: 2
94
+ summary: Simple file based messagine queue using the memcache protocol
95
+ test_files: []
96
+
@@ -1,109 +0,0 @@
1
- require 'fileutils'
2
- module Sparrow
3
- module Queues
4
- class Disk
5
- include Sparrow::Miscel
6
-
7
- TRX_CMD_PUSH = "\000".freeze
8
- TRX_CMD_POP = "\001".freeze
9
-
10
- TRX_PUSH = "\000%s%s".freeze
11
- TRX_POP = "\001".freeze
12
-
13
- attr_accessor :queue_name
14
- attr_accessor :trxr
15
- attr_accessor :trxw
16
- attr_accessor :count_pop
17
- attr_accessor :count_push
18
-
19
- def initialize(queue_name)
20
- self.queue_name = queue_name
21
- self.count_pop = 0
22
- self.count_push = 0
23
- open_queue
24
- end
25
-
26
- def push(value)
27
- value = value.to_s
28
- size = [value.size].pack("I")
29
- data = sprintf(TRX_PUSH, size, value)
30
- trxw.seek(0, IO::SEEK_END)
31
- trxw.write data
32
- # trxw.fsync
33
- rotate_queue if trxw.pos > max_log_size
34
- self.count_push += 1
35
- value
36
- end
37
-
38
- def pop
39
- while !trxr.eof?
40
- s_pos = trxr.pos
41
- cmd = trxr.read(1)
42
- if cmd != TRX_CMD_POP and cmd != TRX_CMD_PUSH
43
- logger.fatal 'Corrupt queue'
44
- return
45
- end
46
- raw_size = trxr.read(4)
47
- size = raw_size.unpack("I").first
48
- value = trxr.read(size)
49
- next if cmd == TRX_CMD_POP
50
- e_pos = trxr.pos
51
- trxr.seek(s_pos, IO::SEEK_SET)
52
- trxr.write(TRX_POP)
53
- # trxr.fsync
54
- trxr.pos = e_pos
55
- next unless value
56
- self.count_pop += 1
57
- return value
58
- end
59
-
60
- if trxr.path == queue_path
61
- File.truncate(trxr.path, 0)
62
- else
63
- FileUtils.rm_rf trxr.path
64
- end
65
- open_reader
66
- nil
67
- end
68
-
69
- def clear
70
- dirs = Dir.glob(queue_path) | Dir.glob(queue_path + '.*')
71
- FileUtils.rm_rf(dirs) unless dirs.empty?
72
- end
73
-
74
- def count
75
- self.count_push - self.count_pop
76
- end
77
-
78
- private
79
-
80
- def queue_path
81
- File.join(base_dir, queue_name)
82
- end
83
-
84
- def rotate_queue
85
- File.rename(queue_path, File.join(base_dir, "#{queue_name}.#{Time.now.to_i}"))
86
- open_writer
87
- end
88
-
89
- def open_writer
90
- self.trxw = File.open(queue_path, 'a+')
91
- end
92
-
93
- def open_reader
94
- old_queue = Dir.glob(queue_path + '.*').first
95
- self.trxr = File.open(old_queue||queue_path, 'r+')
96
- end
97
-
98
- def open_queue
99
- open_writer
100
- open_reader
101
- end
102
-
103
- def max_log_size
104
- @max_log_size ||= (options[:log_size] || 16) * (1024**2) # 16mb
105
- end
106
-
107
- end
108
- end
109
- end