super-poller 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +34 -0
- data/bin/queue +11 -0
- data/bin/scripts/queue_choose +17 -0
- data/bin/scripts/queue_copy +4 -0
- data/bin/scripts/queue_flush +3 -0
- data/bin/scripts/queue_grep +51 -0
- data/bin/scripts/queue_interactive_move +13 -0
- data/bin/scripts/queue_move +4 -0
- data/bin/scripts/queue_scan +3 -0
- data/lib/super_poller/aggregating_error_logger.rb +33 -0
- data/lib/super_poller/buffered_handler.rb +28 -0
- data/lib/super_poller/error_reporter.rb +12 -0
- data/lib/super_poller/handler.rb +22 -0
- data/lib/super_poller/none_blocking_poller.rb +13 -0
- data/lib/super_poller/poller.rb +24 -0
- data/lib/super_poller/queue_itterator.rb +30 -0
- data/lib/super_poller/queue_url.rb +18 -0
- data/lib/super_poller/router.rb +29 -0
- data/lib/super_poller/starling_queue.rb +20 -0
- data/lib/super_poller/test_case.rb +13 -0
- data/lib/super_poller.rb +12 -0
- data/super_poller.gemspec +26 -0
- data/test/aggregating_error_logger_test.rb +65 -0
- data/test/buffered_handler_test.rb +45 -0
- data/test/error_reporter_test.rb +28 -0
- data/test/handler_test.rb +28 -0
- data/test/poller_test.rb +27 -0
- data/test/router_test.rb +32 -0
- data/test/starling_queue_test.rb +32 -0
- data/test/test_helper.rb +7 -0
- metadata +84 -0
data/Rakefile
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
require "rubygems"
|
2
|
+
require "rake/gempackagetask"
|
3
|
+
|
4
|
+
task :default => :test
|
5
|
+
|
6
|
+
require "rake/testtask"
|
7
|
+
Rake::TestTask.new do |t|
|
8
|
+
t.libs << "test"
|
9
|
+
t.test_files = FileList["test/**/*_test.rb"]
|
10
|
+
t.verbose = true
|
11
|
+
end
|
12
|
+
|
13
|
+
spec = Gem::Specification.new do |s|
|
14
|
+
s.name = "super-poller"
|
15
|
+
s.version = "0.1.0"
|
16
|
+
s.summary = "Tools for dealing with queues."
|
17
|
+
s.description = "Small modular library for dealing with queues and pollers."
|
18
|
+
s.author = "Tom Lea"
|
19
|
+
s.email = "contrib@tomlea.co.uk"
|
20
|
+
s.homepage = "http://tomlea.co.uk"
|
21
|
+
s.has_rdoc = true
|
22
|
+
|
23
|
+
s.files = %w(Rakefile super_poller.gemspec) + Dir.glob("{bin,test,lib}/**/*")
|
24
|
+
s.executables = FileList["bin/*"].map { |f| File.basename(f) }
|
25
|
+
|
26
|
+
s.require_paths = ["lib"]
|
27
|
+
s.rubyforge_project = "super-poller"
|
28
|
+
end
|
29
|
+
|
30
|
+
Rake::GemPackageTask.new(spec) do |pkg|
|
31
|
+
pkg.gem_spec = spec
|
32
|
+
file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
|
33
|
+
File.open(file, "w") {|f| f << spec.to_ruby }
|
34
|
+
end
|
data/bin/queue
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
script = ARGV.shift
|
3
|
+
|
4
|
+
if File.exist?("#{File.dirname(__FILE__)}/scripts/queue_#{script}")
|
5
|
+
load "#{File.dirname(__FILE__)}/scripts/queue_#{script}"
|
6
|
+
else
|
7
|
+
STDERR.puts "Available Commands:"
|
8
|
+
Dir.glob("#{File.dirname(__FILE__)}/scripts/queue_*") do |f|
|
9
|
+
STDERR.puts " " + File.basename(f).gsub(/^queue_/, "")
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require "super_poller"
|
2
|
+
queue = SuperPoller::QueueUrl.parse(ARGV.shift).to_queue
|
3
|
+
|
4
|
+
memo = {:memo => Time.now.to_f}
|
5
|
+
queue.push(memo)
|
6
|
+
|
7
|
+
SuperPoller::NoneBlockingPoller.new(queue, lambda{|msg|
|
8
|
+
skip = false
|
9
|
+
begin
|
10
|
+
exit if msg == memo
|
11
|
+
p msg
|
12
|
+
print "Kill? > "
|
13
|
+
skip = gets =~ /^[Yy]/
|
14
|
+
ensure
|
15
|
+
queue.push(msg) unless skip or msg == memo
|
16
|
+
end
|
17
|
+
}).start!
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require "super_poller"
|
2
|
+
require 'optparse'
|
3
|
+
|
4
|
+
options = {}
|
5
|
+
opts = OptionParser.new do |opts|
|
6
|
+
opts.banner = "Usage queue grep QUEUE_URL [options]"
|
7
|
+
opts.on("-f STRING", "--ruby-filter STRING", "Set ruby filter") do |v|
|
8
|
+
options[:ruby_filter] = v
|
9
|
+
end
|
10
|
+
|
11
|
+
opts.on("-a STRING", "--ruby-action STRING", "Set ruby action") do |v|
|
12
|
+
options[:ruby_action] = v
|
13
|
+
end
|
14
|
+
|
15
|
+
opts.on("-d", "--delete", "Delete all matches") do |v|
|
16
|
+
options[:ruby_action] = ":delete"
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on("-c QUEUE", "--copy QUEUE", "Copy all matches") do |v|
|
20
|
+
options[:ruby_action] = "SuperPoller::QueueUrl.parse(#{v.to_s.inspect}).to_queue.push(msg)"
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on("-m QUEUE", "--move QUEUE", "Move all matches") do |v|
|
24
|
+
options[:ruby_action] = "SuperPoller::QueueUrl.parse(#{v.to_s.inspect}).to_queue.push(msg); :delete"
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on("-t TYPE", "--message-type TYPE", "Select messages of a given type") do |v|
|
28
|
+
options[:ruby_filter] = "#{v.to_s.inspect} == msg[:name].to_s rescue false"
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
32
|
+
puts opts
|
33
|
+
exit 1
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
opts.parse!
|
38
|
+
|
39
|
+
unless queue_url = ARGV.shift
|
40
|
+
puts opts
|
41
|
+
exit 1
|
42
|
+
end
|
43
|
+
|
44
|
+
queue = SuperPoller::QueueUrl.parse(queue_url).to_queue
|
45
|
+
|
46
|
+
eval "def matcher_matches?(msg); #{ options[:ruby_filter] || "true" }; end"
|
47
|
+
eval "def action(msg); #{ options[:ruby_action] || "p msg" }; end"
|
48
|
+
|
49
|
+
SuperPoller::QueueItterator.new(queue).each do |msg|
|
50
|
+
action(msg) if matcher_matches? msg
|
51
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require "super_poller"
|
2
|
+
queue_one = SuperPoller::QueueUrl.parse(ARGV.shift).to_queue
|
3
|
+
queue_two = SuperPoller::QueueUrl.parse(ARGV.shift).to_queue
|
4
|
+
|
5
|
+
SuperPoller::QueueItterator.new(queue_one).each do |msg|
|
6
|
+
p msg
|
7
|
+
print "Move to #{queue_two}? > "
|
8
|
+
|
9
|
+
if gets =~ /^[Yy]/
|
10
|
+
queue_two.push(msg)
|
11
|
+
:delete
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
class SuperPoller::AggregatingErrorLogger
|
2
|
+
def initialize(stats_file, queue)
|
3
|
+
@stats_file, @queue = stats_file, queue
|
4
|
+
end
|
5
|
+
|
6
|
+
def call(error, failed_message)
|
7
|
+
update_error_queue(error, failed_message)
|
8
|
+
update_error_stats(error, failed_message[:name] || :unknown)
|
9
|
+
end
|
10
|
+
|
11
|
+
protected
|
12
|
+
def update_error_queue(error, failed_message)
|
13
|
+
error_class_name = error.class.name.to_sym
|
14
|
+
error_description = {:class => error_class_name, :message => error.message}
|
15
|
+
@queue.push(failed_message.merge(:error => error_description))
|
16
|
+
end
|
17
|
+
|
18
|
+
def update_error_stats(error, message_name)
|
19
|
+
stats = load_stats
|
20
|
+
error_class_name = error.class.name.to_sym
|
21
|
+
stats_for_name = (stats[message_name] ||= {})
|
22
|
+
stats_for_name[error_class_name] ||= 0
|
23
|
+
stats_for_name[error_class_name] += 1
|
24
|
+
|
25
|
+
File.open(@stats_file, "w") do |file|
|
26
|
+
file << YAML.dump(stats)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def load_stats
|
31
|
+
File.exists?(@stats_file) && YAML.load(File.read(@stats_file)) || {}
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
class SuperPoller::BufferedHandler < SuperPoller::Handler
|
2
|
+
class << self
|
3
|
+
def buffer_size(size)
|
4
|
+
@max_buffer_size = size
|
5
|
+
end
|
6
|
+
|
7
|
+
def max_buffer_size
|
8
|
+
@max_buffer_size || []
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@buffer = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def call(msg)
|
17
|
+
@buffer.push msg
|
18
|
+
if @buffer.size >= self.class.max_buffer_size
|
19
|
+
handle_batch @buffer
|
20
|
+
@buffer = []
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def handle_batch(batch)
|
25
|
+
raise NotImplementedError, "You must define a batch handler."
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
class SuperPoller::ErrorReporter
|
2
|
+
def initialize(message_handler, error_handler = nil, &block)
|
3
|
+
@message_handler = message_handler
|
4
|
+
@error_handler = error_handler || block
|
5
|
+
end
|
6
|
+
|
7
|
+
def call(*args)
|
8
|
+
@message_handler.call(*args)
|
9
|
+
rescue Object => e
|
10
|
+
@error_handler.call(e, *args)
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class SuperPoller::Handler
|
2
|
+
autoload :TestCase, "super_poller/test_case"
|
3
|
+
|
4
|
+
class << self
|
5
|
+
def handles(*new_message_names)
|
6
|
+
@message_names = (message_names + new_message_names).uniq
|
7
|
+
end
|
8
|
+
|
9
|
+
def message_names
|
10
|
+
@message_names || []
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def can_handle?(message)
|
15
|
+
self.class.message_names.include? message[:name].to_sym
|
16
|
+
end
|
17
|
+
|
18
|
+
def call(message)
|
19
|
+
raise NotImplementedError, "You must define a call handler."
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
class SuperPoller::Poller
|
2
|
+
def initialize(queue, message_handler)
|
3
|
+
@message_handler, @queue = message_handler, queue
|
4
|
+
end
|
5
|
+
|
6
|
+
def poll
|
7
|
+
@message_handler.call(get_message)
|
8
|
+
end
|
9
|
+
|
10
|
+
def start!
|
11
|
+
poll while true
|
12
|
+
end
|
13
|
+
|
14
|
+
protected
|
15
|
+
def get_message
|
16
|
+
@queue.pop
|
17
|
+
rescue Interrupt
|
18
|
+
raise
|
19
|
+
rescue Object => e
|
20
|
+
STDERR.puts "Error while fetching from the queue: #{e.class}: #{e.message}"
|
21
|
+
sleep 10
|
22
|
+
retry
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class SuperPoller::QueueItterator
|
2
|
+
def initialize(queue)
|
3
|
+
@queue = queue
|
4
|
+
end
|
5
|
+
|
6
|
+
def each(&block)
|
7
|
+
@memo = {:memo => Time.now.to_f}
|
8
|
+
@queue.push @memo
|
9
|
+
while @memo
|
10
|
+
begin
|
11
|
+
break unless msg = @queue.fetch
|
12
|
+
if msg == @memo
|
13
|
+
msg = @memo = nil
|
14
|
+
else
|
15
|
+
msg = nil if :delete == block.call(msg)
|
16
|
+
end
|
17
|
+
ensure
|
18
|
+
@queue.push msg if msg
|
19
|
+
end
|
20
|
+
end
|
21
|
+
ensure
|
22
|
+
destroy_the_memo
|
23
|
+
end
|
24
|
+
|
25
|
+
def destroy_the_memo
|
26
|
+
while @memo and msg = @queue.fetch and msg != @memo
|
27
|
+
@queue.push(msg)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require "uri"
|
2
|
+
|
3
|
+
class SuperPoller::QueueUrl < URI::Generic
|
4
|
+
def self.parse(url)
|
5
|
+
url = "starling://localhost:22122/#{url}" if url =~ /^[a-zA-Z0-9_-]+$/
|
6
|
+
new(*URI.parse(url).send(:component_ary))
|
7
|
+
end
|
8
|
+
|
9
|
+
def to_queue
|
10
|
+
raise URI::InvalidURIError unless respond_to? "to_#{scheme}_queue"
|
11
|
+
send("to_#{scheme}_queue")
|
12
|
+
end
|
13
|
+
|
14
|
+
protected
|
15
|
+
def to_starling_queue
|
16
|
+
SuperPoller::StarlingQueue.new(path, "#{host}:#{port}")
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
class SuperPoller::Router
|
2
|
+
RoutingError = Class.new(Exception)
|
3
|
+
|
4
|
+
def initialize()
|
5
|
+
@handlers = []
|
6
|
+
end
|
7
|
+
|
8
|
+
def add_handler(handler)
|
9
|
+
@handlers.push handler
|
10
|
+
@handlers.uniq!
|
11
|
+
self
|
12
|
+
end
|
13
|
+
|
14
|
+
alias << add_handler
|
15
|
+
|
16
|
+
def call(message)
|
17
|
+
handler = best_handler_for_message(message)
|
18
|
+
handler.call(message[:body])
|
19
|
+
end
|
20
|
+
|
21
|
+
protected
|
22
|
+
|
23
|
+
def best_handler_for_message(messsage)
|
24
|
+
@handlers.each do |handler|
|
25
|
+
return handler if handler.can_handle? messsage
|
26
|
+
end
|
27
|
+
raise RoutingError, "No handler found"
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require "starling"
|
2
|
+
|
3
|
+
class SuperPoller::StarlingQueue
|
4
|
+
def initialize(queue_name, *args)
|
5
|
+
@queue_name = queue_name.to_s
|
6
|
+
@queue = Starling.new(*args)
|
7
|
+
end
|
8
|
+
|
9
|
+
def pop
|
10
|
+
@queue.get(@queue_name)
|
11
|
+
end
|
12
|
+
|
13
|
+
def push(v)
|
14
|
+
@queue.set(@queue_name, v)
|
15
|
+
end
|
16
|
+
|
17
|
+
def fetch
|
18
|
+
@queue.fetch(@queue_name)
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
class SuperPoller::Handler::TestCase < (defined?(ActiveSupport::TestCase) ? ActiveSupport::TestCase : Test::Unit::TestCase)
|
2
|
+
|
3
|
+
def handler
|
4
|
+
@handler ||= self.class.name.gsub(/Test$/, "").constantize.new
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.should_handle(name)
|
8
|
+
should "handle a #{name.inspect} message" do
|
9
|
+
assert handler.can_handle?(:name => name)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
data/lib/super_poller.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
module SuperPoller
|
2
|
+
autoload :Handler, "super_poller/handler"
|
3
|
+
autoload :BufferedHandler, "super_poller/buffered_handler"
|
4
|
+
autoload :Router, "super_poller/router"
|
5
|
+
autoload :ErrorReporter, "super_poller/error_reporter"
|
6
|
+
autoload :StarlingQueue, "super_poller/starling_queue"
|
7
|
+
autoload :Poller, "super_poller/poller"
|
8
|
+
autoload :AggregatingErrorLogger, "super_poller/aggregating_error_logger"
|
9
|
+
autoload :NoneBlockingPoller, "super_poller/none_blocking_poller"
|
10
|
+
autoload :QueueUrl, "super_poller/queue_url"
|
11
|
+
autoload :QueueItterator, "super_poller/queue_itterator"
|
12
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{super_poller}
|
5
|
+
s.version = "0.1.0"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Your name"]
|
9
|
+
s.date = %q{2009-09-28}
|
10
|
+
s.email = %q{you@example.com}
|
11
|
+
s.files = ["test/aggregating_error_logger_test.rb", "test/buffered_handler_test.rb", "test/error_reporter_test.rb", "test/handler_test.rb", "test/poller_test.rb", "test/router_test.rb", "test/starling_queue_test.rb", "test/test_helper.rb", "lib/super_poller", "lib/super_poller/aggregating_error_logger.rb", "lib/super_poller/buffered_handler.rb", "lib/super_poller/error_reporter.rb", "lib/super_poller/handler.rb", "lib/super_poller/poller.rb", "lib/super_poller/router.rb", "lib/super_poller/starling_queue.rb", "lib/super_poller/test_case.rb", "lib/super_poller.rb"]
|
12
|
+
s.homepage = %q{http://example.com}
|
13
|
+
s.require_paths = ["lib"]
|
14
|
+
s.rubygems_version = %q{1.3.5}
|
15
|
+
s.summary = %q{What this thing does}
|
16
|
+
|
17
|
+
if s.respond_to? :specification_version then
|
18
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
19
|
+
s.specification_version = 3
|
20
|
+
|
21
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
22
|
+
else
|
23
|
+
end
|
24
|
+
else
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper")
|
2
|
+
|
3
|
+
|
4
|
+
class AggregatingErrorLoggerTest < Test::Unit::TestCase
|
5
|
+
include SuperPoller
|
6
|
+
attr_reader :queued_message
|
7
|
+
|
8
|
+
def log_contents
|
9
|
+
YAML.load(File.open(@path))
|
10
|
+
end
|
11
|
+
|
12
|
+
context "With an error logger" do
|
13
|
+
setup do
|
14
|
+
@log_file = Tempfile.new("AggregatingErrorLoggerTest")
|
15
|
+
@path = @log_file.path
|
16
|
+
@queue = Queue.new
|
17
|
+
@logger = AggregatingErrorLogger.new(@path, @queue)
|
18
|
+
end
|
19
|
+
|
20
|
+
context "with a exception and message passed in, it" do
|
21
|
+
setup do
|
22
|
+
@exception = Exception.new("The house is on fire")
|
23
|
+
@message = {:name => :foo, :body => "hello"}
|
24
|
+
@logger.call(@exception, @message)
|
25
|
+
@queued_message = @queue.pop(nonblocking = true)
|
26
|
+
end
|
27
|
+
|
28
|
+
should "add errors to the error queue" do
|
29
|
+
assert_equal :foo, queued_message[:name]
|
30
|
+
assert_equal "hello", queued_message[:body]
|
31
|
+
end
|
32
|
+
|
33
|
+
should "add error info to the queued message" do
|
34
|
+
assert_equal( {:class => :Exception, :message => "The house is on fire"}, queued_message[:error])
|
35
|
+
end
|
36
|
+
|
37
|
+
should "log errors to the log file" do
|
38
|
+
assert_equal( {:foo => {:Exception => 1} }, log_contents)
|
39
|
+
end
|
40
|
+
|
41
|
+
should "aggregate duplicate errors" do
|
42
|
+
@logger.call(@exception, @message)
|
43
|
+
assert_equal( {:foo => {:Exception => 2} }, log_contents)
|
44
|
+
end
|
45
|
+
|
46
|
+
should "separate different exceptions" do
|
47
|
+
@logger.call(ArgumentError.new(""), @message)
|
48
|
+
assert_equal( {:foo => {:Exception => 1, :ArgumentError => 1} }, log_contents)
|
49
|
+
end
|
50
|
+
|
51
|
+
should "separate errors for different message names" do
|
52
|
+
@logger.call(@exception, {:name => :bar, :body => "hello"})
|
53
|
+
assert_equal( {:foo => {:Exception => 1}, :bar => {:Exception => 1} }, log_contents)
|
54
|
+
end
|
55
|
+
|
56
|
+
should "reset the counts to 0 if we delete the log file" do
|
57
|
+
assert_equal( {:foo => {:Exception => 1} }, log_contents)
|
58
|
+
File.delete(@path)
|
59
|
+
@logger.call(@exception, {:name => :bar, :body => "hello"})
|
60
|
+
assert_equal( {:bar => {:Exception => 1} }, log_contents)
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper")
|
2
|
+
|
3
|
+
class BufferedHandlerTest < Test::Unit::TestCase
|
4
|
+
include SuperPoller
|
5
|
+
def self.should_submit_batches_for_messages(args = {})
|
6
|
+
context "that receives #{args[:messages]} messages" do
|
7
|
+
setup{ args[:messages].times{@handler.call(:your_mum)} }
|
8
|
+
|
9
|
+
before_should "handle #{args[:batches]} batches" do
|
10
|
+
@handler.expects(:handle_batch).with([:your_mum] * args[:batch_size]).times(args[:batches])
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
context "With a buffered handler with a buffer size of 10" do
|
16
|
+
setup{ @handler = Class.new(BufferedHandler){ buffer_size 10 }.new }
|
17
|
+
|
18
|
+
should_submit_batches_for_messages :batch_size => 10, :messages => 0, :batches => 0
|
19
|
+
|
20
|
+
should_submit_batches_for_messages :batch_size => 10, :messages => 9, :batches => 0
|
21
|
+
should_submit_batches_for_messages :batch_size => 10, :messages => 10, :batches => 1
|
22
|
+
should_submit_batches_for_messages :batch_size => 10, :messages => 11, :batches => 1
|
23
|
+
|
24
|
+
should_submit_batches_for_messages :batch_size => 10, :messages => 19, :batches => 1
|
25
|
+
should_submit_batches_for_messages :batch_size => 10, :messages => 20, :batches => 2
|
26
|
+
should_submit_batches_for_messages :batch_size => 10, :messages => 21, :batches => 2
|
27
|
+
end
|
28
|
+
|
29
|
+
context "With a buffered handler with a buffer size of 8" do
|
30
|
+
setup{ @handler = Class.new(BufferedHandler){ buffer_size 8 }.new }
|
31
|
+
|
32
|
+
should_submit_batches_for_messages :batch_size => 8, :messages => 7, :batches => 0
|
33
|
+
should_submit_batches_for_messages :batch_size => 8, :messages => 8, :batches => 1
|
34
|
+
|
35
|
+
should_submit_batches_for_messages :batch_size => 8, :messages => 16, :batches => 2
|
36
|
+
end
|
37
|
+
|
38
|
+
context "With a buffered handler with a buffer size of 0" do
|
39
|
+
setup{ @handler = Class.new(BufferedHandler){ buffer_size 0 }.new }
|
40
|
+
|
41
|
+
should_submit_batches_for_messages :batch_size => 1, :messages => 0, :batches => 0
|
42
|
+
should_submit_batches_for_messages :batch_size => 1, :messages => 1, :batches => 1
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper")
|
2
|
+
|
3
|
+
class ErrorReporterTest < Test::Unit::TestCase
|
4
|
+
include SuperPoller
|
5
|
+
|
6
|
+
should "catch errors and report them to the provided error handler" do
|
7
|
+
has_caught = false
|
8
|
+
raiser = proc{|msg| raise "Error doing a #{msg}" }
|
9
|
+
handler = proc{|e, message| has_caught = (e.message == "Error doing a Hello" and message == "Hello") }
|
10
|
+
|
11
|
+
ErrorReporter.new(raiser, handler).call("Hello")
|
12
|
+
|
13
|
+
assert has_caught
|
14
|
+
end
|
15
|
+
|
16
|
+
should "catch errors and report them to the provided error handling block" do
|
17
|
+
has_caught = false
|
18
|
+
raiser = proc{|msg| raise "Error doing a #{msg}" }
|
19
|
+
|
20
|
+
ErrorReporter.new(raiser){|e, message| has_caught = (e.message == "Error doing a Hello" and message == "Hello") }.call("Hello")
|
21
|
+
|
22
|
+
assert has_caught
|
23
|
+
end
|
24
|
+
|
25
|
+
should "not catch errors if they don't happen" do
|
26
|
+
ErrorReporter.new(stub("handler", :call)){|*args| flunk "Should not have tried to raise!" }.call("Hello")
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper")
|
2
|
+
|
3
|
+
class HandlerTest < Test::Unit::TestCase
|
4
|
+
include SuperPoller
|
5
|
+
|
6
|
+
context "With handlers for :foo and :bar messages" do
|
7
|
+
setup do
|
8
|
+
@foo_handler = Class.new(Handler){ handles :foo }.new
|
9
|
+
@bar_handler = Class.new(Handler){ handles :bar }.new
|
10
|
+
end
|
11
|
+
|
12
|
+
should "only foo_handler should handle a message named :foo" do
|
13
|
+
assert @foo_handler.can_handle?({:name => :foo})
|
14
|
+
assert !@bar_handler.can_handle?({:name => :foo})
|
15
|
+
end
|
16
|
+
|
17
|
+
should "only bar_handler should handle a message named :bar" do
|
18
|
+
assert !@foo_handler.can_handle?({:name => :bar})
|
19
|
+
assert @bar_handler.can_handle?({:name => :bar})
|
20
|
+
end
|
21
|
+
|
22
|
+
should "accept strings or symbols as names" do
|
23
|
+
assert @foo_handler.can_handle?({:name => "foo"})
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
data/test/poller_test.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper")
|
2
|
+
|
3
|
+
class PollerTest < Test::Unit::TestCase
|
4
|
+
include SuperPoller
|
5
|
+
|
6
|
+
def setup
|
7
|
+
@queue = Queue.new
|
8
|
+
@message_handler = stub("message_handler")
|
9
|
+
@poller = Poller.new(@queue, @message_handler)
|
10
|
+
end
|
11
|
+
|
12
|
+
should "pull things from the queue to the message_handler" do
|
13
|
+
@queue.push "Oh Hi!"
|
14
|
+
@message_handler.expects(:call).with("Oh Hi!").once
|
15
|
+
@poller.poll
|
16
|
+
end
|
17
|
+
|
18
|
+
should "pull more than one thing from the queue to the message_handler" do
|
19
|
+
@queue.push "First"
|
20
|
+
@queue.push "Second"
|
21
|
+
|
22
|
+
@message_handler.expects(:call).with("First").once
|
23
|
+
@message_handler.expects(:call).with("Second").once
|
24
|
+
|
25
|
+
2.times{ @poller.poll }
|
26
|
+
end
|
27
|
+
end
|
data/test/router_test.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper")
|
2
|
+
|
3
|
+
class RouterTest < Test::Unit::TestCase
|
4
|
+
include SuperPoller
|
5
|
+
|
6
|
+
context "With a router that has two handlers" do
|
7
|
+
|
8
|
+
setup do
|
9
|
+
@router = Router.new
|
10
|
+
@foo_handler = Class.new(Handler){ handles :foo }.new
|
11
|
+
@bar_handler = Class.new(Handler){ handles :bar }.new
|
12
|
+
@router << @foo_handler << @bar_handler
|
13
|
+
end
|
14
|
+
|
15
|
+
should "route foo messages to the correct handler" do
|
16
|
+
@foo_handler.expects(:call).with("test body").once
|
17
|
+
@bar_handler.expects(:call).never
|
18
|
+
@router.call({:name => :foo, :body => "test body"})
|
19
|
+
end
|
20
|
+
|
21
|
+
should "route bar messages to the correct handler" do
|
22
|
+
@foo_handler.expects(:call).never
|
23
|
+
@bar_handler.expects(:call).with("test body").once
|
24
|
+
@router.call({:name => :bar, :body => "test body"})
|
25
|
+
end
|
26
|
+
|
27
|
+
should "raise RoutingError if handler not found" do
|
28
|
+
assert_raise(Router::RoutingError){ @router.call(:name => :your_mum) }
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper")
|
2
|
+
|
3
|
+
class StarlingQueueTest < Test::Unit::TestCase
|
4
|
+
include SuperPoller
|
5
|
+
|
6
|
+
STARLING_QUEUE_PATH = '/tmp/reevoo_superpoller_starling'
|
7
|
+
STARLING_PID_PATH = '/tmp/reevoo_superpoller_starling.pid'
|
8
|
+
STARLING_PORT = '13891'
|
9
|
+
|
10
|
+
context "with a running starling" do
|
11
|
+
setup do
|
12
|
+
FileUtils.rm_r STARLING_QUEUE_PATH if File.exists? STARLING_QUEUE_PATH
|
13
|
+
FileUtils.rm_r STARLING_PID_PATH if File.exists? STARLING_PID_PATH
|
14
|
+
system("starling -p #{STARLING_PORT} -P #{STARLING_PID_PATH} -q #{STARLING_QUEUE_PATH} -d 2>&1")
|
15
|
+
sleep 1
|
16
|
+
end
|
17
|
+
|
18
|
+
teardown do
|
19
|
+
system("kill -9 `cat #{STARLING_PID_PATH}`")
|
20
|
+
end
|
21
|
+
|
22
|
+
should "write to and read from a queue in the correct order" do
|
23
|
+
queue = StarlingQueue.new("TEST_QUEUE", ["localhost:#{STARLING_PORT}"])
|
24
|
+
queue.push("first")
|
25
|
+
queue.push("second")
|
26
|
+
assert_equal "first", queue.pop
|
27
|
+
assert_equal "second", queue.pop
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
data/test/test_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: super-poller
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tom Lea
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-10-09 00:00:00 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: Small modular library for dealing with queues and pollers.
|
17
|
+
email: contrib@tomlea.co.uk
|
18
|
+
executables:
|
19
|
+
- queue
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- Rakefile
|
26
|
+
- super_poller.gemspec
|
27
|
+
- bin/queue
|
28
|
+
- bin/scripts/queue_choose
|
29
|
+
- bin/scripts/queue_copy
|
30
|
+
- bin/scripts/queue_flush
|
31
|
+
- bin/scripts/queue_grep
|
32
|
+
- bin/scripts/queue_interactive_move
|
33
|
+
- bin/scripts/queue_move
|
34
|
+
- bin/scripts/queue_scan
|
35
|
+
- test/aggregating_error_logger_test.rb
|
36
|
+
- test/buffered_handler_test.rb
|
37
|
+
- test/error_reporter_test.rb
|
38
|
+
- test/handler_test.rb
|
39
|
+
- test/poller_test.rb
|
40
|
+
- test/router_test.rb
|
41
|
+
- test/starling_queue_test.rb
|
42
|
+
- test/test_helper.rb
|
43
|
+
- lib/super_poller/aggregating_error_logger.rb
|
44
|
+
- lib/super_poller/buffered_handler.rb
|
45
|
+
- lib/super_poller/error_reporter.rb
|
46
|
+
- lib/super_poller/handler.rb
|
47
|
+
- lib/super_poller/none_blocking_poller.rb
|
48
|
+
- lib/super_poller/poller.rb
|
49
|
+
- lib/super_poller/queue_itterator.rb
|
50
|
+
- lib/super_poller/queue_url.rb
|
51
|
+
- lib/super_poller/router.rb
|
52
|
+
- lib/super_poller/starling_queue.rb
|
53
|
+
- lib/super_poller/test_case.rb
|
54
|
+
- lib/super_poller.rb
|
55
|
+
has_rdoc: true
|
56
|
+
homepage: http://tomlea.co.uk
|
57
|
+
licenses: []
|
58
|
+
|
59
|
+
post_install_message:
|
60
|
+
rdoc_options: []
|
61
|
+
|
62
|
+
require_paths:
|
63
|
+
- lib
|
64
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: "0"
|
69
|
+
version:
|
70
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: "0"
|
75
|
+
version:
|
76
|
+
requirements: []
|
77
|
+
|
78
|
+
rubyforge_project: super-poller
|
79
|
+
rubygems_version: 1.3.5
|
80
|
+
signing_key:
|
81
|
+
specification_version: 3
|
82
|
+
summary: Tools for dealing with queues.
|
83
|
+
test_files: []
|
84
|
+
|