super-poller 0.1.1 → 0.1.2

data/bin/scripts/queue_choose

@@ -0,0 +1,17 @@
+require "super_poller"
+queue = SuperPoller::QueueUrl.parse(ARGV.shift).to_queue
+
+memo = {:memo => Time.now.to_f}
+queue.push(memo)
+
+SuperPoller::NoneBlockingPoller.new(queue, lambda{|msg|
+  skip = false
+  begin
+    exit if msg == memo
+    p msg
+    print "Kill? > "
+    skip = gets =~ /^[Yy]/
+  ensure
+    queue.push(msg) unless skip or msg == memo
+  end
+}).start!

data/bin/scripts/queue_copy

@@ -0,0 +1,4 @@
+require "super_poller"
+in_queue, out_queue = [nil,nil].map{|url| SuperPoller::QueueUrl.parse(ARGV.shift).to_queue }
+
+SuperPoller::QueueItterator.new(in_queue).each(&out_queue.method(:push))

data/bin/scripts/queue_flush

@@ -0,0 +1,3 @@
+require "super_poller"
+queue = SuperPoller::QueueUrl.parse(ARGV.shift).to_queue
+SuperPoller::NoneBlockingPoller.new(queue, lambda{|msg| }).start!

data/bin/scripts/queue_grep

@@ -0,0 +1,51 @@
+require "super_poller"
+require 'optparse'
+
+options = {}
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage queue grep QUEUE_URL [options]"
+  opts.on("-f STRING", "--ruby-filter STRING", "Set ruby filter") do |v|
+    options[:ruby_filter] = v
+  end
+
+  opts.on("-a STRING", "--ruby-action STRING", "Set ruby action") do |v|
+    options[:ruby_action] = v
+  end
+
+  opts.on("-d", "--delete", "Delete all matches") do |v|
+    options[:ruby_action] = ":delete"
+  end
+
+  opts.on("-c QUEUE", "--copy QUEUE", "Copy all matches") do |v|
+    options[:ruby_action] = "SuperPoller::QueueUrl.parse(#{v.to_s.inspect}).to_queue.push(msg)"
+  end
+
+  opts.on("-m QUEUE", "--move QUEUE", "Move all matches") do |v|
+    options[:ruby_action] = "SuperPoller::QueueUrl.parse(#{v.to_s.inspect}).to_queue.push(msg); :delete"
+  end
+
+  opts.on("-t TYPE", "--message-type TYPE", "Select messages of a given type") do |v|
+    options[:ruby_filter] = "#{v.to_s.inspect} == msg[:name].to_s rescue false"
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit 1
+  end
+end
+
+opts.parse!
+
+unless queue_url = ARGV.shift
+  puts opts
+  exit 1
+end
+
+queue = SuperPoller::QueueUrl.parse(queue_url).to_queue
+
+eval "def matcher_matches?(msg); #{ options[:ruby_filter] || "true" }; end"
+eval "def action(msg); #{ options[:ruby_action] || "p msg" }; end"
+
+SuperPoller::QueueItterator.new(queue).each do |msg|
+  action(msg) if matcher_matches? msg
+end

data/bin/scripts/queue_interactive_move

@@ -0,0 +1,13 @@
+require "super_poller"
+queue_one = SuperPoller::QueueUrl.parse(ARGV.shift).to_queue
+queue_two = SuperPoller::QueueUrl.parse(ARGV.shift).to_queue
+
+SuperPoller::QueueItterator.new(queue_one).each do |msg|
+  p msg
+  print "Move to #{queue_two}? > "
+
+  if gets =~ /^[Yy]/
+    queue_two.push(msg)
+    :delete
+  end
+end

data/bin/scripts/queue_move

@@ -0,0 +1,4 @@
+require "super_poller"
+in_queue, out_queue = [nil,nil].map{|url| SuperPoller::QueueUrl.parse(ARGV.shift).to_queue }
+
+SuperPoller::NoneBlockingPoller.new(in_queue, out_queue.method(:push)).start!

data/bin/scripts/queue_scan

@@ -0,0 +1,3 @@
+require "super_poller"
+queue = SuperPoller::QueueUrl.parse(ARGV.shift).to_queue
+SuperPoller::QueueItterator.new(queue).each(&method(:p))

data/lib/super_poller/aggregating_error_logger.rb

@@ -0,0 +1,33 @@
+class SuperPoller::AggregatingErrorLogger
+  def initialize(stats_file, queue)
+    @stats_file, @queue = stats_file, queue
+  end
+
+  def call(error, failed_message)
+    update_error_queue(error, failed_message)
+    update_error_stats(error, failed_message[:name] || :unknown)
+  end
+
+protected
+  def update_error_queue(error, failed_message)
+    error_class_name = error.class.name.to_sym
+    error_description = {:class => error_class_name, :message => error.message}
+    @queue.push(failed_message.merge(:error => error_description))
+  end
+
+  def update_error_stats(error, message_name)
+    stats = load_stats
+    error_class_name = error.class.name.to_sym
+    stats_for_name = (stats[message_name] ||= {})
+    stats_for_name[error_class_name] ||= 0
+    stats_for_name[error_class_name] += 1
+
+    File.open(@stats_file, "w") do |file|
+      file << YAML.dump(stats)
+    end
+  end
+
+  def load_stats
+    File.exists?(@stats_file) && YAML.load(File.read(@stats_file)) || {}
+  end
+end

data/lib/super_poller/buffered_handler.rb

@@ -0,0 +1,28 @@
+class SuperPoller::BufferedHandler < SuperPoller::Handler
+  class << self
+    def buffer_size(size)
+      @max_buffer_size = size
+    end
+
+    def max_buffer_size
+      @max_buffer_size || []
+    end
+  end
+
+  def initialize
+    @buffer = []
+  end
+
+  def call(msg)
+    @buffer.push msg
+    if @buffer.size >= self.class.max_buffer_size
+      handle_batch @buffer
+      @buffer = []
+    end
+  end
+
+  def handle_batch(batch)
+    raise NotImplementedError, "You must define a batch handler."
+  end
+
+end

data/lib/super_poller/error_reporter.rb

@@ -0,0 +1,12 @@
+class SuperPoller::ErrorReporter
+  def initialize(message_handler, error_handler = nil, &block)
+    @message_handler = message_handler
+    @error_handler = error_handler || block
+  end
+  
+  def call(*args)
+    @message_handler.call(*args)
+  rescue StandardError => e
+    @error_handler.call(e, *args)
+  end
+end

data/lib/super_poller/handler.rb

@@ -0,0 +1,22 @@
+class SuperPoller::Handler
+  autoload :TestCase, "super_poller/test_case"
+  
+  class << self
+    def handles(*new_message_names)
+      @message_names = (message_names + new_message_names).uniq
+    end
+
+    def message_names
+      @message_names || []
+    end
+  end
+
+  def can_handle?(message)
+    self.class.message_names.include? message[:name].to_sym
+  end
+
+  def call(message)
+    raise NotImplementedError, "You must define a call handler."
+  end
+
+end

data/lib/super_poller/none_blocking_poller.rb

@@ -0,0 +1,13 @@
+class SuperPoller::NoneBlockingPoller < SuperPoller::Poller
+  EmptyQueue = Class.new(Exception)
+
+  def start!
+    super
+  rescue EmptyQueue
+  end
+
+protected
+  def get_message
+    @queue.fetch or raise EmptyQueue
+  end
+end

data/lib/super_poller/poller.rb

@@ -0,0 +1,24 @@
+class SuperPoller::Poller
+  def initialize(queue, message_handler)
+    @message_handler, @queue = message_handler, queue
+  end
+
+  def poll
+    @message_handler.call(get_message)
+  end
+
+  def start!
+    poll while true
+  end
+
+protected
+  def get_message
+    @queue.pop
+  rescue Interrupt
+    raise
+  rescue Object => e
+    STDERR.puts "Error while fetching from the queue: #{e.class}: #{e.message}"
+    sleep 10
+    retry
+  end
+end

data/lib/super_poller/queue_itterator.rb

@@ -0,0 +1,30 @@
+class SuperPoller::QueueItterator
+  def initialize(queue)
+    @queue = queue
+  end
+
+  def each(&block)
+    @memo = {:memo => Time.now.to_f}
+    @queue.push @memo
+    while @memo
+      begin
+        break unless msg = @queue.fetch
+        if msg == @memo
+          msg = @memo = nil
+        else
+          msg = nil if :delete == block.call(msg)
+        end
+      ensure
+        @queue.push msg if msg
+      end
+    end
+  ensure
+    destroy_the_memo
+  end
+
+  def destroy_the_memo
+    while @memo and msg = @queue.fetch and msg != @memo
+      @queue.push(msg)
+    end
+  end
+end

data/lib/super_poller/queue_url.rb

@@ -0,0 +1,18 @@
+require "uri"
+
+class SuperPoller::QueueUrl < URI::Generic
+  def self.parse(url)
+    url = "starling://localhost:22122/#{url}" if url =~ /^[a-zA-Z0-9_-]+$/
+    new(*URI.parse(url).send(:component_ary))
+  end
+
+  def to_queue
+    raise URI::InvalidURIError unless respond_to? "to_#{scheme}_queue"
+    send("to_#{scheme}_queue")
+  end
+
+protected
+  def to_starling_queue
+    SuperPoller::StarlingQueue.new(path, "#{host}:#{port}")
+  end
+end

data/lib/super_poller/router.rb

@@ -0,0 +1,29 @@
+class SuperPoller::Router
+  RoutingError = Class.new(Exception)
+
+  def initialize()
+    @handlers = []
+  end
+
+  def add_handler(handler)
+    @handlers.push handler
+    @handlers.uniq!
+    self
+  end
+
+  alias << add_handler
+
+  def call(message)
+    handler = best_handler_for_message(message)
+    handler.call(message[:body])
+  end
+
+protected
+
+  def best_handler_for_message(messsage)
+    @handlers.each do |handler|
+      return handler if handler.can_handle? messsage
+    end
+    raise RoutingError, "No handler found"
+  end
+end

data/lib/super_poller/starling_queue.rb

@@ -0,0 +1,20 @@
+require "starling"
+
+class SuperPoller::StarlingQueue
+  def initialize(queue_name, *args)
+    @queue_name = queue_name.to_s
+    @queue = Starling.new(*args)
+  end
+
+  def pop
+    @queue.get(@queue_name)
+  end
+
+  def push(v)
+    @queue.set(@queue_name, v)
+  end
+
+  def fetch
+    @queue.fetch(@queue_name)
+  end
+end

data/lib/super_poller/test_case.rb

@@ -0,0 +1,13 @@
+class SuperPoller::Handler::TestCase < (defined?(ActiveSupport::TestCase) ? ActiveSupport::TestCase : Test::Unit::TestCase)
+
+  def handler
+    @handler ||= self.class.name.gsub(/Test$/, "").constantize.new
+  end
+
+  def self.should_handle(name)
+    should "handle a #{name.inspect} message" do
+      assert handler.can_handle?(:name => name)
+    end
+  end
+
+end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - Tom Lea
@@ -29,6 +29,24 @@ extra_rdoc_files: []
 
 files: 
 - bin/queue
+- bin/scripts/queue_choose
+- bin/scripts/queue_copy
+- bin/scripts/queue_flush
+- bin/scripts/queue_grep
+- bin/scripts/queue_interactive_move
+- bin/scripts/queue_move
+- bin/scripts/queue_scan
+- lib/super_poller/aggregating_error_logger.rb
+- lib/super_poller/buffered_handler.rb
+- lib/super_poller/error_reporter.rb
+- lib/super_poller/handler.rb
+- lib/super_poller/none_blocking_poller.rb
+- lib/super_poller/poller.rb
+- lib/super_poller/queue_itterator.rb
+- lib/super_poller/queue_url.rb
+- lib/super_poller/router.rb
+- lib/super_poller/starling_queue.rb
+- lib/super_poller/test_case.rb
 - lib/super_poller.rb
 - test/aggregating_error_logger_test.rb
 - test/buffered_handler_test.rb