sneakers_custom_bunny 1.0.4

data/lib/sneakers/spawner.rb

@@ -0,0 +1,27 @@
+require 'yaml'
+
+module Sneakers
+  class Spawner
+
+    def self.spawn
+      worker_group_config_file = ENV['WORKER_GROUP_CONFIG'] || "./config/sneaker_worker_groups.yml"
+      unless File.exists?(worker_group_config_file)
+        puts "No worker group file found."
+        puts "Specify via ENV 'WORKER_GROUP_CONFIG' or by convention ./config/sneaker_worker_groups.yml"
+      end
+      @pids = []
+      @exec_string = "bundle exec rake sneakers:run"
+      worker_config = YAML.load(File.read(worker_group_config_file))
+      worker_config.keys.each do |group_name|
+        @pids << fork do
+          @exec_hash = {"WORKERS"=> worker_config[group_name]['classes'], "WORKER_COUNT" => worker_config[group_name]["workers"].to_s}
+          Kernel.exec(@exec_hash, @exec_string)
+        end
+      end
+      ["TERM", "USR1", "HUP", "USR2"].each do |signal|
+        Signal.trap(signal){ @pids.each{|pid| Process.kill(signal, pid) } }
+      end
+      Process.waitall
+    end
+  end
+end

data/lib/sneakers/support/production_formatter.rb

@@ -0,0 +1,11 @@
+require 'time'
+module Sneakers
+  module Support
+    class ProductionFormatter < Logger::Formatter
+        def self.call(severity, time, program_name, message)
+          "#{time.utc.iso8601} p-#{Process.pid} t-#{Thread.current.object_id.to_s(36)} #{severity}: #{message}\n"
+        end
+    end
+  end
+end
+

data/lib/sneakers/support/utils.rb

@@ -0,0 +1,18 @@
+class Sneakers::Utils
+  def self.make_worker_id(namespace)
+    "worker-#{namespace}:#{'1'}:#{rand(36**6).floor.to_s(36)}"  # jid, worker id. include date.
+  end
+  def self.parse_workers(workerstring)
+    missing_workers = []
+    workers = (workerstring || '').split(',').map do |k|
+      begin
+        w = k.split('::').inject(Kernel){|s, c| s.const_get(c)}
+      rescue
+        missing_workers << k
+      end
+      w
+    end.compact
+
+    [workers, missing_workers]
+  end
+end

data/lib/sneakers/tasks.rb

@@ -0,0 +1,34 @@
+require 'sneakers'
+require 'sneakers/runner'
+
+task :environment
+
+namespace :sneakers do
+  desc "Start work (set $WORKERS=Klass1,Klass2)"
+  task :run  => :environment do
+
+    workers, missing_workers = Sneakers::Utils.parse_workers(ENV['WORKERS'])
+
+    unless missing_workers.empty?
+      puts "Missing workers: #{missing_workers.join(', ')}" if missing_workers
+      puts "Did you `require` properly?"
+      exit(1)
+    end
+
+    if workers.empty?
+      puts <<EOF
+Error: No workers found.
+Please set the classes of the workers you want to run like so:
+
+  $ export WORKERS=MyWorker,FooWorker
+  $ rake sneakers:run
+
+EOF
+      exit(1)
+    end
+    opts = (ENV['WORKER_COUNT'].present? ? {:workers => ENV['WORKER_COUNT'].to_i} : {})
+    r = Sneakers::Runner.new(workers, opts)
+
+    r.run
+  end
+end

data/lib/sneakers/version.rb

@@ -0,0 +1,3 @@
+module Sneakers
+  VERSION = "1.0.4"
+end

data/lib/sneakers/worker.rb

@@ -0,0 +1,151 @@
+require 'sneakers/queue'
+require 'sneakers/support/utils'
+require 'timeout'
+
+module Sneakers
+  module Worker
+    attr_reader :queue, :id, :opts
+
+    # For now, a worker is hardly dependant on these concerns
+    # (because it uses methods from them directly.)
+    include Concerns::Logging
+    include Concerns::Metrics
+
+    def initialize(queue = nil, pool = nil, opts = {})
+      opts = opts.merge(self.class.queue_opts || {})
+      queue_name = self.class.queue_name
+      opts = Sneakers::CONFIG.merge(opts)
+
+      @should_ack =  opts[:ack]
+      @timeout_after = opts[:timeout_job_after]
+      @pool = pool || Thread.pool(opts[:threads]) # XXX config threads
+      @call_with_params = respond_to?(:work_with_params)
+
+      @queue = queue || Sneakers::Queue.new(
+        queue_name,
+        opts
+      )
+
+      @opts = opts
+      @id = Utils.make_worker_id(queue_name)
+    end
+
+    def ack!; :ack end
+    def reject!; :reject; end
+    def requeue!; :requeue; end
+
+    def publish(msg, opts)
+      to_queue = opts.delete(:to_queue)
+      opts[:routing_key] ||= to_queue
+      return unless opts[:routing_key]
+      @queue.exchange.publish(msg, opts)
+    end
+
+    def do_work(delivery_info, metadata, msg, handler)
+      worker_trace "Working off: #{msg}"
+
+      @pool.process do
+        res = nil
+        error = nil
+
+        begin
+          metrics.increment("work.#{self.class.name}.started")
+          Timeout.timeout(@timeout_after, Timeout::Error) do
+            metrics.timing("work.#{self.class.name}.time") do
+              if @call_with_params
+                res = work_with_params(msg, delivery_info, metadata)
+              else
+                res = work(msg)
+              end
+            end
+          end
+        rescue Timeout::Error
+          res = :timeout
+          worker_error('timeout')
+        rescue => ex
+          res = :error
+          error = ex
+          worker_error('unexpected error', ex)
+        end
+
+        if @should_ack
+
+          if res == :ack
+            # note to future-self. never acknowledge multiple (multiple=true) messages under threads.
+            handler.acknowledge(delivery_info, metadata, msg)
+          elsif res == :timeout
+            handler.timeout(delivery_info, metadata, msg)
+          elsif res == :error
+            handler.error(delivery_info, metadata, msg, error)
+          elsif res == :reject
+            handler.reject(delivery_info, metadata, msg)
+          elsif res == :requeue
+            handler.reject(delivery_info, metadata, msg, true)
+          else
+            handler.noop(delivery_info, metadata, msg)
+          end
+          metrics.increment("work.#{self.class.name}.handled.#{res || 'noop'}")
+        end
+
+        metrics.increment("work.#{self.class.name}.ended")
+      end #process
+    end
+
+    def stop
+      worker_trace "Stopping worker: unsubscribing."
+      @queue.unsubscribe
+      worker_trace "Stopping worker: I'm gone."
+    end
+
+    def run
+      worker_trace "New worker: subscribing."
+      @queue.subscribe(self)
+      worker_trace "New worker: I'm alive."
+    end
+
+    # Construct a log message with some standard prefix for this worker
+    def log_msg(msg)
+      "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
+    end
+
+    # Helper to log an error message with an optional exception
+    def worker_error(msg, exception = nil)
+      s = log_msg(msg)
+      if exception
+        s += " [Exception error=#{exception.message.inspect} error_class=#{exception.class}"
+        s += " backtrace=#{exception.backtrace.take(50).join(',')}" unless exception.backtrace.nil?
+        s += "]"
+      end
+      logger.error(s)
+    end
+
+    def worker_trace(msg)
+      logger.debug(log_msg(msg))
+    end
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      attr_reader :queue_opts
+      attr_reader :queue_name
+
+      def from_queue(q, opts={})
+        @queue_name = q.to_s
+        @queue_opts = opts
+      end
+
+      def enqueue(msg)
+        publisher.publish(msg, :to_queue => @queue_name)
+      end
+
+      private
+
+      def publisher
+        @publisher ||= Sneakers::Publisher.new
+      end
+    end
+  end
+end
+

data/lib/sneakers/workergroup.rb

@@ -0,0 +1,47 @@
+module Sneakers
+  module WorkerGroup
+    @workers = []
+
+    def initialize
+      @stop_flag = ServerEngine::BlockingFlag.new
+    end
+
+    def before_fork
+      fbefore = Sneakers::CONFIG[:hooks][:before_fork]
+      fbefore.call if fbefore
+    end
+
+    def after_fork # note! this is not Serverengine#after_start, this is ours!
+      fafter = Sneakers::CONFIG[:hooks][:after_fork]
+      fafter.call if fafter
+    end
+
+    def run
+      after_fork
+
+      @workers = config[:worker_classes].map{|w| w.new }
+      # if more than one worker this should be per worker
+      # accumulate clients and consumers as well
+      @workers.each do |worker|
+        worker.run
+      end
+      # end per worker
+      #
+      until @stop_flag.wait_for_set(10.0)
+        Sneakers.logger.debug("Heartbeat: running threads [#{Thread.list.count}]")
+        # report aggregated stats?
+      end
+
+    end
+
+    def stop
+      Sneakers.logger.info("Shutting down workers")
+      @workers.each do |worker|
+        worker.stop
+      end
+      @stop_flag.set!
+    end
+
+  end
+end
+

data/sneakers.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sneakers/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'sneakers_custom_bunny'
+  gem.version       = Sneakers::VERSION
+  gem.authors       = ['Dotan Nahum']
+  gem.email         = ['jondotan@gmail.com']
+  gem.description   = %q( Fast background processing framework for Ruby and RabbitMQ )
+  gem.summary       = %q( Fast background processing framework for Ruby and RabbitMQ )
+  gem.homepage      = ''
+
+  gem.files         = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' }
+  gem.executables   = gem.files.grep(/^bin/).map { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(/^(test|spec|features)\//)
+  gem.require_paths = ['lib']
+  gem.add_dependency 'serverengine', '~> 1.5.5'
+  gem.add_dependency 'bunny', '~> 1.7.0'
+  gem.add_dependency 'thread', '~> 0.1.7'
+  gem.add_dependency 'thor'
+
+  gem.add_development_dependency 'rr'
+  gem.add_development_dependency 'ruby-prof'
+  gem.add_development_dependency 'nokogiri'
+  gem.add_development_dependency 'guard-minitest'
+  gem.add_development_dependency 'metric_fu'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'simplecov-rcov-text'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'minitest'
+  gem.add_development_dependency 'guard'
+end
+

data/spec/fixtures/require_worker.rb

@@ -0,0 +1,17 @@
+require 'sneakers'
+require 'open-uri'
+require 'nokogiri'
+
+
+class TitleScraper
+  include Sneakers::Worker
+
+  from_queue 'downloads'
+
+  def work(msg)
+    doc = Nokogiri::HTML(open(msg))
+    worker_trace "FOUND <#{doc.css('title').text}>"
+    ack!
+  end
+end
+

data/spec/sneakers/cli_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+require 'sneakers'
+require 'sneakers/cli'
+require 'sneakers/runner'
+
+describe Sneakers::CLI do
+  describe "#work" do
+    before do
+      any_instance_of(Sneakers::Runner) do |runner|
+        stub(runner).run{ true }
+      end
+    end
+
+    describe 'with dirty class loading' do
+      after do
+        # require cleanup
+        Object.send(:remove_const, :TitleScraper)
+      end
+
+      it "should perform a run" do
+        any_instance_of(Sneakers::Runner) do |runner|
+          mock(runner).run{ true }
+        end
+        out = capture_io{ Sneakers::CLI.start [
+          'work',
+          "TitleScraper",
+          "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
+        ]}.join ''
+
+        out.must_match(/Workers.*:.*TitleScraper.*/)
+
+      end
+
+      it "should be able to run as front-running process" do
+        out = capture_io{ Sneakers::CLI.start [
+          'work',
+          "TitleScraper",
+          "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
+        ]}.join ''
+
+        out.must_match(/Log.*Console/)
+      end
+
+      it "should be able to run as daemonized process" do
+        out = capture_io{ Sneakers::CLI.start [
+          'work',
+          "TitleScraper",
+          "--daemonize",
+          "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
+        ]}.join ''
+
+        out.must_match(/sneakers.log/)
+      end
+    end
+
+    it "should fail when no workers found" do
+      out = capture_io{ Sneakers::CLI.start ['work', 'TitleScraper'] }.join ''
+      out.must_match(/Missing workers: TitleScraper/)
+    end
+
+  end
+end
+

data/spec/sneakers/concerns/logging_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+require 'sneakers'
+require 'logger'
+
+
+class Foobar
+  include Sneakers::Concerns::Logging
+end
+
+describe Sneakers::Concerns::Logging do
+  describe ".configure" do
+    before do
+      Foobar.logger = nil
+    end
+
+    it "should configure a default logger when included" do
+      Foobar.logger.must_be_nil
+      Foobar.configure_logger
+      Foobar.logger.wont_be_nil
+      Foobar.logger.formatter.must_equal Sneakers::Support::ProductionFormatter
+    end
+
+    it "should supply accessible instance logger" do
+      Foobar.logger.must_be_nil
+      Foobar.configure_logger
+      f = Foobar.new
+      f.logger.must_equal Foobar.logger
+      f.logger.wont_be_nil
+    end
+
+    it "should configure a given logger when specified" do
+      Foobar.logger.must_be_nil
+      log = Logger.new(STDOUT)
+      Foobar.configure_logger(log)
+      Foobar.logger.must_equal log
+    end
+  end
+end
+