sneakers 0.0.1 → 0.0.2

data/lib/sneakers/runner.rb

@@ -0,0 +1,20 @@
+require 'serverengine'
+require 'sneakers/workergroup'
+require 'sneakers/runner_config'
+
+module Sneakers
+  class Runner
+    def initialize(worker_classes, opts={})
+      @runnerconfig = RunnerConfig.new(worker_classes)
+    end
+
+    def run
+      @se = ServerEngine.create(nil, WorkerGroup) { @runnerconfig.reload_config! }
+      @se.run
+    end
+
+    def stop
+      @se.stop
+    end
+  end
+end

data/lib/sneakers/runner_config.rb

@@ -0,0 +1,55 @@
+module Sneakers
+  class RunnerConfig
+    def method_missing(meth, *args, &block)
+      if %w{ before_fork after_fork }.include? meth.to_s
+        @conf[meth] = block
+      elsif %w{ workers start_worker_delay }.include? meth.to_s
+        @conf[meth] = args.first
+      else
+        super
+      end
+    end
+
+    def initialize(worker_classes)
+      @worker_classes = worker_classes
+      @conf = {}
+    end
+
+    def to_h
+      @conf
+    end
+
+
+    def reload_config!
+      Sneakers.logger.warn("Loading runner configuration...")
+      config_file = Sneakers::Config[:runner_config_file]
+
+      if config_file
+        begin
+          instance_eval(File.read(config_file), config_file)
+          Sneakers.logger.info("Loading config with file: #{config_file}")
+        rescue
+          Sneakers.logger.error("Cannot load from file '#{config_file}', #{$!}")
+        end
+      end
+
+      config = make_serverengine_config
+
+      [:before_fork, :after_fork].each do | hook |
+        Sneakers::Config[:hooks][hook] = config.delete(hook) if config[hook]
+      end
+
+
+      Sneakers.logger.info("New configuration: #{config.inspect}")
+      config
+    end
+
+  private
+    def make_serverengine_config
+      Sneakers::Config.merge(@conf).merge({
+        :worker_type => 'process',
+        :worker_classes => @worker_classes
+      })
+    end
+  end
+end

data/lib/sneakers/support/production_formatter.rb

@@ -0,0 +1,11 @@
+require 'time'
+module Sneakers
+  module Support
+    class ProductionFormatter < Logger::Formatter
+        def self.call(severity, time, program_name, message)
+          "#{time.utc.iso8601} p-#{Process.pid} t-#{Thread.current.object_id.to_s(36)} #{severity}: #{message}\n"
+        end
+    end
+  end
+end
+

data/lib/sneakers/support/queue_name.rb

@@ -0,0 +1,14 @@
+module Sneakers
+  module Support
+    class QueueName
+      def initialize(queue, opts)
+        @queue = queue
+        @opts = opts
+      end
+
+      def to_s
+        [@queue, @opts[:env]].compact.join('_')
+      end
+    end
+  end
+end

data/lib/sneakers/support/utils.rb

@@ -0,0 +1,18 @@
+class Sneakers::Utils
+  def self.make_worker_id(namespace)
+    "worker-#{namespace}:#{'1'}:#{rand(36**6).floor.to_s(36)}"  # jid, worker id. include date.
+  end
+  def self.parse_workers(workerstring)
+    missing_workers = []
+    workers = (workerstring || '').split(',').map do |k|
+      begin
+        w = Object.const_get(k)
+      rescue
+        missing_workers << k
+      end
+      w
+    end.compact
+
+    [workers, missing_workers]
+  end
+end

data/lib/sneakers/tasks.rb

@@ -0,0 +1,34 @@
+require 'sneakers'
+require 'sneakers/runner'
+
+task :environment
+
+namespace :sneakers do
+  desc "Start work (set $WORKERS=Klass1,Klass2)"
+  task :run  => :environment do
+
+    workers, missing_workers = Sneakers::Utils.parse_workers(ENV['WORKERS'])
+
+    unless missing_workers.empty?
+      puts "Missing workers: #{missing_workers.join(', ')}" if missing_workers
+      puts "Did you `require` properly?"
+      exit(1)
+    end
+
+    if workers.empty?
+      puts <<EOF
+Error: No workers found.
+Please set the classes of the workers you want to run like so:
+
+  $ export WORKERS=MyWorker,FooWorker
+  $ rake sneakers:work
+
+EOF
+      exit(1)
+    end
+
+    r = Sneakers::Runner.new(workers)
+
+    r.run
+  end
+end

data/lib/sneakers/version.rb

@@ -1,3 +1,3 @@
 module Sneakers
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/sneakers/worker.rb

@@ -0,0 +1,120 @@
+require 'sneakers/queue'
+require 'sneakers/support/utils'
+require 'sneakers/support/queue_name'
+require 'timeout'
+
+module Sneakers
+  module Worker
+    attr_reader :queue, :id
+
+    # For now, a worker is hardly dependant on these concerns
+    # (because it uses methods from them directly.)
+    include Concerns::Logging
+    include Concerns::Metrics
+
+    def initialize(queue=nil, pool=nil, opts=nil)
+      opts = self.class.queue_opts
+      queue_name = self.class.queue_name
+
+      opts = Sneakers::Config.merge(opts)
+      queue_name = Support::QueueName.new(queue_name, opts).to_s
+
+      @should_ack =  opts[:ack]
+      @timeout_after = opts[:timeout_job_after]
+      @pool = pool || Thread.pool(opts[:threads]) # XXX config threads
+
+      @queue = queue || Sneakers::Queue.new(
+        queue_name,
+        :prefetch => opts[:prefetch],
+        :durable => opts[:durable],
+        :ack => @should_ack,
+        :heartbeat_interval => opts[:heartbeat_interval],
+        :exchange => opts[:exchange]
+      )
+
+      @opts = opts
+      @id = Utils.make_worker_id(queue_name)
+    end
+
+    def ack!; :ack end
+    def nack!; :nack end
+    def reject!; :reject; end
+
+    def publish(msg, routing)
+      return unless routing[:to_queue]
+      @queue.exchange.publish(msg, :routing_key => QueueName.new(routing[:to_queue], @opts).to_s)
+    end
+
+    def do_work(hdr, props, msg, handler)
+      worker_trace "Working off: #{msg}"
+
+      @pool.process do
+        res = nil
+        error = nil
+
+        begin
+          metrics.increment("work.#{self.class.name}.started")
+          Timeout.timeout(@timeout_after) do
+            metrics.timing("work.#{self.class.name}.time") do
+              res = work(msg)
+            end
+          end
+        rescue Timeout::Error
+          res = :timeout
+          logger.error("timeout")
+        rescue => ex
+          res = :error
+          error = ex
+          logger.error(ex)
+        end
+
+        if @should_ack
+          if res == :ack
+            # note to future-self. never acknowledge multiple (multiple=true) messages under threads.
+            handler.acknowledge(hdr.delivery_tag)
+          elsif res == :timeout
+            handler.timeout(hdr.delivery_tag)
+          elsif res == :error
+            handler.error(hdr.delivery_tag, error)
+          else
+            handler.reject(hdr.delivery_tag)
+          end
+          metrics.increment("work.#{self.class.name}.handled.#{res || 'reject'}")
+        end
+
+        metrics.increment("work.#{self.class.name}.ended")
+      end #process
+    end
+
+    def stop
+      worker_trace "Stopping worker: unsubscribing."
+      @queue.unsubscribe
+      worker_trace "Stopping worker: I'm gone."
+    end
+
+    def run
+      worker_trace "New worker: subscribing."
+      @queue.subscribe(self)
+      worker_trace "New worker: I'm alive."
+    end
+
+    def worker_trace(msg)
+      logger.debug "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
+    end
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      attr_reader :queue_opts
+      attr_reader :queue_name
+
+      def from_queue(q, opts={})
+        @queue_name = q.to_s
+        @queue_opts = opts
+      end
+    end
+  end
+end
+

data/lib/sneakers/workergroup.rb

@@ -0,0 +1,47 @@
+module Sneakers
+  module WorkerGroup
+    @workers = []
+
+    def initialize
+      @stop_flag = ServerEngine::BlockingFlag.new
+    end
+
+    def before_fork
+      fbefore = Sneakers::Config[:hooks][:before_fork]
+      fbefore.call if fbefore
+    end
+
+    def after_fork # note! this is not Serverengine#after_start, this is ours!
+      fafter = Sneakers::Config[:hooks][:after_fork]
+      fafter.call if fafter
+    end
+
+    def run
+      after_fork
+
+      @workers = config[:worker_classes].map{|w| w.new }
+      # if more than one worker this should be per worker
+      # accumulate clients and consumers as well
+      @workers.each do |worker|
+        worker.run
+      end
+      # end per worker
+
+      until @stop_flag.wait_for_set(10.0)
+        Sneakers.logger.info("Heartbeat: running threads [#{Thread.list.count}]")
+        # report aggregated stats?
+      end
+
+    end
+
+    def stop
+      Sneakers.logger.info("Shutting down workers")
+      @workers.each do |worker|
+        worker.stop
+      end
+      @stop_flag.set!
+    end
+
+  end
+end
+

data/sneakers.gemspec

@@ -1,23 +1,33 @@
-# coding: utf-8
+# -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'sneakers/version'
 
-Gem::Specification.new do |spec|
-  spec.name          = "sneakers"
-  spec.version       = Sneakers::VERSION
-  spec.authors       = ["jondot"]
-  spec.email         = ["jondotan@gmail.com"]
-  spec.description   = %q{}
-  spec.summary       = %q{}
-  spec.homepage      = ""
-  spec.license       = "MIT"
+Gem::Specification.new do |gem|
+  gem.name          = "sneakers"
+  gem.version       = Sneakers::VERSION
+  gem.authors       = ["Dotan Nahum"]
+  gem.email         = ["jondotan@gmail.com"]
+  gem.description   = %q{Fast background processing framework for Ruby and RabbitMQ}
+  gem.summary       = %q{Fast background processing framework for Ruby and RabbitMQ}
+  gem.homepage      = ""
 
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency "serverengine"
+  gem.add_dependency "bunny", ">= 0.9.0.rc2"
+  gem.add_dependency "thread"
+  gem.add_dependency "thor"
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
+  gem.add_development_dependency "rr"
+  gem.add_development_dependency "ruby-prof"
+  gem.add_development_dependency "nokogiri"
+  gem.add_development_dependency "guard-minitest"
+  gem.add_development_dependency "metric_fu"
+  gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "simplecov-rcov-text"
+  gem.add_development_dependency "rake"
 end
+

data/spec/fixtures/require_worker.rb

@@ -0,0 +1,17 @@
+require 'sneakers'
+require 'open-uri'
+require 'nokogiri'
+
+
+class TitleScraper
+  include Sneakers::Worker
+
+  from_queue 'downloads'
+
+  def work(msg)
+    doc = Nokogiri::HTML(open(msg))
+    worker_trace "FOUND <#{doc.css('title').text}>"
+    ack!
+  end
+end
+

data/spec/sneakers/cli_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+require 'sneakers'
+require 'sneakers/cli'
+require 'sneakers/runner'
+
+describe Sneakers::CLI do
+  describe "#work" do
+    before do
+      any_instance_of(Sneakers::Runner) do |runner|
+        stub(runner).run{ true }
+      end
+    end
+
+    describe 'with dirty class loading' do
+      after do
+        # require cleanup
+        Object.send(:remove_const, :TitleScraper)
+      end
+
+      it "should perform a run" do
+        any_instance_of(Sneakers::Runner) do |runner|
+          mock(runner).run{ true }
+        end
+        out = capture_io{ Sneakers::CLI.start [
+          'work',
+          "TitleScraper",
+          "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
+        ]}.join ''
+
+        out.must_match(/Workers.*:.*TitleScraper.*/)
+
+      end
+
+      it "should be able to run as front-running process" do
+        out = capture_io{ Sneakers::CLI.start [
+          'work',
+          "TitleScraper",
+          "--front",
+          "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
+        ]}.join ''
+
+        out.must_match(/Log.*Console/)
+      end
+    end
+
+    it "should fail when no workers found" do
+      out = capture_io{ Sneakers::CLI.start ['work', 'TitleScraper'] }.join ''
+      out.must_match(/Missing workers: TitleScraper/)
+    end
+
+  end
+end
+