sneakers 0.0.1 → 0.0.2

data/examples/sneakers.conf.rb.example

@@ -0,0 +1,10 @@
+workers 2
+
+before_fork do
+  Sneakers::logger.info " ** im before-fork'en ** "
+end
+
+
+after_fork do
+  Sneakers::logger.info " !! im after forke'n !! "
+end

data/examples/title_scraper.rb

@@ -0,0 +1,20 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+require 'open-uri'
+require 'nokogiri'
+
+
+class TitleScraper
+  include Sneakers::Worker
+
+  from_queue 'downloads'
+
+  def work(msg)
+    doc = Nokogiri::HTML(open(msg))
+    worker_trace "FOUND <#{doc.css('title').text}>"
+    ack!
+  end
+end
+
+
+

data/examples/workflow_worker.rb

@@ -0,0 +1,24 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+
+class WorkflowWorker
+  include Sneakers::Worker
+  from_queue 'downloads',
+             :env => 'test',
+             :durable => false,
+             :ack => true,
+             :threads => 50,
+             :prefetch => 50,
+             :timeout_job_after => 1,
+             :exchange => 'dummy',
+             :heartbeat_interval => 5
+
+  def work(msg)
+    logger.info("Seriously, i'm DONE.")
+    publish "cleaned up", :to_queue => "foobar"
+    logger.info("Published to 'foobar'")
+    ack!
+  end
+end
+
+

data/lib/sneakers.rb

@@ -1,5 +1,84 @@
 require "sneakers/version"
+require 'thread/pool'
+require 'bunny'
+
+
+module Sneakers
+  module Handlers
+  end
+  module Concerns
+  end
+end
+
+require 'sneakers/support/production_formatter'
+require 'sneakers/concerns/logging'
+require 'sneakers/concerns/metrics'
+require 'sneakers/handlers/oneshot'
+require 'sneakers/worker'
+require 'sneakers/publisher'
 
 module Sneakers
-  # Your code goes here...
+
+  Config = {
+    # runner
+    :runner_config_file => nil,
+    :metrics => nil,
+    :daemonize => true,
+    :start_worker_delay => 0.2,
+    :workers => 4,
+    :log  => 'sneakers.log',
+    :pid_path => 'sneakers.pid',
+
+    #workers
+    :timeout_job_after => 5,
+    :prefetch => 10,
+    :threads => 10,
+    :env => ENV['RACK_ENV'],
+    :durable => true,
+    :ack => true,
+    :heartbeat_interval => 2,
+    :exchange => 'sneakers',
+    :hooks => {}
+  }
+
+  def self.configure(opts={})
+    # worker > userland > defaults
+    Config.merge!(opts)
+
+    setup_general_logger!
+    setup_worker_concerns!
+    setup_general_publisher!
+    @configured = true
+  end
+
+  def self.logger
+    @logger
+  end
+
+  def self.publish(msg, routing)
+    @publisher.publish(msg, routing)
+  end
+
+  def self.configured?
+    @configured
+  end
+
+
+private
+
+  def self.setup_general_logger!
+    @logger = Logger.new(Config[:log])
+    @logger.formatter = Sneakers::Support::ProductionFormatter
+  end
+
+  def self.setup_worker_concerns!
+    Worker.configure_logger(Sneakers::logger)
+    Worker.configure_metrics(Config[:metrics])
+    Config[:handler] ||= Sneakers::Handlers::Oneshot
+  end
+
+  def self.setup_general_publisher!
+    @publisher = Sneakers::Publisher.new
+  end
 end
+

data/lib/sneakers/cli.rb

@@ -0,0 +1,107 @@
+require 'thor'
+require 'sneakers/runner'
+
+
+#
+# $ sneakers run TitleWorker,FooWorker
+# $ sneakers stop
+# $ sneakers recycle
+# $ sneakers reload
+# $ sneakers init
+#
+#
+module Sneakers
+  class CLI < Thor
+
+    SNEAKERS=<<-EOF
+
+        __
+    ,--'  >  Sneakers
+    `=====
+
+    EOF
+
+    BANNER = SNEAKERS
+
+    method_option :debug
+    method_option :front
+    method_option :require
+    desc "work FirstWorker,SecondWorker ... ,NthWorker", "Run workers"
+    def work(workers)
+      require_boot File.expand_path(options[:require]) if options[:require]
+
+      workers, missing_workers = Sneakers::Utils.parse_workers(workers)
+
+      unless missing_workers.empty?
+        say "Missing workers: #{missing_workers.join(', ')}" if missing_workers
+        say "Did you `require` properly?"
+        return
+      end
+
+      if workers.empty?
+        say <<-EOF
+        Error: No workers found.
+        Please require your worker classes before specifying in CLI
+
+          $ sneakers run FooWorker
+                         ^- require this in your code
+
+        EOF
+        return
+      end
+
+      opts = {
+        :daemonize => !options[:front],
+        :log => options[:front] ? STDOUT : Sneakers::Config[:log]
+      }
+
+      Sneakers.configure(opts)
+      r = Sneakers::Runner.new(workers)
+
+      pid = Sneakers::Config[:pid_path]
+
+      say SNEAKERS
+      say "Workers ....: #{em workers.join(', ')}"
+      say "Log ........: #{em (Sneakers::Config[:log] == STDOUT ? 'Console' : Sneakers::Config[:log]) }"
+      say "PID ........: #{em pid}"
+      say ""
+      say (" "*31)+"Process control"
+      say "="*80
+      say "Stop (nicely) ..............: kill -SIGTERM `cat #{pid}`"
+      say "Stop (immediate) ...........: kill -SIGQUIT `cat #{pid}`"
+      say "Restart (nicely) ...........: kill -SIGUSR1 `cat #{pid}`"
+      say "Restart (immediate) ........: kill -SIGHUP `cat #{pid}`"
+      say "Reconfigure ................: kill -SIGUSR2 `cat #{pid}`"
+      say "Scale workers ..............: reconfigure, then restart"
+      say "="*80
+      say ""
+
+      if options[:debug]
+        say "==== configuration ==="
+        say Sneakers::Config.inspect
+        say "======================"
+      end
+
+      r.run
+    end
+
+
+  private
+    def require_boot(file)
+      load file
+    end
+
+    def em(text)
+      shell.set_color(text, nil, true)
+    end
+
+    def ok(detail=nil)
+      text = detail ? "OK, #{detail}." : "OK."
+      say text, :green
+    end
+
+    def error(detail)
+      say detail, :red
+    end
+  end
+end

data/lib/sneakers/concerns/logging.rb

@@ -0,0 +1,34 @@
+module Sneakers
+  module Concerns
+    module Logging
+      def self.included(base)
+        base.extend ClassMethods
+        base.send :define_method, :logger do
+          base.logger
+        end
+      end
+
+      module ClassMethods
+        def logger
+          @logger
+        end
+
+        def logger=(logger)
+          @logger = logger
+        end
+
+        def configure_logger(log=nil)
+          if log
+            @logger = log
+          else
+            @logger = Logger.new(STDOUT)
+            @logger.level = Logger::INFO
+            @logger.formatter = ProductionFormatter
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/sneakers/concerns/metrics.rb

@@ -0,0 +1,34 @@
+require 'sneakers/metrics/null_metrics'
+
+module Sneakers
+  module Concerns
+    module Metrics
+      def self.included(base)
+        base.extend ClassMethods
+        base.send :define_method, :metrics do
+          base.metrics
+        end
+      end
+
+      module ClassMethods
+        def metrics
+          @metrics
+        end
+
+        def metrics=(metrics)
+          @metrics = metrics
+        end
+
+        def configure_metrics(metrics=nil)
+          if metrics
+            @metrics = metrics
+          else
+            @metrics = Sneakers::Metrics::NullMetrics.new
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/sneakers/handlers/oneshot.rb

@@ -0,0 +1,25 @@
+module Sneakers
+  module Handlers
+    class Oneshot
+      def initialize(channel)
+        @channel = channel
+      end
+
+      def acknowledge(tag)
+        @channel.acknowledge(tag, false)
+      end
+
+      def reject(tag)
+        @channel.reject(tag, false)
+      end
+
+      def error(tag, err)
+        reject(tag)
+      end
+
+      def timeout(tag)
+        reject(tag)
+      end
+    end
+  end
+end

data/lib/sneakers/metrics/logging_metrics.rb

@@ -0,0 +1,16 @@
+module Sneakers
+  module Metrics
+    class LoggingMetrics
+      def increment(metric)
+        Sneakers.logger.info("INC: #{metric}")
+      end
+
+      def timing(metric, &block)
+        start = Time.now
+        block.call
+        Sneakers.logger.info("TIME: #{metric} #{Time.now - start}")
+      end
+    end
+  end
+end
+

data/lib/sneakers/metrics/null_metrics.rb

@@ -0,0 +1,13 @@
+module Sneakers
+  module Metrics
+    class NullMetrics
+      def increment(metric)
+      end
+
+      def timing(metric, &block)
+        block.call
+      end
+    end
+  end
+end
+

data/lib/sneakers/metrics/statsd_metrics.rb

@@ -0,0 +1,21 @@
+module Sneakers
+  module Metrics
+    class StatsdMetrics
+      def initialize(conn)
+        @connection = conn
+      end
+
+      def increment(metric)
+        @connection.incrememnt(metric)
+      end
+
+      def timing(metric, &block)
+        start = Time.now
+        block.call
+        @connection.timing(metric, ((Time.now - start)*1000).floor)
+      end
+
+    end
+  end
+end
+

data/lib/sneakers/publisher.rb

@@ -0,0 +1,35 @@
+require 'sneakers/support/queue_name'
+
+module Sneakers
+  class Publisher
+    attr_accessor :exchange
+
+    def initialize(opts={})
+      @mutex = Mutex.new
+      @opts = Sneakers::Config.merge(opts)
+    end
+
+    def publish(msg, routing)
+      @mutex.synchronize do
+        ensure_connection! unless connected?
+      end
+      Sneakers.logger.info("publishing <#{msg}> to [#{Support::QueueName.new(routing[:to_queue], @opts).to_s}]")
+      @exchange.publish(msg, :routing_key => Support::QueueName.new(routing[:to_queue], @opts).to_s)
+    end
+
+
+  private
+
+    def ensure_connection!
+      @bunny = Bunny.new(:heartbeat_interval => @opts[:heartbeat_interval])
+      @bunny.start
+      @channel = @bunny.create_channel
+      @exchange = @channel.exchange(@opts[:exchange], :type => :direct, :durable => @opts[:durable])
+    end
+
+    def connected?
+      @bunny && @bunny.connected?
+    end
+  end
+end
+

data/lib/sneakers/queue.rb

@@ -0,0 +1,42 @@
+
+class Sneakers::Queue
+  attr_reader :name, :opts, :exchange
+
+  def initialize(name, opts)
+    @name = name
+    @opts = opts
+    @handler_klass = Sneakers::Config[:handler]
+  end
+
+  #
+  # :exchange
+  # :heartbeat_interval
+  # :prefetch
+  # :durable
+  # :ack
+  #
+  def subscribe(worker)
+    @bunny = Bunny.new(:heartbeat_interval => @opts[:heartbeat_interval])
+    @bunny.start
+
+    @channel = @bunny.create_channel
+    @channel.prefetch(@opts[:prefetch])
+
+    @exchange = @channel.exchange(@opts[:exchange], :type => :direct, :durable => @opts[:durable])
+    handler = @handler_klass.new(@channel)
+
+    queue = @channel.queue(@name, :durable => @opts[:durable])
+    queue.bind(@exchange, :routing_key => @name)
+
+    @consumer = queue.subscribe(:block => false, :ack => @opts[:ack]) do | hdr, props, msg | 
+      worker.do_work(hdr, props, msg, handler)
+    end
+    nil
+  end
+
+  def unsubscribe
+    # XXX can we cancel bunny and channel too?
+    @consumer.cancel if @consumer
+    @consumer = nil
+  end
+end