kicks 3.0.0.pre

data/lib/sneakers/tasks.rb

@@ -0,0 +1,66 @@
+require 'sneakers'
+require 'sneakers/runner'
+
+task :environment
+
+namespace :sneakers do
+  desc "Start work (set $WORKERS=Klass1,Klass2)"
+  task :run do
+    Sneakers.server = true
+    Rake::Task['environment'].invoke
+
+    if defined?(::Rails)
+      if Rails.autoloaders.zeitwerk_enabled?
+        ::Zeitwerk::Loader.eager_load_all
+      else
+        ::Rails.application.eager_load!
+      end
+    end
+
+    workers, missing_workers = get_worker_classes
+
+    unless missing_workers.nil? || missing_workers.empty?
+      puts "Missing workers: #{missing_workers.join(', ')}" if missing_workers
+      puts "Did you `require` properly?"
+      exit(1)
+    end
+
+    if workers.empty?
+      puts <<EOF
+Error: No workers found.
+Please set the classes of the workers you want to run like so:
+
+  $ export WORKERS=MyWorker,FooWorker
+  $ rake sneakers:run
+
+You can also configure them with
+  $ Sneakers.rake_worker_classes
+
+If you use something that responds to :call it will execute that
+
+Eventually, if nothing before applied, every class is used where you directly included the Sneakers::Worker
+EOF
+      exit(1)
+    end
+    opts = (!ENV['WORKER_COUNT'].nil? ? {:workers => ENV['WORKER_COUNT'].to_i} : {})
+    r = Sneakers::Runner.new(workers, opts)
+
+    r.run
+  end
+
+  private
+
+  def get_worker_classes
+    if ENV["WORKERS"]
+      Sneakers::Utils.parse_workers(ENV['WORKERS'])
+    elsif Sneakers.rake_worker_classes
+      if Sneakers.rake_worker_classes.respond_to?(:call)
+        [Sneakers.rake_worker_classes.call]
+      else
+        [Sneakers.rake_worker_classes]
+      end
+    else
+      [Sneakers::Worker::Classes]
+    end || [[]]
+  end
+end

data/lib/sneakers/version.rb

@@ -0,0 +1,3 @@
+module Sneakers
+  VERSION = "3.0.0.pre"
+end

data/lib/sneakers/worker.rb

@@ -0,0 +1,162 @@
+require 'sneakers/queue'
+require 'sneakers/support/utils'
+
+module Sneakers
+  module Worker
+    attr_reader :queue, :id, :opts
+
+    # For now, a worker is hardly dependant on these concerns
+    # (because it uses methods from them directly.)
+    include Concerns::Logging
+    include Concerns::Metrics
+    include Sneakers::ErrorReporter
+
+    def initialize(queue = nil, pool = nil, opts = {})
+      opts = opts.merge(self.class.queue_opts || {})
+      queue_name = self.class.queue_name
+      opts = Sneakers::CONFIG.merge(opts)
+
+      @should_ack =  opts[:ack]
+      @pool = pool || Concurrent::FixedThreadPool.new(opts[:threads] || Sneakers::Configuration::DEFAULTS[:threads])
+      @call_with_params = respond_to?(:work_with_params)
+      @content_type = opts[:content_type]
+      @content_encoding = opts[:content_encoding]
+
+      @queue = queue || Sneakers::Queue.new(
+        queue_name,
+        opts
+      )
+
+      @opts = opts
+      @id = Utils.make_worker_id(queue_name)
+    end
+
+    def ack!; :ack end
+    def reject!; :reject; end
+    def requeue!; :requeue; end
+
+    def publish(msg, opts)
+      to_queue = opts.delete(:to_queue)
+      opts[:routing_key] ||= to_queue
+      return unless opts[:routing_key]
+      serialized_msg = Sneakers::ContentType.serialize(msg, opts[:content_type])
+      encoded_msg = Sneakers::ContentEncoding.encode(serialized_msg, opts[:content_encoding])
+      @queue.exchange.publish(encoded_msg, **opts)
+    end
+
+    def do_work(delivery_info, metadata, msg, handler)
+      worker_trace "Working off: #{msg.inspect}"
+
+      @pool.post do
+        process_work(delivery_info, metadata, msg, handler)
+      end
+    end
+
+    def process_work(delivery_info, metadata, msg, handler)
+      res = nil
+      error = nil
+
+      begin
+        metrics.increment("work.#{self.class.name}.started")
+        metrics.timing("work.#{self.class.name}.time") do
+          decoded_msg = ContentEncoding.decode(msg, @content_encoding || metadata && metadata[:content_encoding])
+          deserialized_msg = ContentType.deserialize(decoded_msg, @content_type || metadata && metadata[:content_type])
+
+          app = -> (deserialized_msg, delivery_info, metadata, handler) do
+            if @call_with_params
+              work_with_params(deserialized_msg, delivery_info, metadata)
+            else
+              work(deserialized_msg)
+            end
+          end
+
+          middlewares = Sneakers.middleware.to_a
+          block_to_call = middlewares.reverse.reduce(app) do |mem, h|
+            h[:class].new(mem, *h[:args])
+          end
+          res = block_to_call.call(deserialized_msg, delivery_info, metadata, handler)
+        end
+      rescue SignalException, SystemExit
+        # ServerEngine handles these exceptions, so they are not expected to be raised within the worker.
+        # Nevertheless, they are listed here to ensure that they are not caught by the rescue block below.
+        raise
+      rescue Exception => ex
+        res = :error
+        error = ex
+        worker_error(ex, log_msg: log_msg(msg), class: self.class.name,
+                     message: msg, delivery_info: delivery_info, metadata: metadata)
+      ensure
+        if @should_ack
+          case res
+          # note to future-self. never acknowledge multiple (multiple=true) messages under threads.
+          when :ack then handler.acknowledge(delivery_info, metadata, msg)
+          when :error then handler.error(delivery_info, metadata, msg, error)
+          when :reject then handler.reject(delivery_info, metadata, msg)
+          when :requeue then handler.reject(delivery_info, metadata, msg, true)
+          else
+            handler.noop(delivery_info, metadata, msg)
+          end
+          metrics.increment("work.#{self.class.name}.handled.#{res || 'noop'}")
+        end
+
+        metrics.increment("work.#{self.class.name}.ended")
+      end
+    end
+
+    def stop
+      worker_trace "Stopping worker: unsubscribing."
+      @queue.unsubscribe
+      worker_trace "Stopping worker: shutting down thread pool."
+      @pool.shutdown
+      @pool.wait_for_termination
+      worker_trace "Stopping worker: I'm gone."
+    end
+
+    def run
+      worker_trace "New worker: subscribing."
+      @queue.subscribe(self)
+      worker_trace "New worker: I'm alive."
+    end
+
+    # Construct a log message with some standard prefix for this worker
+    def log_msg(msg)
+      "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
+    end
+
+    def worker_trace(msg)
+      logger.debug(log_msg(msg))
+    end
+
+    Classes = []
+
+    def self.included(base)
+      base.extend ClassMethods
+      Classes << base if base.is_a? Class
+    end
+
+    module ClassMethods
+      attr_reader :queue_opts
+      attr_reader :queue_name
+
+      def from_queue(q, opts={})
+        @queue_name = q.to_s
+        @queue_opts = opts
+      end
+
+      def enqueue(msg, opts={})
+        opts[:routing_key] ||= @queue_opts[:routing_key]
+        opts[:content_type] ||= @queue_opts[:content_type]
+        opts[:content_encoding] ||= @queue_opts[:content_encoding]
+        opts[:to_queue] ||= @queue_name
+
+        publisher.publish(msg, opts)
+      end
+
+      private
+
+      def publisher
+        @publisher ||= Sneakers::Publisher.new(queue_opts)
+      end
+    end
+  end
+end

data/lib/sneakers/workergroup.rb

@@ -0,0 +1,60 @@
+module Sneakers
+  module WorkerGroup
+    @workers = []
+
+    def initialize
+      @stop_flag = ServerEngine::BlockingFlag.new
+    end
+
+    def before_fork
+      fbefore = Sneakers::CONFIG[:hooks][:before_fork]
+      fbefore.call if fbefore
+    end
+
+    def after_fork # note! this is not Serverengine#after_start, this is ours!
+      fafter = Sneakers::CONFIG[:hooks][:after_fork]
+      fafter.call if fafter
+    end
+
+    def run
+      after_fork
+
+      # Allocate single thread pool if share_threads is set. This improves load balancing
+      # when used with many workers.
+      pool = config[:share_threads] ? Concurrent::FixedThreadPool.new(config[:threads]) : nil
+
+      worker_classes = config[:worker_classes]
+
+      if worker_classes.respond_to? :call
+        worker_classes = worker_classes.call
+      end
+
+      # If we don't provide a connection to a worker,
+      # the queue used in the worker will create a new one
+
+      @workers = worker_classes.map do |worker_class|
+        worker_class.new(nil, pool, { connection: config[:connection] })
+      end
+
+      # if more than one worker this should be per worker
+      # accumulate clients and consumers as well
+      @workers.each do |worker|
+        worker.run
+      end
+      # end per worker
+      #
+      until @stop_flag.wait_for_set(Sneakers::CONFIG[:amqp_heartbeat])
+        Sneakers.logger.debug("Heartbeat: running threads [#{Thread.list.count}]")
+        # report aggregated stats?
+      end
+    end
+
+    def stop
+      Sneakers.logger.info("Shutting down workers")
+      @workers.each do |worker|
+        worker.stop
+      end
+      @stop_flag.set!
+    end
+  end
+end

data/lib/sneakers.rb

@@ -0,0 +1,125 @@
+require 'sneakers/version'
+require 'concurrent/executors'
+require 'bunny'
+require 'logger'
+require 'serverengine'
+
+module Sneakers
+  module Handlers
+  end
+  module Concerns
+  end
+end
+
+require 'sneakers/configuration'
+require 'sneakers/errors'
+require 'sneakers/support/production_formatter'
+require 'sneakers/concerns/logging'
+require 'sneakers/concerns/metrics'
+require 'sneakers/handlers/oneshot'
+require 'sneakers/content_type'
+require 'sneakers/content_encoding'
+require 'sneakers/middleware/config'
+require 'sneakers/worker'
+require 'sneakers/publisher'
+
+module Sneakers
+  extend self
+
+  CONFIG = Configuration.new
+
+  def configure(opts={})
+    # worker > userland > defaults
+    CONFIG.merge!(opts)
+    setup_general_logger!
+    setup_worker_concerns!
+    setup_general_publisher!
+    @configured = true
+  end
+
+  def clear!
+    CONFIG.clear
+    @logger = nil
+    @publisher = nil
+    @configured = false
+  end
+
+  def daemonize!(loglevel=Logger::INFO)
+    CONFIG[:log] = 'sneakers.log'
+    CONFIG[:daemonize] = true
+    setup_general_logger!
+    logger.level = loglevel
+  end
+
+  def rake_worker_classes=(worker_classes)
+    @rake_worker_classes = worker_classes
+  end
+
+  def rake_worker_classes
+    @rake_worker_classes
+  end
+
+  def logger=(logger)
+    @logger = logger
+  end
+
+  def logger
+    @logger
+  end
+
+  def publish(msg, routing)
+    @publisher.publish(msg, routing)
+  end
+
+  def configured?
+    @configured
+  end
+
+  def server=(server)
+    @server = server
+  end
+
+  def server?
+    @server
+  end
+
+  def configure_server
+    yield self if server?
+  end
+
+  # Register a proc to handle any error which occurs within the Sneakers process.
+  #
+  #   Sneakers.error_reporters << proc { |exception, worker, context_hash| MyErrorService.notify(exception, context_hash) }
+  #
+  # The default error handler logs errors to Sneakers.logger.
+  # Ripped off from https://github.com/mperham/sidekiq/blob/6ad6a3aa330deebd76c6cf0d353f66abd3bef93b/lib/sidekiq.rb#L165-L174
+  def error_reporters
+    CONFIG[:error_reporters]
+  end
+
+  def middleware
+    @middleware ||= Sneakers::Middleware::Config
+  end
+
+  private
+
+  def setup_general_logger!
+    if [:info, :debug, :error, :warn].all?{ |meth| CONFIG[:log].respond_to?(meth) }
+      @logger = CONFIG[:log]
+    else
+      @logger = ServerEngine::DaemonLogger.new(CONFIG[:log])
+      @logger.formatter = Sneakers::Support::ProductionFormatter
+    end
+  end
+
+  def setup_worker_concerns!
+    Worker.configure_logger(Sneakers::logger)
+    Worker.configure_metrics(CONFIG[:metrics])
+    CONFIG[:handler] ||= Sneakers::Handlers::Oneshot
+  end
+
+  def setup_general_publisher!
+    @publisher = Sneakers::Publisher.new
+  end
+end
+

data/scripts/local_integration

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker-compose run -e INTEGRATION=1 -e INTEGRATION_LOG=1 sneakers rake test

data/scripts/local_worker

@@ -0,0 +1,3 @@
+#!/bin/sh
+docker-compose run sneakers sneakers work TitleScraper --require examples/title_scraper.rb
+

data/spec/fixtures/integration_worker.rb

@@ -0,0 +1,18 @@
+require 'sneakers'
+require 'thread'
+require 'redis'
+
+
+redis_addr = compose_or_localhost("redis")
+puts "REDIS is at #{redis_addr}"
+$redis = Redis.new(:host => redis_addr)
+
+class IntegrationWorker
+  include Sneakers::Worker
+  
+  def work(msg)
+    $redis.incr(self.class.queue_name)
+    ack!
+  end
+end
+

data/spec/fixtures/require_worker.rb

@@ -0,0 +1,23 @@
+require 'sneakers'
+require 'open-uri'
+
+
+class TitleScraper
+  include Sneakers::Worker
+
+  from_queue 'downloads'
+
+  def work(msg)
+    title = extract_title(open(msg))
+    logger.info "FOUND <#{title}>"
+    ack!
+  end
+
+  private
+
+  def extract_title(html)
+    html =~ /<title>(.*?)<\/title>/
+    $1
+  end
+end
+

data/spec/gzip_helper.rb

@@ -0,0 +1,15 @@
+require 'stringio'
+require 'zlib'
+
+# Simple gzip encoder/decoder for testing
+def gzip_compress(s)
+  io = StringIO.new('w')
+  w = Zlib::GzipWriter.new(io)
+  w.write(s)
+  w.close
+  io.string
+end
+
+def gzip_decompress(s)
+  Zlib::GzipReader.new(StringIO.new(s, 'rb')).read
+end

data/spec/sneakers/cli_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+require 'sneakers'
+require 'sneakers/cli'
+require 'sneakers/runner'
+
+describe Sneakers::CLI do
+  describe "#work" do
+    before do
+      any_instance_of(Sneakers::Runner) do |runner|
+        stub(runner).run{ true }
+      end
+    end
+
+    after do
+      # require cleanup
+      Object.send(:remove_const, :TitleScraper) if Object.constants.include?(:TitleScraper)
+    end
+
+    describe 'with dirty class loading' do
+      it "should perform a run" do
+        any_instance_of(Sneakers::Runner) do |runner|
+          mock(runner).run{ true }
+        end
+        out = capture_io{ Sneakers::CLI.start [
+          'work',
+          "TitleScraper",
+          "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
+        ]}.join ''
+
+        _(out).must_match(/Workers.*:.*TitleScraper.*/)
+
+      end
+
+      it "should be able to run as front-running process" do
+        out = capture_io{ Sneakers::CLI.start [
+          'work',
+          "TitleScraper",
+          "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
+        ]}.join ''
+
+        _(out).must_match(/Log.*Console/)
+      end
+
+      it "should be able to run as daemonized process" do
+        out = capture_io{ Sneakers::CLI.start [
+          'work',
+          "TitleScraper",
+          "--daemonize",
+          "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
+        ]}.join ''
+
+        _(out).must_match(/sneakers.log/)
+      end
+    end
+
+    it "should fail when no workers found" do
+      out = capture_io{ Sneakers::CLI.start ['work', 'TitleScraper'] }.join ''
+      _(out).must_match(/Missing workers: TitleScraper/)
+    end
+
+    it "should run all workers when run without specifying any" do
+      out = capture_io{ Sneakers::CLI.start [
+        "work",
+        "--require=#{File.expand_path("../fixtures/require_worker.rb", File.dirname(__FILE__))}"
+      ]}.join ''
+
+      _(out).must_match(/Workers.*:.*TitleScraper.*/)
+    end
+
+    after do
+      # require cleanup
+      Object.send(:remove_const, :TitleScraper) if Object.constants.include?(:TitleScraper)
+    end
+  end
+end

data/spec/sneakers/concerns/logging_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+require 'sneakers'
+require 'logger'
+
+
+class Foobar
+  include Sneakers::Concerns::Logging
+end
+
+describe Sneakers::Concerns::Logging do
+  describe ".configure" do
+    before do
+      Foobar.logger = nil
+    end
+
+    it "should configure a default logger when included" do
+      _(Foobar.logger).must_be_nil
+      Foobar.configure_logger
+      _(Foobar.logger).wont_be_nil
+      _(Foobar.logger.formatter).must_equal Sneakers::Support::ProductionFormatter
+    end
+
+    it "should supply accessible instance logger" do
+      _(Foobar.logger).must_be_nil
+      Foobar.configure_logger
+      f = Foobar.new
+      _(f.logger).must_equal Foobar.logger
+      _(f.logger).wont_be_nil
+    end
+
+    it "should configure a given logger when specified" do
+      _(Foobar.logger).must_be_nil
+      log = Logger.new(STDOUT)
+      Foobar.configure_logger(log)
+      _(Foobar.logger).must_equal log
+    end
+  end
+end
+

data/spec/sneakers/concerns/metrics_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+require 'sneakers'
+require 'logger'
+
+
+class Foometrics
+  include Sneakers::Concerns::Metrics
+end
+
+describe Sneakers::Concerns::Metrics do
+  describe ".configure" do
+    before do
+      Foometrics.metrics = nil
+    end
+
+    it "should configure a default logger when included" do
+      _(Foometrics.metrics).must_be_nil
+      Foometrics.configure_metrics
+      _(Foometrics.metrics).wont_be_nil
+    end
+
+    it "should supply accessible instance logger" do
+      _(Foometrics.metrics).must_be_nil
+      Foometrics.configure_metrics
+      f = Foometrics.new
+      _(f.metrics).must_equal Foometrics.metrics
+      _(f.metrics).wont_be_nil
+    end
+
+    it "should configure a given metrics when specified" do
+      _(Foometrics.metrics).must_be_nil
+      o = Object.new
+      Foometrics.configure_metrics(o)
+      _(Foometrics.metrics).must_equal o
+    end
+  end
+end
+