queue_kit 0.0.6

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gem "rake"
+

data/README.md

@@ -0,0 +1,21 @@
+# QueueKit
+
+A set of classes for building a process that pops jobs off a queue.  Provides
+assistance with loops, signal handling, round robin client shuffling, etc.
+
+Sing this to the tune of "Push It" by Salt-n-Pepa
+
+Note: This is still considered alpha until QueueKit v0.1.0.
+
+## TODO?
+
+* Forked processes
+* Pausing/resuming workers
+* More nunes.
+
+## Shout outs
+
+* Resque
+* John Nunemaker
+* Eric Lindvall
+

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+

data/lib/queue_kit/clients/command_timeout.rb

@@ -0,0 +1,43 @@
+module QueueKit
+  module Clients
+    module CommandTimeout
+      def self.with_ivars(klass)
+        mod = self
+        klass.class_eval do
+          include mod
+          attr_accessor :command_timeout_ms
+          attr_accessor :max_command_timeout_ms
+        end
+      end
+
+      def command_timeout(attempts = 0)
+        timeout = command_timeout_ms
+        timeout += timeout * (attempts / command_clients_size).floor
+
+        if timeout > (max = max_command_timeout_ms)
+          timeout = max
+        end
+
+        timeout
+      end
+
+      def command_timeout_from(options)
+        @command_timeout_ms = options[:command_timeout_ms] || 10
+        @max_command_timeout_ms = options[:max_command_timeout_ms] || 1000
+      end
+
+      def command_timeout_ms
+        10
+      end
+
+      def max_command_timeout_ms
+        1000
+      end
+
+      def command_clients_size
+        1
+      end
+    end
+  end
+end
+

data/lib/queue_kit/clients/round_robin_shuffler.rb

@@ -0,0 +1,67 @@
+module QueueKit
+  module Clients
+    module RoundRobinShuffler
+      def self.with_ivars(klass)
+        mod = self
+        klass.class_eval do
+          include mod
+          attr_accessor :commands_per_client
+
+          def command_clients_size
+            @clients.size
+          end
+        end
+      end
+
+      def client_command_with_retries(retries = 10)
+        attempts = 0
+
+        while attempts < retries
+          if data = (yield client, attempts)
+            return data
+          end
+
+          rotate_client
+          attempts += 1
+        end
+
+        nil
+      end
+
+      def client
+        @client_command_count += 1
+
+        if @client_command_count > commands_per_client
+          rotate_client
+        end
+
+        clients[@client_index]
+      end
+
+      def round_robin_from(options)
+        @commands_per_client = options[:commands_per_client] || 100
+      end
+
+      def rotate_client
+        @client_index ||= -1
+        @client_len ||= clients.size
+
+        @client_command_count = 0
+        @client_index += 1
+
+        if @client_index >= @client_len
+          @client_index = 0
+        end
+      end
+
+      def commands_per_client
+        100
+      end
+
+      def clients
+        []
+      end
+    end
+  end
+end
+

data/lib/queue_kit/signal_checker.rb

@@ -0,0 +1,50 @@
+module QueueKit
+  class SignalChecker
+    COMMON_SIGNALS = %w(TERM INT)
+    JRUBY_SIGNALS = %w(QUIT USR1)
+    OPTIONAL_SIGNALS = %w(USR2 CONT HUP)
+
+    attr_reader :worker
+    attr_reader :handler
+
+    def self.trap(worker, handler)
+      new(worker, handler).trap_signals
+    end
+
+    def initialize(worker, handler)
+      @worker = worker
+      @handler = handler
+    end
+
+    def trap_signals(signals = nil)
+      if signals.nil?
+        trap_signals(COMMON_SIGNALS)
+        trap_signals(JRUBY_SIGNALS) unless defined?(JRUBY_VERSION)
+        trap_signals(OPTIONAL_SIGNALS)
+      else
+        signals.each { |sig| trap_signal(sig) }
+      end
+
+    rescue ArgumentError
+      warn "Signals are not supported: #{signals.inspect}"
+    end
+
+    def trap_signal(sig)
+      trap_method = "trap_#{sig}"
+      return unless @handler.respond_to?(trap_method)
+
+      debug :setup, sig
+
+      old_handler = trap sig do
+        debug :trap, sig
+        @handler.send(trap_method, @worker)
+        old_handler.call if old_handler.respond_to?(:call)
+      end
+    end
+
+    def debug(key, sig)
+      @worker.debug { ["signals.#{key}", {:signal => sig}] }
+    end
+  end
+end
+

data/lib/queue_kit/signal_handlers/graceful_quit.rb

@@ -0,0 +1,14 @@
+module QueueKit
+  require_lib 'signal_checker'
+
+  module GracefulQuit
+    extend self
+
+    def trap_TERM(worker)
+      worker.stop
+    end
+
+    alias trap_INT trap_TERM
+  end
+end
+

data/lib/queue_kit/worker.rb

@@ -0,0 +1,115 @@
+module QueueKit
+  module Worker
+    def initialize(queue, options = {})
+      @queue = queue
+      @processor = options.fetch(:processor) { method(:process) }
+      @cooler = options.fetch(:cooler) { method(:cool) }
+      @error_handler = options.fetch(:error_handler) { method(:handle_error) }
+      @instrumenter = options.fetch(:instrumenter) { PutsInstrumenter.new }
+      @stopped = true
+
+      if options.fetch(:debug) { false }
+        class << self
+          alias debug force_debug
+        end
+      end
+    end
+
+    def process(item)
+      raise NotImplementedError, "This worker can't do anything with #{item.inspect}"
+    end
+
+    def cool
+    end
+
+    def handle_error(err)
+      raise err
+    end
+
+    def trap(signal_handler)
+      SignalChecker.trap(self, signal_handler)
+    end
+
+    def run
+      start
+      interval_debugger = lambda { "worker.interval" }
+
+      loop do
+        working? ? work : break
+        debug(&interval_debugger)
+      end
+    end
+
+    def procline(string)
+      $0 = "QueueKit-#{QueueKit::VERSION}: #{string}"
+      debug { ["worker.procline", {:message => string}] }
+    end
+
+    def work
+      wrap_error { work! }
+    end
+
+    def work!
+      if item = @queue.pop
+        set_working_procline
+        @processor.call(item)
+        set_popping_procline
+      else
+        @cooler.call
+      end
+    end
+
+    def wrap_error
+      yield
+    rescue Exception => exception
+      @error_handler.call(exception)
+    end
+
+    def name
+      @name ||= "#{self.class} #{Socket.gethostname}:#{Process.pid}"
+    end
+
+    def start
+      instrument "worker.start"
+      set_popping_procline
+      @stopped = false
+    end
+
+    def stop
+      instrument "worker.stop"
+      @stopped = true
+    end
+
+    def working?
+      !@stopped
+    end
+
+    def instrument(name, payload = nil)
+      (payload ||= {}).update(:worker => self)
+      @instrumenter.instrument("queuekit.#{name}", payload)
+    end
+
+    def set_working_procline
+      procline("Processing since #{Time.now.to_i}")
+    end
+
+    def set_popping_procline
+      @last_job_at = Time.now
+      procline("Waiting since #{@last_job_at.to_i}")
+    end
+
+    def force_debug
+      instrument(*yield)
+    end
+
+    def debug
+    end
+
+    class PutsInstrumenter
+      def instrument(name, payload = nil)
+        puts "[#{Time.now}] #{name}: #{payload.inspect}"
+      end
+    end
+  end
+end
+

data/lib/queue_kit.rb

@@ -0,0 +1,17 @@
+module QueueKit
+  VERSION = "0.0.6"
+  ROOT = File.expand_path("../queue_kit", __FILE__)
+
+  def self.require_lib(*libs)
+    libs.each do |lib|
+      require File.join(ROOT, lib.to_s)
+    end
+  end
+
+  class << self
+    alias require_libs require_lib
+  end
+
+  require_lib "worker"
+end
+

data/queue_kit.gemspec

@@ -0,0 +1,31 @@
+lib = "queue_kit"
+lib_file = File.expand_path("../lib/#{lib}.rb", __FILE__)
+File.read(lib_file) =~ /\bVERSION\s*=\s*["'](.+?)["']/
+version = $1
+
+Gem::Specification.new do |spec|
+  spec.specification_version = 2 if spec.respond_to? :specification_version=
+  spec.required_rubygems_version = '>= 1.3.6'
+
+  spec.name    = lib
+  spec.version = version
+
+  spec.summary = "Job Queue hobby kit"
+
+  spec.authors  = ["Rick Olson"]
+  spec.email    = 'technoweenie@gmail.com'
+  spec.homepage = 'https://github.com/technoweenie/queue_kit'
+  spec.licenses = ['MIT']
+
+  spec.files = %w(Gemfile README.md Rakefile)
+  spec.files << "#{lib}.gemspec"
+  spec.files += Dir.glob("lib/**/*.rb")
+  spec.files += Dir.glob("test/**/*.rb")
+  spec.files += Dir.glob("script/*")
+
+  dev_null    = File.exist?('/dev/null') ? '/dev/null' : 'NUL'
+  git_files   = `git ls-files -z 2>#{dev_null}`
+  spec.files &= git_files.split("\0") if $?.success?
+
+  spec.test_files = Dir.glob("test/**/*.rb")
+end

data/script/bootstrap

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Usage: script/bootstrap
+# Ensures all gems are installed locally.
+
+bundle install --binstubs
+

data/script/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Usage: script/console
+# Starts an IRB console with this library loaded.
+
+gemspec="$(ls *.gemspec | head -1)"
+exec bundle exec irb -r "./lib/${gemspec%.*}"
+

data/script/package

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Usage: script/gem
+# Updates the gemspec and builds a new gem in the pkg directory.
+
+mkdir -p pkg
+gem build *.gemspec
+mv *.gem pkg
+

data/script/release

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Usage: script/release
+# Build the package, tag a commit, push it to origin, and then release the
+# package publicly.
+
+set -e
+
+version="$(script/package | grep Version: | awk '{print $2}')"
+[ -n "$version" ] || exit 1
+
+git commit --allow-empty -a -m "Release $version"
+git tag "v$version"
+git push origin
+git push origin "v$version"
+git push legacy
+git push legacy "v$version"
+gem push pkg/*-${version}.gem
+

data/script/test

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Usage: script/test
+# Runs the library's test suite.
+
+bundle exec rake test
+

data/test/command_timeout_test.rb

@@ -0,0 +1,39 @@
+require File.expand_path("../helper", __FILE__)
+QueueKit.require_lib 'clients/command_timeout'
+
+class CommandTimeoutTest < Test::Unit::TestCase
+  include QueueKit::Clients::CommandTimeout
+
+  def test_with_ivars
+    object = FakeQueue.new
+    assert_nil object.command_timeout_ms
+    assert_nil object.max_command_timeout_ms
+
+    object.command_timeout_from({})
+    assert_equal 10, object.command_timeout_ms
+    assert_equal 1000, object.max_command_timeout_ms
+
+    object.command_timeout_from \
+      :command_timeout_ms => 1, :max_command_timeout_ms => 2
+    assert_equal 1, object.command_timeout_ms
+    assert_equal 2, object.max_command_timeout_ms
+  end
+
+  def test_gets_timeout_for_first_attempt
+    assert_equal 10, command_timeout(attempts=0)
+  end
+
+  def test_backs_off
+    assert_equal 20, command_timeout(attempts=1)
+    assert_equal 30, command_timeout(attempts=2)
+  end
+
+  def test_enforces_max_timeout
+    assert_equal 1000, command_timeout(attempts=1000)
+  end
+
+  class FakeQueue
+    QueueKit::Clients::CommandTimeout.with_ivars(self)
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,4 @@
+require 'bundler'
+require 'test/unit'
+require File.expand_path("../../lib/queue_kit", __FILE__)
+

data/test/round_robin_shuffler_test.rb

@@ -0,0 +1,79 @@
+require File.expand_path("../helper", __FILE__)
+QueueKit.require_lib 'clients/round_robin_shuffler'
+
+class RoundRobinShufflerTest < Test::Unit::TestCase
+  include QueueKit::Clients::RoundRobinShuffler
+
+  attr_reader :clients
+
+  def test_client_command_with_retries
+    clients = []
+
+    set_clients 1, 2
+
+    value = client_command_with_retries 3 do |client|
+      clients << client
+      nil
+    end
+
+    assert_equal [1, 2, 1], clients
+    assert_nil value
+  end
+
+  def test_client_command_with_value
+    clients = []
+
+    set_clients 1, 2
+
+    value = client_command_with_retries 3 do |client, attempts|
+      assert_equal clients.size, attempts
+      clients << client
+      client == 2 ? :booya : nil
+    end
+
+    assert_equal [1, 2], clients
+    assert_equal :booya, value
+  end
+
+  def test_with_ivars
+    object = FakeQueue.new
+    assert_nil object.commands_per_client
+
+    object.round_robin_from({})
+    assert_equal 100, object.commands_per_client
+
+    object.round_robin_from :commands_per_client => 1
+    assert_equal 1, object.commands_per_client
+  end
+
+  def test_shuffles_solitary_client
+    set_clients 1
+
+    assert_equal 1, client
+    assert_equal 1, client
+    assert_equal 1, client
+  end
+
+  def test_shuffles_clients
+    set_clients 1, 2
+
+    assert_equal 1, client
+    assert_equal 1, client
+    assert_equal 2, client
+  end
+
+  def commands_per_client
+    2
+  end
+
+  def set_clients(*clients)
+    @client_index = @client_len = nil
+    @clients = clients
+    rotate_client
+  end
+
+  class FakeQueue
+    QueueKit::Clients::RoundRobinShuffler.with_ivars(self)
+  end
+end
+

data/test/worker_test.rb

@@ -0,0 +1,81 @@
+require File.expand_path("../helper", __FILE__)
+
+class WorkerTest < Test::Unit::TestCase
+  def test_cooler
+    cooled = false
+    cooler = lambda { cooled = true }
+
+    worker = new_worker [], :processor => lambda { |_| fail 'item found?' },
+      :cooler => cooler
+
+    worker.work
+    assert cooled
+  end
+
+  def test_custom_on_error_handler
+    called = false
+    error_handler = lambda do |exc|
+      called = true
+      assert_equal 'booya', exc.message
+    end
+
+    worker = new_worker [1], :processor => lambda { |_| raise 'booya' },
+      :error_handler => error_handler
+
+    worker.work
+
+    assert called
+  end
+
+  def test_default_error_handler
+    processor = lambda do |item|
+      raise item.to_s
+    end
+
+    worker = new_worker [1], :processor => processor
+
+    begin
+      worker.work
+    rescue RuntimeError => err
+      assert_equal '1', err.message
+    else
+      fail "no exception raised"
+    end
+  end
+
+  def test_breaks_when_stopped
+    called = false
+    worker = nil
+
+    processor = lambda do |item|
+      fail "callback called multiple times" if called
+      assert_equal 1, item
+      called = true
+      worker.stop
+    end
+
+    worker = new_worker [1, nil], :processor => processor
+    worker.run
+
+    assert called
+  end
+
+  def test_new_worker_is_not_working
+    assert !new_worker.working?
+  end
+
+  def new_worker(queue = [], options = {})
+    options[:instrumenter] ||= NullInstrumenter.new
+    Worker.new(queue, options)
+  end
+
+  class Worker
+    include QueueKit::Worker
+  end
+
+  class NullInstrumenter
+    def instrument(name, payload = nil)
+    end
+  end
+end
+

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: queue_kit
+version: !ruby/object:Gem::Version
+  version: 0.0.6
+  prerelease: 
+platform: ruby
+authors:
+- Rick Olson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-19 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: technoweenie@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- queue_kit.gemspec
+- lib/queue_kit/clients/command_timeout.rb
+- lib/queue_kit/clients/round_robin_shuffler.rb
+- lib/queue_kit/signal_checker.rb
+- lib/queue_kit/signal_handlers/graceful_quit.rb
+- lib/queue_kit/worker.rb
+- lib/queue_kit.rb
+- test/command_timeout_test.rb
+- test/helper.rb
+- test/round_robin_shuffler_test.rb
+- test/worker_test.rb
+- script/bootstrap
+- script/console
+- script/package
+- script/release
+- script/test
+homepage: https://github.com/technoweenie/queue_kit
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 2
+summary: Job Queue hobby kit
+test_files:
+- test/command_timeout_test.rb
+- test/helper.rb
+- test/round_robin_shuffler_test.rb
+- test/worker_test.rb