queue_kit 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+ gem "rake"
3
+
data/README.md ADDED
@@ -0,0 +1,21 @@
1
+ # QueueKit
2
+
3
+ A set of classes for building a process that pops jobs off a queue. Provides
4
+ assistance with loops, signal handling, round robin client shuffling, etc.
5
+
6
+ Sing this to the tune of "Push It" by Salt-n-Pepa
7
+
8
+ Note: This is still considered alpha until QueueKit v0.1.0.
9
+
10
+ ## TODO?
11
+
12
+ * Forked processes
13
+ * Pausing/resuming workers
14
+ * More nunes.
15
+
16
+ ## Shout outs
17
+
18
+ * Resque
19
+ * John Nunemaker
20
+ * Eric Lindvall
21
+
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require 'rubygems'
2
+ require 'bundler/setup'
3
+ require 'rake/testtask'
4
+
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/*_test.rb'
8
+ test.verbose = true
9
+ end
10
+
@@ -0,0 +1,43 @@
1
+ module QueueKit
2
+ module Clients
3
+ module CommandTimeout
4
+ def self.with_ivars(klass)
5
+ mod = self
6
+ klass.class_eval do
7
+ include mod
8
+ attr_accessor :command_timeout_ms
9
+ attr_accessor :max_command_timeout_ms
10
+ end
11
+ end
12
+
13
+ def command_timeout(attempts = 0)
14
+ timeout = command_timeout_ms
15
+ timeout += timeout * (attempts / command_clients_size).floor
16
+
17
+ if timeout > (max = max_command_timeout_ms)
18
+ timeout = max
19
+ end
20
+
21
+ timeout
22
+ end
23
+
24
+ def command_timeout_from(options)
25
+ @command_timeout_ms = options[:command_timeout_ms] || 10
26
+ @max_command_timeout_ms = options[:max_command_timeout_ms] || 1000
27
+ end
28
+
29
+ def command_timeout_ms
30
+ 10
31
+ end
32
+
33
+ def max_command_timeout_ms
34
+ 1000
35
+ end
36
+
37
+ def command_clients_size
38
+ 1
39
+ end
40
+ end
41
+ end
42
+ end
43
+
@@ -0,0 +1,67 @@
1
+ module QueueKit
2
+ module Clients
3
+ module RoundRobinShuffler
4
+ def self.with_ivars(klass)
5
+ mod = self
6
+ klass.class_eval do
7
+ include mod
8
+ attr_accessor :commands_per_client
9
+
10
+ def command_clients_size
11
+ @clients.size
12
+ end
13
+ end
14
+ end
15
+
16
+ def client_command_with_retries(retries = 10)
17
+ attempts = 0
18
+
19
+ while attempts < retries
20
+ if data = (yield client, attempts)
21
+ return data
22
+ end
23
+
24
+ rotate_client
25
+ attempts += 1
26
+ end
27
+
28
+ nil
29
+ end
30
+
31
+ def client
32
+ @client_command_count += 1
33
+
34
+ if @client_command_count > commands_per_client
35
+ rotate_client
36
+ end
37
+
38
+ clients[@client_index]
39
+ end
40
+
41
+ def round_robin_from(options)
42
+ @commands_per_client = options[:commands_per_client] || 100
43
+ end
44
+
45
+ def rotate_client
46
+ @client_index ||= -1
47
+ @client_len ||= clients.size
48
+
49
+ @client_command_count = 0
50
+ @client_index += 1
51
+
52
+ if @client_index >= @client_len
53
+ @client_index = 0
54
+ end
55
+ end
56
+
57
+ def commands_per_client
58
+ 100
59
+ end
60
+
61
+ def clients
62
+ []
63
+ end
64
+ end
65
+ end
66
+ end
67
+
@@ -0,0 +1,50 @@
1
+ module QueueKit
2
+ class SignalChecker
3
+ COMMON_SIGNALS = %w(TERM INT)
4
+ JRUBY_SIGNALS = %w(QUIT USR1)
5
+ OPTIONAL_SIGNALS = %w(USR2 CONT HUP)
6
+
7
+ attr_reader :worker
8
+ attr_reader :handler
9
+
10
+ def self.trap(worker, handler)
11
+ new(worker, handler).trap_signals
12
+ end
13
+
14
+ def initialize(worker, handler)
15
+ @worker = worker
16
+ @handler = handler
17
+ end
18
+
19
+ def trap_signals(signals = nil)
20
+ if signals.nil?
21
+ trap_signals(COMMON_SIGNALS)
22
+ trap_signals(JRUBY_SIGNALS) unless defined?(JRUBY_VERSION)
23
+ trap_signals(OPTIONAL_SIGNALS)
24
+ else
25
+ signals.each { |sig| trap_signal(sig) }
26
+ end
27
+
28
+ rescue ArgumentError
29
+ warn "Signals are not supported: #{signals.inspect}"
30
+ end
31
+
32
+ def trap_signal(sig)
33
+ trap_method = "trap_#{sig}"
34
+ return unless @handler.respond_to?(trap_method)
35
+
36
+ debug :setup, sig
37
+
38
+ old_handler = trap sig do
39
+ debug :trap, sig
40
+ @handler.send(trap_method, @worker)
41
+ old_handler.call if old_handler.respond_to?(:call)
42
+ end
43
+ end
44
+
45
+ def debug(key, sig)
46
+ @worker.debug { ["signals.#{key}", {:signal => sig}] }
47
+ end
48
+ end
49
+ end
50
+
@@ -0,0 +1,14 @@
1
+ module QueueKit
2
+ require_lib 'signal_checker'
3
+
4
+ module GracefulQuit
5
+ extend self
6
+
7
+ def trap_TERM(worker)
8
+ worker.stop
9
+ end
10
+
11
+ alias trap_INT trap_TERM
12
+ end
13
+ end
14
+
@@ -0,0 +1,115 @@
1
+ module QueueKit
2
+ module Worker
3
+ def initialize(queue, options = {})
4
+ @queue = queue
5
+ @processor = options.fetch(:processor) { method(:process) }
6
+ @cooler = options.fetch(:cooler) { method(:cool) }
7
+ @error_handler = options.fetch(:error_handler) { method(:handle_error) }
8
+ @instrumenter = options.fetch(:instrumenter) { PutsInstrumenter.new }
9
+ @stopped = true
10
+
11
+ if options.fetch(:debug) { false }
12
+ class << self
13
+ alias debug force_debug
14
+ end
15
+ end
16
+ end
17
+
18
+ def process(item)
19
+ raise NotImplementedError, "This worker can't do anything with #{item.inspect}"
20
+ end
21
+
22
+ def cool
23
+ end
24
+
25
+ def handle_error(err)
26
+ raise err
27
+ end
28
+
29
+ def trap(signal_handler)
30
+ SignalChecker.trap(self, signal_handler)
31
+ end
32
+
33
+ def run
34
+ start
35
+ interval_debugger = lambda { "worker.interval" }
36
+
37
+ loop do
38
+ working? ? work : break
39
+ debug(&interval_debugger)
40
+ end
41
+ end
42
+
43
+ def procline(string)
44
+ $0 = "QueueKit-#{QueueKit::VERSION}: #{string}"
45
+ debug { ["worker.procline", {:message => string}] }
46
+ end
47
+
48
+ def work
49
+ wrap_error { work! }
50
+ end
51
+
52
+ def work!
53
+ if item = @queue.pop
54
+ set_working_procline
55
+ @processor.call(item)
56
+ set_popping_procline
57
+ else
58
+ @cooler.call
59
+ end
60
+ end
61
+
62
+ def wrap_error
63
+ yield
64
+ rescue Exception => exception
65
+ @error_handler.call(exception)
66
+ end
67
+
68
+ def name
69
+ @name ||= "#{self.class} #{Socket.gethostname}:#{Process.pid}"
70
+ end
71
+
72
+ def start
73
+ instrument "worker.start"
74
+ set_popping_procline
75
+ @stopped = false
76
+ end
77
+
78
+ def stop
79
+ instrument "worker.stop"
80
+ @stopped = true
81
+ end
82
+
83
+ def working?
84
+ !@stopped
85
+ end
86
+
87
+ def instrument(name, payload = nil)
88
+ (payload ||= {}).update(:worker => self)
89
+ @instrumenter.instrument("queuekit.#{name}", payload)
90
+ end
91
+
92
+ def set_working_procline
93
+ procline("Processing since #{Time.now.to_i}")
94
+ end
95
+
96
+ def set_popping_procline
97
+ @last_job_at = Time.now
98
+ procline("Waiting since #{@last_job_at.to_i}")
99
+ end
100
+
101
+ def force_debug
102
+ instrument(*yield)
103
+ end
104
+
105
+ def debug
106
+ end
107
+
108
+ class PutsInstrumenter
109
+ def instrument(name, payload = nil)
110
+ puts "[#{Time.now}] #{name}: #{payload.inspect}"
111
+ end
112
+ end
113
+ end
114
+ end
115
+
data/lib/queue_kit.rb ADDED
@@ -0,0 +1,17 @@
1
+ module QueueKit
2
+ VERSION = "0.0.6"
3
+ ROOT = File.expand_path("../queue_kit", __FILE__)
4
+
5
+ def self.require_lib(*libs)
6
+ libs.each do |lib|
7
+ require File.join(ROOT, lib.to_s)
8
+ end
9
+ end
10
+
11
+ class << self
12
+ alias require_libs require_lib
13
+ end
14
+
15
+ require_lib "worker"
16
+ end
17
+
data/queue_kit.gemspec ADDED
@@ -0,0 +1,31 @@
1
+ lib = "queue_kit"
2
+ lib_file = File.expand_path("../lib/#{lib}.rb", __FILE__)
3
+ File.read(lib_file) =~ /\bVERSION\s*=\s*["'](.+?)["']/
4
+ version = $1
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.specification_version = 2 if spec.respond_to? :specification_version=
8
+ spec.required_rubygems_version = '>= 1.3.6'
9
+
10
+ spec.name = lib
11
+ spec.version = version
12
+
13
+ spec.summary = "Job Queue hobby kit"
14
+
15
+ spec.authors = ["Rick Olson"]
16
+ spec.email = 'technoweenie@gmail.com'
17
+ spec.homepage = 'https://github.com/technoweenie/queue_kit'
18
+ spec.licenses = ['MIT']
19
+
20
+ spec.files = %w(Gemfile README.md Rakefile)
21
+ spec.files << "#{lib}.gemspec"
22
+ spec.files += Dir.glob("lib/**/*.rb")
23
+ spec.files += Dir.glob("test/**/*.rb")
24
+ spec.files += Dir.glob("script/*")
25
+
26
+ dev_null = File.exist?('/dev/null') ? '/dev/null' : 'NUL'
27
+ git_files = `git ls-files -z 2>#{dev_null}`
28
+ spec.files &= git_files.split("\0") if $?.success?
29
+
30
+ spec.test_files = Dir.glob("test/**/*.rb")
31
+ end
data/script/bootstrap ADDED
@@ -0,0 +1,6 @@
1
+ #!/bin/sh
2
+ # Usage: script/bootstrap
3
+ # Ensures all gems are installed locally.
4
+
5
+ bundle install --binstubs
6
+
data/script/console ADDED
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env bash
2
+ # Usage: script/console
3
+ # Starts an IRB console with this library loaded.
4
+
5
+ gemspec="$(ls *.gemspec | head -1)"
6
+ exec bundle exec irb -r "./lib/${gemspec%.*}"
7
+
data/script/package ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ # Usage: script/gem
3
+ # Updates the gemspec and builds a new gem in the pkg directory.
4
+
5
+ mkdir -p pkg
6
+ gem build *.gemspec
7
+ mv *.gem pkg
8
+
data/script/release ADDED
@@ -0,0 +1,18 @@
1
+ #!/usr/bin/env bash
2
+ # Usage: script/release
3
+ # Build the package, tag a commit, push it to origin, and then release the
4
+ # package publicly.
5
+
6
+ set -e
7
+
8
+ version="$(script/package | grep Version: | awk '{print $2}')"
9
+ [ -n "$version" ] || exit 1
10
+
11
+ git commit --allow-empty -a -m "Release $version"
12
+ git tag "v$version"
13
+ git push origin
14
+ git push origin "v$version"
15
+ git push legacy
16
+ git push legacy "v$version"
17
+ gem push pkg/*-${version}.gem
18
+
data/script/test ADDED
@@ -0,0 +1,6 @@
1
+ #!/bin/sh
2
+ # Usage: script/test
3
+ # Runs the library's test suite.
4
+
5
+ bundle exec rake test
6
+
@@ -0,0 +1,39 @@
1
+ require File.expand_path("../helper", __FILE__)
2
+ QueueKit.require_lib 'clients/command_timeout'
3
+
4
+ class CommandTimeoutTest < Test::Unit::TestCase
5
+ include QueueKit::Clients::CommandTimeout
6
+
7
+ def test_with_ivars
8
+ object = FakeQueue.new
9
+ assert_nil object.command_timeout_ms
10
+ assert_nil object.max_command_timeout_ms
11
+
12
+ object.command_timeout_from({})
13
+ assert_equal 10, object.command_timeout_ms
14
+ assert_equal 1000, object.max_command_timeout_ms
15
+
16
+ object.command_timeout_from \
17
+ :command_timeout_ms => 1, :max_command_timeout_ms => 2
18
+ assert_equal 1, object.command_timeout_ms
19
+ assert_equal 2, object.max_command_timeout_ms
20
+ end
21
+
22
+ def test_gets_timeout_for_first_attempt
23
+ assert_equal 10, command_timeout(attempts=0)
24
+ end
25
+
26
+ def test_backs_off
27
+ assert_equal 20, command_timeout(attempts=1)
28
+ assert_equal 30, command_timeout(attempts=2)
29
+ end
30
+
31
+ def test_enforces_max_timeout
32
+ assert_equal 1000, command_timeout(attempts=1000)
33
+ end
34
+
35
+ class FakeQueue
36
+ QueueKit::Clients::CommandTimeout.with_ivars(self)
37
+ end
38
+ end
39
+
data/test/helper.rb ADDED
@@ -0,0 +1,4 @@
1
+ require 'bundler'
2
+ require 'test/unit'
3
+ require File.expand_path("../../lib/queue_kit", __FILE__)
4
+
@@ -0,0 +1,79 @@
1
+ require File.expand_path("../helper", __FILE__)
2
+ QueueKit.require_lib 'clients/round_robin_shuffler'
3
+
4
+ class RoundRobinShufflerTest < Test::Unit::TestCase
5
+ include QueueKit::Clients::RoundRobinShuffler
6
+
7
+ attr_reader :clients
8
+
9
+ def test_client_command_with_retries
10
+ clients = []
11
+
12
+ set_clients 1, 2
13
+
14
+ value = client_command_with_retries 3 do |client|
15
+ clients << client
16
+ nil
17
+ end
18
+
19
+ assert_equal [1, 2, 1], clients
20
+ assert_nil value
21
+ end
22
+
23
+ def test_client_command_with_value
24
+ clients = []
25
+
26
+ set_clients 1, 2
27
+
28
+ value = client_command_with_retries 3 do |client, attempts|
29
+ assert_equal clients.size, attempts
30
+ clients << client
31
+ client == 2 ? :booya : nil
32
+ end
33
+
34
+ assert_equal [1, 2], clients
35
+ assert_equal :booya, value
36
+ end
37
+
38
+ def test_with_ivars
39
+ object = FakeQueue.new
40
+ assert_nil object.commands_per_client
41
+
42
+ object.round_robin_from({})
43
+ assert_equal 100, object.commands_per_client
44
+
45
+ object.round_robin_from :commands_per_client => 1
46
+ assert_equal 1, object.commands_per_client
47
+ end
48
+
49
+ def test_shuffles_solitary_client
50
+ set_clients 1
51
+
52
+ assert_equal 1, client
53
+ assert_equal 1, client
54
+ assert_equal 1, client
55
+ end
56
+
57
+ def test_shuffles_clients
58
+ set_clients 1, 2
59
+
60
+ assert_equal 1, client
61
+ assert_equal 1, client
62
+ assert_equal 2, client
63
+ end
64
+
65
+ def commands_per_client
66
+ 2
67
+ end
68
+
69
+ def set_clients(*clients)
70
+ @client_index = @client_len = nil
71
+ @clients = clients
72
+ rotate_client
73
+ end
74
+
75
+ class FakeQueue
76
+ QueueKit::Clients::RoundRobinShuffler.with_ivars(self)
77
+ end
78
+ end
79
+
@@ -0,0 +1,81 @@
1
+ require File.expand_path("../helper", __FILE__)
2
+
3
+ class WorkerTest < Test::Unit::TestCase
4
+ def test_cooler
5
+ cooled = false
6
+ cooler = lambda { cooled = true }
7
+
8
+ worker = new_worker [], :processor => lambda { |_| fail 'item found?' },
9
+ :cooler => cooler
10
+
11
+ worker.work
12
+ assert cooled
13
+ end
14
+
15
+ def test_custom_on_error_handler
16
+ called = false
17
+ error_handler = lambda do |exc|
18
+ called = true
19
+ assert_equal 'booya', exc.message
20
+ end
21
+
22
+ worker = new_worker [1], :processor => lambda { |_| raise 'booya' },
23
+ :error_handler => error_handler
24
+
25
+ worker.work
26
+
27
+ assert called
28
+ end
29
+
30
+ def test_default_error_handler
31
+ processor = lambda do |item|
32
+ raise item.to_s
33
+ end
34
+
35
+ worker = new_worker [1], :processor => processor
36
+
37
+ begin
38
+ worker.work
39
+ rescue RuntimeError => err
40
+ assert_equal '1', err.message
41
+ else
42
+ fail "no exception raised"
43
+ end
44
+ end
45
+
46
+ def test_breaks_when_stopped
47
+ called = false
48
+ worker = nil
49
+
50
+ processor = lambda do |item|
51
+ fail "callback called multiple times" if called
52
+ assert_equal 1, item
53
+ called = true
54
+ worker.stop
55
+ end
56
+
57
+ worker = new_worker [1, nil], :processor => processor
58
+ worker.run
59
+
60
+ assert called
61
+ end
62
+
63
+ def test_new_worker_is_not_working
64
+ assert !new_worker.working?
65
+ end
66
+
67
+ def new_worker(queue = [], options = {})
68
+ options[:instrumenter] ||= NullInstrumenter.new
69
+ Worker.new(queue, options)
70
+ end
71
+
72
+ class Worker
73
+ include QueueKit::Worker
74
+ end
75
+
76
+ class NullInstrumenter
77
+ def instrument(name, payload = nil)
78
+ end
79
+ end
80
+ end
81
+
metadata ADDED
@@ -0,0 +1,68 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: queue_kit
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.6
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Rick Olson
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-04-19 00:00:00.000000000 Z
13
+ dependencies: []
14
+ description:
15
+ email: technoweenie@gmail.com
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - Gemfile
21
+ - README.md
22
+ - Rakefile
23
+ - queue_kit.gemspec
24
+ - lib/queue_kit/clients/command_timeout.rb
25
+ - lib/queue_kit/clients/round_robin_shuffler.rb
26
+ - lib/queue_kit/signal_checker.rb
27
+ - lib/queue_kit/signal_handlers/graceful_quit.rb
28
+ - lib/queue_kit/worker.rb
29
+ - lib/queue_kit.rb
30
+ - test/command_timeout_test.rb
31
+ - test/helper.rb
32
+ - test/round_robin_shuffler_test.rb
33
+ - test/worker_test.rb
34
+ - script/bootstrap
35
+ - script/console
36
+ - script/package
37
+ - script/release
38
+ - script/test
39
+ homepage: https://github.com/technoweenie/queue_kit
40
+ licenses:
41
+ - MIT
42
+ post_install_message:
43
+ rdoc_options: []
44
+ require_paths:
45
+ - lib
46
+ required_ruby_version: !ruby/object:Gem::Requirement
47
+ none: false
48
+ requirements:
49
+ - - ! '>='
50
+ - !ruby/object:Gem::Version
51
+ version: '0'
52
+ required_rubygems_version: !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ! '>='
56
+ - !ruby/object:Gem::Version
57
+ version: 1.3.6
58
+ requirements: []
59
+ rubyforge_project:
60
+ rubygems_version: 1.8.23
61
+ signing_key:
62
+ specification_version: 2
63
+ summary: Job Queue hobby kit
64
+ test_files:
65
+ - test/command_timeout_test.rb
66
+ - test/helper.rb
67
+ - test/round_robin_shuffler_test.rb
68
+ - test/worker_test.rb