sneakers_custom_bunny 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +20 -0
- data/Gemfile +3 -0
- data/Guardfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +172 -0
- data/ROADMAP.md +18 -0
- data/Rakefile +11 -0
- data/bin/sneakers +5 -0
- data/examples/benchmark_worker.rb +20 -0
- data/examples/max_retry_handler.rb +78 -0
- data/examples/metrics_worker.rb +28 -0
- data/examples/newrelic_metrics_worker.rb +40 -0
- data/examples/profiling_worker.rb +69 -0
- data/examples/sneakers.conf.rb.example +11 -0
- data/examples/title_scraper.rb +23 -0
- data/examples/workflow_worker.rb +23 -0
- data/lib/sneakers.rb +83 -0
- data/lib/sneakers/cli.rb +115 -0
- data/lib/sneakers/concerns/logging.rb +34 -0
- data/lib/sneakers/concerns/metrics.rb +34 -0
- data/lib/sneakers/configuration.rb +59 -0
- data/lib/sneakers/handlers/maxretry.rb +191 -0
- data/lib/sneakers/handlers/oneshot.rb +30 -0
- data/lib/sneakers/metrics/logging_metrics.rb +16 -0
- data/lib/sneakers/metrics/newrelic_metrics.rb +37 -0
- data/lib/sneakers/metrics/null_metrics.rb +13 -0
- data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
- data/lib/sneakers/publisher.rb +34 -0
- data/lib/sneakers/queue.rb +65 -0
- data/lib/sneakers/runner.rb +82 -0
- data/lib/sneakers/spawner.rb +27 -0
- data/lib/sneakers/support/production_formatter.rb +11 -0
- data/lib/sneakers/support/utils.rb +18 -0
- data/lib/sneakers/tasks.rb +34 -0
- data/lib/sneakers/version.rb +3 -0
- data/lib/sneakers/worker.rb +151 -0
- data/lib/sneakers/workergroup.rb +47 -0
- data/sneakers.gemspec +35 -0
- data/spec/fixtures/require_worker.rb +17 -0
- data/spec/sneakers/cli_spec.rb +63 -0
- data/spec/sneakers/concerns/logging_spec.rb +39 -0
- data/spec/sneakers/concerns/metrics_spec.rb +38 -0
- data/spec/sneakers/configuration_spec.rb +75 -0
- data/spec/sneakers/publisher_spec.rb +83 -0
- data/spec/sneakers/queue_spec.rb +115 -0
- data/spec/sneakers/runner_spec.rb +26 -0
- data/spec/sneakers/sneakers_spec.rb +75 -0
- data/spec/sneakers/support/utils_spec.rb +44 -0
- data/spec/sneakers/worker_handlers_spec.rb +390 -0
- data/spec/sneakers/worker_spec.rb +463 -0
- data/spec/spec_helper.rb +13 -0
- metadata +306 -0
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Sneakers
|
4
|
+
class Spawner
|
5
|
+
|
6
|
+
def self.spawn
|
7
|
+
worker_group_config_file = ENV['WORKER_GROUP_CONFIG'] || "./config/sneaker_worker_groups.yml"
|
8
|
+
unless File.exists?(worker_group_config_file)
|
9
|
+
puts "No worker group file found."
|
10
|
+
puts "Specify via ENV 'WORKER_GROUP_CONFIG' or by convention ./config/sneaker_worker_groups.yml"
|
11
|
+
end
|
12
|
+
@pids = []
|
13
|
+
@exec_string = "bundle exec rake sneakers:run"
|
14
|
+
worker_config = YAML.load(File.read(worker_group_config_file))
|
15
|
+
worker_config.keys.each do |group_name|
|
16
|
+
@pids << fork do
|
17
|
+
@exec_hash = {"WORKERS"=> worker_config[group_name]['classes'], "WORKER_COUNT" => worker_config[group_name]["workers"].to_s}
|
18
|
+
Kernel.exec(@exec_hash, @exec_string)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
["TERM", "USR1", "HUP", "USR2"].each do |signal|
|
22
|
+
Signal.trap(signal){ @pids.each{|pid| Process.kill(signal, pid) } }
|
23
|
+
end
|
24
|
+
Process.waitall
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require 'time'
|
2
|
+
module Sneakers
|
3
|
+
module Support
|
4
|
+
class ProductionFormatter < Logger::Formatter
|
5
|
+
def self.call(severity, time, program_name, message)
|
6
|
+
"#{time.utc.iso8601} p-#{Process.pid} t-#{Thread.current.object_id.to_s(36)} #{severity}: #{message}\n"
|
7
|
+
end
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class Sneakers::Utils
|
2
|
+
def self.make_worker_id(namespace)
|
3
|
+
"worker-#{namespace}:#{'1'}:#{rand(36**6).floor.to_s(36)}" # jid, worker id. include date.
|
4
|
+
end
|
5
|
+
def self.parse_workers(workerstring)
|
6
|
+
missing_workers = []
|
7
|
+
workers = (workerstring || '').split(',').map do |k|
|
8
|
+
begin
|
9
|
+
w = k.split('::').inject(Kernel){|s, c| s.const_get(c)}
|
10
|
+
rescue
|
11
|
+
missing_workers << k
|
12
|
+
end
|
13
|
+
w
|
14
|
+
end.compact
|
15
|
+
|
16
|
+
[workers, missing_workers]
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'sneakers'
|
2
|
+
require 'sneakers/runner'
|
3
|
+
|
4
|
+
task :environment
|
5
|
+
|
6
|
+
namespace :sneakers do
|
7
|
+
desc "Start work (set $WORKERS=Klass1,Klass2)"
|
8
|
+
task :run => :environment do
|
9
|
+
|
10
|
+
workers, missing_workers = Sneakers::Utils.parse_workers(ENV['WORKERS'])
|
11
|
+
|
12
|
+
unless missing_workers.empty?
|
13
|
+
puts "Missing workers: #{missing_workers.join(', ')}" if missing_workers
|
14
|
+
puts "Did you `require` properly?"
|
15
|
+
exit(1)
|
16
|
+
end
|
17
|
+
|
18
|
+
if workers.empty?
|
19
|
+
puts <<EOF
|
20
|
+
Error: No workers found.
|
21
|
+
Please set the classes of the workers you want to run like so:
|
22
|
+
|
23
|
+
$ export WORKERS=MyWorker,FooWorker
|
24
|
+
$ rake sneakers:run
|
25
|
+
|
26
|
+
EOF
|
27
|
+
exit(1)
|
28
|
+
end
|
29
|
+
opts = (ENV['WORKER_COUNT'].present? ? {:workers => ENV['WORKER_COUNT'].to_i} : {})
|
30
|
+
r = Sneakers::Runner.new(workers, opts)
|
31
|
+
|
32
|
+
r.run
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,151 @@
|
|
1
|
+
require 'sneakers/queue'
|
2
|
+
require 'sneakers/support/utils'
|
3
|
+
require 'timeout'
|
4
|
+
|
5
|
+
module Sneakers
|
6
|
+
module Worker
|
7
|
+
attr_reader :queue, :id, :opts
|
8
|
+
|
9
|
+
# For now, a worker is hardly dependant on these concerns
|
10
|
+
# (because it uses methods from them directly.)
|
11
|
+
include Concerns::Logging
|
12
|
+
include Concerns::Metrics
|
13
|
+
|
14
|
+
def initialize(queue = nil, pool = nil, opts = {})
|
15
|
+
opts = opts.merge(self.class.queue_opts || {})
|
16
|
+
queue_name = self.class.queue_name
|
17
|
+
opts = Sneakers::CONFIG.merge(opts)
|
18
|
+
|
19
|
+
@should_ack = opts[:ack]
|
20
|
+
@timeout_after = opts[:timeout_job_after]
|
21
|
+
@pool = pool || Thread.pool(opts[:threads]) # XXX config threads
|
22
|
+
@call_with_params = respond_to?(:work_with_params)
|
23
|
+
|
24
|
+
@queue = queue || Sneakers::Queue.new(
|
25
|
+
queue_name,
|
26
|
+
opts
|
27
|
+
)
|
28
|
+
|
29
|
+
@opts = opts
|
30
|
+
@id = Utils.make_worker_id(queue_name)
|
31
|
+
end
|
32
|
+
|
33
|
+
def ack!; :ack end
|
34
|
+
def reject!; :reject; end
|
35
|
+
def requeue!; :requeue; end
|
36
|
+
|
37
|
+
def publish(msg, opts)
|
38
|
+
to_queue = opts.delete(:to_queue)
|
39
|
+
opts[:routing_key] ||= to_queue
|
40
|
+
return unless opts[:routing_key]
|
41
|
+
@queue.exchange.publish(msg, opts)
|
42
|
+
end
|
43
|
+
|
44
|
+
def do_work(delivery_info, metadata, msg, handler)
|
45
|
+
worker_trace "Working off: #{msg}"
|
46
|
+
|
47
|
+
@pool.process do
|
48
|
+
res = nil
|
49
|
+
error = nil
|
50
|
+
|
51
|
+
begin
|
52
|
+
metrics.increment("work.#{self.class.name}.started")
|
53
|
+
Timeout.timeout(@timeout_after, Timeout::Error) do
|
54
|
+
metrics.timing("work.#{self.class.name}.time") do
|
55
|
+
if @call_with_params
|
56
|
+
res = work_with_params(msg, delivery_info, metadata)
|
57
|
+
else
|
58
|
+
res = work(msg)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
rescue Timeout::Error
|
63
|
+
res = :timeout
|
64
|
+
worker_error('timeout')
|
65
|
+
rescue => ex
|
66
|
+
res = :error
|
67
|
+
error = ex
|
68
|
+
worker_error('unexpected error', ex)
|
69
|
+
end
|
70
|
+
|
71
|
+
if @should_ack
|
72
|
+
|
73
|
+
if res == :ack
|
74
|
+
# note to future-self. never acknowledge multiple (multiple=true) messages under threads.
|
75
|
+
handler.acknowledge(delivery_info, metadata, msg)
|
76
|
+
elsif res == :timeout
|
77
|
+
handler.timeout(delivery_info, metadata, msg)
|
78
|
+
elsif res == :error
|
79
|
+
handler.error(delivery_info, metadata, msg, error)
|
80
|
+
elsif res == :reject
|
81
|
+
handler.reject(delivery_info, metadata, msg)
|
82
|
+
elsif res == :requeue
|
83
|
+
handler.reject(delivery_info, metadata, msg, true)
|
84
|
+
else
|
85
|
+
handler.noop(delivery_info, metadata, msg)
|
86
|
+
end
|
87
|
+
metrics.increment("work.#{self.class.name}.handled.#{res || 'noop'}")
|
88
|
+
end
|
89
|
+
|
90
|
+
metrics.increment("work.#{self.class.name}.ended")
|
91
|
+
end #process
|
92
|
+
end
|
93
|
+
|
94
|
+
def stop
|
95
|
+
worker_trace "Stopping worker: unsubscribing."
|
96
|
+
@queue.unsubscribe
|
97
|
+
worker_trace "Stopping worker: I'm gone."
|
98
|
+
end
|
99
|
+
|
100
|
+
def run
|
101
|
+
worker_trace "New worker: subscribing."
|
102
|
+
@queue.subscribe(self)
|
103
|
+
worker_trace "New worker: I'm alive."
|
104
|
+
end
|
105
|
+
|
106
|
+
# Construct a log message with some standard prefix for this worker
|
107
|
+
def log_msg(msg)
|
108
|
+
"[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
|
109
|
+
end
|
110
|
+
|
111
|
+
# Helper to log an error message with an optional exception
|
112
|
+
def worker_error(msg, exception = nil)
|
113
|
+
s = log_msg(msg)
|
114
|
+
if exception
|
115
|
+
s += " [Exception error=#{exception.message.inspect} error_class=#{exception.class}"
|
116
|
+
s += " backtrace=#{exception.backtrace.take(50).join(',')}" unless exception.backtrace.nil?
|
117
|
+
s += "]"
|
118
|
+
end
|
119
|
+
logger.error(s)
|
120
|
+
end
|
121
|
+
|
122
|
+
def worker_trace(msg)
|
123
|
+
logger.debug(log_msg(msg))
|
124
|
+
end
|
125
|
+
|
126
|
+
def self.included(base)
|
127
|
+
base.extend ClassMethods
|
128
|
+
end
|
129
|
+
|
130
|
+
module ClassMethods
|
131
|
+
attr_reader :queue_opts
|
132
|
+
attr_reader :queue_name
|
133
|
+
|
134
|
+
def from_queue(q, opts={})
|
135
|
+
@queue_name = q.to_s
|
136
|
+
@queue_opts = opts
|
137
|
+
end
|
138
|
+
|
139
|
+
def enqueue(msg)
|
140
|
+
publisher.publish(msg, :to_queue => @queue_name)
|
141
|
+
end
|
142
|
+
|
143
|
+
private
|
144
|
+
|
145
|
+
def publisher
|
146
|
+
@publisher ||= Sneakers::Publisher.new
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Sneakers
|
2
|
+
module WorkerGroup
|
3
|
+
@workers = []
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@stop_flag = ServerEngine::BlockingFlag.new
|
7
|
+
end
|
8
|
+
|
9
|
+
def before_fork
|
10
|
+
fbefore = Sneakers::CONFIG[:hooks][:before_fork]
|
11
|
+
fbefore.call if fbefore
|
12
|
+
end
|
13
|
+
|
14
|
+
def after_fork # note! this is not Serverengine#after_start, this is ours!
|
15
|
+
fafter = Sneakers::CONFIG[:hooks][:after_fork]
|
16
|
+
fafter.call if fafter
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
after_fork
|
21
|
+
|
22
|
+
@workers = config[:worker_classes].map{|w| w.new }
|
23
|
+
# if more than one worker this should be per worker
|
24
|
+
# accumulate clients and consumers as well
|
25
|
+
@workers.each do |worker|
|
26
|
+
worker.run
|
27
|
+
end
|
28
|
+
# end per worker
|
29
|
+
#
|
30
|
+
until @stop_flag.wait_for_set(10.0)
|
31
|
+
Sneakers.logger.debug("Heartbeat: running threads [#{Thread.list.count}]")
|
32
|
+
# report aggregated stats?
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
def stop
|
38
|
+
Sneakers.logger.info("Shutting down workers")
|
39
|
+
@workers.each do |worker|
|
40
|
+
worker.stop
|
41
|
+
end
|
42
|
+
@stop_flag.set!
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
data/sneakers.gemspec
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'sneakers/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = 'sneakers_custom_bunny'
|
8
|
+
gem.version = Sneakers::VERSION
|
9
|
+
gem.authors = ['Dotan Nahum']
|
10
|
+
gem.email = ['jondotan@gmail.com']
|
11
|
+
gem.description = %q( Fast background processing framework for Ruby and RabbitMQ )
|
12
|
+
gem.summary = %q( Fast background processing framework for Ruby and RabbitMQ )
|
13
|
+
gem.homepage = ''
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' }
|
16
|
+
gem.executables = gem.files.grep(/^bin/).map { |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(/^(test|spec|features)\//)
|
18
|
+
gem.require_paths = ['lib']
|
19
|
+
gem.add_dependency 'serverengine', '~> 1.5.5'
|
20
|
+
gem.add_dependency 'bunny', '~> 1.7.0'
|
21
|
+
gem.add_dependency 'thread', '~> 0.1.7'
|
22
|
+
gem.add_dependency 'thor'
|
23
|
+
|
24
|
+
gem.add_development_dependency 'rr'
|
25
|
+
gem.add_development_dependency 'ruby-prof'
|
26
|
+
gem.add_development_dependency 'nokogiri'
|
27
|
+
gem.add_development_dependency 'guard-minitest'
|
28
|
+
gem.add_development_dependency 'metric_fu'
|
29
|
+
gem.add_development_dependency 'simplecov'
|
30
|
+
gem.add_development_dependency 'simplecov-rcov-text'
|
31
|
+
gem.add_development_dependency 'rake'
|
32
|
+
gem.add_development_dependency 'minitest'
|
33
|
+
gem.add_development_dependency 'guard'
|
34
|
+
end
|
35
|
+
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'sneakers'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'nokogiri'
|
4
|
+
|
5
|
+
|
6
|
+
class TitleScraper
|
7
|
+
include Sneakers::Worker
|
8
|
+
|
9
|
+
from_queue 'downloads'
|
10
|
+
|
11
|
+
def work(msg)
|
12
|
+
doc = Nokogiri::HTML(open(msg))
|
13
|
+
worker_trace "FOUND <#{doc.css('title').text}>"
|
14
|
+
ack!
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
require 'sneakers/cli'
|
4
|
+
require 'sneakers/runner'
|
5
|
+
|
6
|
+
describe Sneakers::CLI do
|
7
|
+
describe "#work" do
|
8
|
+
before do
|
9
|
+
any_instance_of(Sneakers::Runner) do |runner|
|
10
|
+
stub(runner).run{ true }
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe 'with dirty class loading' do
|
15
|
+
after do
|
16
|
+
# require cleanup
|
17
|
+
Object.send(:remove_const, :TitleScraper)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should perform a run" do
|
21
|
+
any_instance_of(Sneakers::Runner) do |runner|
|
22
|
+
mock(runner).run{ true }
|
23
|
+
end
|
24
|
+
out = capture_io{ Sneakers::CLI.start [
|
25
|
+
'work',
|
26
|
+
"TitleScraper",
|
27
|
+
"--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
|
28
|
+
]}.join ''
|
29
|
+
|
30
|
+
out.must_match(/Workers.*:.*TitleScraper.*/)
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should be able to run as front-running process" do
|
35
|
+
out = capture_io{ Sneakers::CLI.start [
|
36
|
+
'work',
|
37
|
+
"TitleScraper",
|
38
|
+
"--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
|
39
|
+
]}.join ''
|
40
|
+
|
41
|
+
out.must_match(/Log.*Console/)
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should be able to run as daemonized process" do
|
45
|
+
out = capture_io{ Sneakers::CLI.start [
|
46
|
+
'work',
|
47
|
+
"TitleScraper",
|
48
|
+
"--daemonize",
|
49
|
+
"--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
|
50
|
+
]}.join ''
|
51
|
+
|
52
|
+
out.must_match(/sneakers.log/)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should fail when no workers found" do
|
57
|
+
out = capture_io{ Sneakers::CLI.start ['work', 'TitleScraper'] }.join ''
|
58
|
+
out.must_match(/Missing workers: TitleScraper/)
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
|
6
|
+
class Foobar
|
7
|
+
include Sneakers::Concerns::Logging
|
8
|
+
end
|
9
|
+
|
10
|
+
describe Sneakers::Concerns::Logging do
|
11
|
+
describe ".configure" do
|
12
|
+
before do
|
13
|
+
Foobar.logger = nil
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should configure a default logger when included" do
|
17
|
+
Foobar.logger.must_be_nil
|
18
|
+
Foobar.configure_logger
|
19
|
+
Foobar.logger.wont_be_nil
|
20
|
+
Foobar.logger.formatter.must_equal Sneakers::Support::ProductionFormatter
|
21
|
+
end
|
22
|
+
|
23
|
+
it "should supply accessible instance logger" do
|
24
|
+
Foobar.logger.must_be_nil
|
25
|
+
Foobar.configure_logger
|
26
|
+
f = Foobar.new
|
27
|
+
f.logger.must_equal Foobar.logger
|
28
|
+
f.logger.wont_be_nil
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should configure a given logger when specified" do
|
32
|
+
Foobar.logger.must_be_nil
|
33
|
+
log = Logger.new(STDOUT)
|
34
|
+
Foobar.configure_logger(log)
|
35
|
+
Foobar.logger.must_equal log
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|