sneakers_custom_bunny 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +9 -0
  3. data/.travis.yml +6 -0
  4. data/CHANGELOG.md +20 -0
  5. data/Gemfile +3 -0
  6. data/Guardfile +8 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +172 -0
  9. data/ROADMAP.md +18 -0
  10. data/Rakefile +11 -0
  11. data/bin/sneakers +5 -0
  12. data/examples/benchmark_worker.rb +20 -0
  13. data/examples/max_retry_handler.rb +78 -0
  14. data/examples/metrics_worker.rb +28 -0
  15. data/examples/newrelic_metrics_worker.rb +40 -0
  16. data/examples/profiling_worker.rb +69 -0
  17. data/examples/sneakers.conf.rb.example +11 -0
  18. data/examples/title_scraper.rb +23 -0
  19. data/examples/workflow_worker.rb +23 -0
  20. data/lib/sneakers.rb +83 -0
  21. data/lib/sneakers/cli.rb +115 -0
  22. data/lib/sneakers/concerns/logging.rb +34 -0
  23. data/lib/sneakers/concerns/metrics.rb +34 -0
  24. data/lib/sneakers/configuration.rb +59 -0
  25. data/lib/sneakers/handlers/maxretry.rb +191 -0
  26. data/lib/sneakers/handlers/oneshot.rb +30 -0
  27. data/lib/sneakers/metrics/logging_metrics.rb +16 -0
  28. data/lib/sneakers/metrics/newrelic_metrics.rb +37 -0
  29. data/lib/sneakers/metrics/null_metrics.rb +13 -0
  30. data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
  31. data/lib/sneakers/publisher.rb +34 -0
  32. data/lib/sneakers/queue.rb +65 -0
  33. data/lib/sneakers/runner.rb +82 -0
  34. data/lib/sneakers/spawner.rb +27 -0
  35. data/lib/sneakers/support/production_formatter.rb +11 -0
  36. data/lib/sneakers/support/utils.rb +18 -0
  37. data/lib/sneakers/tasks.rb +34 -0
  38. data/lib/sneakers/version.rb +3 -0
  39. data/lib/sneakers/worker.rb +151 -0
  40. data/lib/sneakers/workergroup.rb +47 -0
  41. data/sneakers.gemspec +35 -0
  42. data/spec/fixtures/require_worker.rb +17 -0
  43. data/spec/sneakers/cli_spec.rb +63 -0
  44. data/spec/sneakers/concerns/logging_spec.rb +39 -0
  45. data/spec/sneakers/concerns/metrics_spec.rb +38 -0
  46. data/spec/sneakers/configuration_spec.rb +75 -0
  47. data/spec/sneakers/publisher_spec.rb +83 -0
  48. data/spec/sneakers/queue_spec.rb +115 -0
  49. data/spec/sneakers/runner_spec.rb +26 -0
  50. data/spec/sneakers/sneakers_spec.rb +75 -0
  51. data/spec/sneakers/support/utils_spec.rb +44 -0
  52. data/spec/sneakers/worker_handlers_spec.rb +390 -0
  53. data/spec/sneakers/worker_spec.rb +463 -0
  54. data/spec/spec_helper.rb +13 -0
  55. metadata +306 -0
@@ -0,0 +1,27 @@
1
+ require 'yaml'
2
+
3
+ module Sneakers
4
+ class Spawner
5
+
6
+ def self.spawn
7
+ worker_group_config_file = ENV['WORKER_GROUP_CONFIG'] || "./config/sneaker_worker_groups.yml"
8
+ unless File.exists?(worker_group_config_file)
9
+ puts "No worker group file found."
10
+ puts "Specify via ENV 'WORKER_GROUP_CONFIG' or by convention ./config/sneaker_worker_groups.yml"
11
+ end
12
+ @pids = []
13
+ @exec_string = "bundle exec rake sneakers:run"
14
+ worker_config = YAML.load(File.read(worker_group_config_file))
15
+ worker_config.keys.each do |group_name|
16
+ @pids << fork do
17
+ @exec_hash = {"WORKERS"=> worker_config[group_name]['classes'], "WORKER_COUNT" => worker_config[group_name]["workers"].to_s}
18
+ Kernel.exec(@exec_hash, @exec_string)
19
+ end
20
+ end
21
+ ["TERM", "USR1", "HUP", "USR2"].each do |signal|
22
+ Signal.trap(signal){ @pids.each{|pid| Process.kill(signal, pid) } }
23
+ end
24
+ Process.waitall
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,11 @@
1
+ require 'time'
2
+ module Sneakers
3
+ module Support
4
+ class ProductionFormatter < Logger::Formatter
5
+ def self.call(severity, time, program_name, message)
6
+ "#{time.utc.iso8601} p-#{Process.pid} t-#{Thread.current.object_id.to_s(36)} #{severity}: #{message}\n"
7
+ end
8
+ end
9
+ end
10
+ end
11
+
@@ -0,0 +1,18 @@
1
+ class Sneakers::Utils
2
+ def self.make_worker_id(namespace)
3
+ "worker-#{namespace}:#{'1'}:#{rand(36**6).floor.to_s(36)}" # jid, worker id. include date.
4
+ end
5
+ def self.parse_workers(workerstring)
6
+ missing_workers = []
7
+ workers = (workerstring || '').split(',').map do |k|
8
+ begin
9
+ w = k.split('::').inject(Kernel){|s, c| s.const_get(c)}
10
+ rescue
11
+ missing_workers << k
12
+ end
13
+ w
14
+ end.compact
15
+
16
+ [workers, missing_workers]
17
+ end
18
+ end
@@ -0,0 +1,34 @@
1
+ require 'sneakers'
2
+ require 'sneakers/runner'
3
+
4
+ task :environment
5
+
6
+ namespace :sneakers do
7
+ desc "Start work (set $WORKERS=Klass1,Klass2)"
8
+ task :run => :environment do
9
+
10
+ workers, missing_workers = Sneakers::Utils.parse_workers(ENV['WORKERS'])
11
+
12
+ unless missing_workers.empty?
13
+ puts "Missing workers: #{missing_workers.join(', ')}" if missing_workers
14
+ puts "Did you `require` properly?"
15
+ exit(1)
16
+ end
17
+
18
+ if workers.empty?
19
+ puts <<EOF
20
+ Error: No workers found.
21
+ Please set the classes of the workers you want to run like so:
22
+
23
+ $ export WORKERS=MyWorker,FooWorker
24
+ $ rake sneakers:run
25
+
26
+ EOF
27
+ exit(1)
28
+ end
29
+ opts = (ENV['WORKER_COUNT'].present? ? {:workers => ENV['WORKER_COUNT'].to_i} : {})
30
+ r = Sneakers::Runner.new(workers, opts)
31
+
32
+ r.run
33
+ end
34
+ end
@@ -0,0 +1,3 @@
1
+ module Sneakers
2
+ VERSION = "1.0.4"
3
+ end
@@ -0,0 +1,151 @@
1
+ require 'sneakers/queue'
2
+ require 'sneakers/support/utils'
3
+ require 'timeout'
4
+
5
+ module Sneakers
6
+ module Worker
7
+ attr_reader :queue, :id, :opts
8
+
9
+ # For now, a worker is hardly dependant on these concerns
10
+ # (because it uses methods from them directly.)
11
+ include Concerns::Logging
12
+ include Concerns::Metrics
13
+
14
+ def initialize(queue = nil, pool = nil, opts = {})
15
+ opts = opts.merge(self.class.queue_opts || {})
16
+ queue_name = self.class.queue_name
17
+ opts = Sneakers::CONFIG.merge(opts)
18
+
19
+ @should_ack = opts[:ack]
20
+ @timeout_after = opts[:timeout_job_after]
21
+ @pool = pool || Thread.pool(opts[:threads]) # XXX config threads
22
+ @call_with_params = respond_to?(:work_with_params)
23
+
24
+ @queue = queue || Sneakers::Queue.new(
25
+ queue_name,
26
+ opts
27
+ )
28
+
29
+ @opts = opts
30
+ @id = Utils.make_worker_id(queue_name)
31
+ end
32
+
33
+ def ack!; :ack end
34
+ def reject!; :reject; end
35
+ def requeue!; :requeue; end
36
+
37
+ def publish(msg, opts)
38
+ to_queue = opts.delete(:to_queue)
39
+ opts[:routing_key] ||= to_queue
40
+ return unless opts[:routing_key]
41
+ @queue.exchange.publish(msg, opts)
42
+ end
43
+
44
+ def do_work(delivery_info, metadata, msg, handler)
45
+ worker_trace "Working off: #{msg}"
46
+
47
+ @pool.process do
48
+ res = nil
49
+ error = nil
50
+
51
+ begin
52
+ metrics.increment("work.#{self.class.name}.started")
53
+ Timeout.timeout(@timeout_after, Timeout::Error) do
54
+ metrics.timing("work.#{self.class.name}.time") do
55
+ if @call_with_params
56
+ res = work_with_params(msg, delivery_info, metadata)
57
+ else
58
+ res = work(msg)
59
+ end
60
+ end
61
+ end
62
+ rescue Timeout::Error
63
+ res = :timeout
64
+ worker_error('timeout')
65
+ rescue => ex
66
+ res = :error
67
+ error = ex
68
+ worker_error('unexpected error', ex)
69
+ end
70
+
71
+ if @should_ack
72
+
73
+ if res == :ack
74
+ # note to future-self. never acknowledge multiple (multiple=true) messages under threads.
75
+ handler.acknowledge(delivery_info, metadata, msg)
76
+ elsif res == :timeout
77
+ handler.timeout(delivery_info, metadata, msg)
78
+ elsif res == :error
79
+ handler.error(delivery_info, metadata, msg, error)
80
+ elsif res == :reject
81
+ handler.reject(delivery_info, metadata, msg)
82
+ elsif res == :requeue
83
+ handler.reject(delivery_info, metadata, msg, true)
84
+ else
85
+ handler.noop(delivery_info, metadata, msg)
86
+ end
87
+ metrics.increment("work.#{self.class.name}.handled.#{res || 'noop'}")
88
+ end
89
+
90
+ metrics.increment("work.#{self.class.name}.ended")
91
+ end #process
92
+ end
93
+
94
+ def stop
95
+ worker_trace "Stopping worker: unsubscribing."
96
+ @queue.unsubscribe
97
+ worker_trace "Stopping worker: I'm gone."
98
+ end
99
+
100
+ def run
101
+ worker_trace "New worker: subscribing."
102
+ @queue.subscribe(self)
103
+ worker_trace "New worker: I'm alive."
104
+ end
105
+
106
+ # Construct a log message with some standard prefix for this worker
107
+ def log_msg(msg)
108
+ "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
109
+ end
110
+
111
+ # Helper to log an error message with an optional exception
112
+ def worker_error(msg, exception = nil)
113
+ s = log_msg(msg)
114
+ if exception
115
+ s += " [Exception error=#{exception.message.inspect} error_class=#{exception.class}"
116
+ s += " backtrace=#{exception.backtrace.take(50).join(',')}" unless exception.backtrace.nil?
117
+ s += "]"
118
+ end
119
+ logger.error(s)
120
+ end
121
+
122
+ def worker_trace(msg)
123
+ logger.debug(log_msg(msg))
124
+ end
125
+
126
+ def self.included(base)
127
+ base.extend ClassMethods
128
+ end
129
+
130
+ module ClassMethods
131
+ attr_reader :queue_opts
132
+ attr_reader :queue_name
133
+
134
+ def from_queue(q, opts={})
135
+ @queue_name = q.to_s
136
+ @queue_opts = opts
137
+ end
138
+
139
+ def enqueue(msg)
140
+ publisher.publish(msg, :to_queue => @queue_name)
141
+ end
142
+
143
+ private
144
+
145
+ def publisher
146
+ @publisher ||= Sneakers::Publisher.new
147
+ end
148
+ end
149
+ end
150
+ end
151
+
@@ -0,0 +1,47 @@
1
+ module Sneakers
2
+ module WorkerGroup
3
+ @workers = []
4
+
5
+ def initialize
6
+ @stop_flag = ServerEngine::BlockingFlag.new
7
+ end
8
+
9
+ def before_fork
10
+ fbefore = Sneakers::CONFIG[:hooks][:before_fork]
11
+ fbefore.call if fbefore
12
+ end
13
+
14
+ def after_fork # note! this is not Serverengine#after_start, this is ours!
15
+ fafter = Sneakers::CONFIG[:hooks][:after_fork]
16
+ fafter.call if fafter
17
+ end
18
+
19
+ def run
20
+ after_fork
21
+
22
+ @workers = config[:worker_classes].map{|w| w.new }
23
+ # if more than one worker this should be per worker
24
+ # accumulate clients and consumers as well
25
+ @workers.each do |worker|
26
+ worker.run
27
+ end
28
+ # end per worker
29
+ #
30
+ until @stop_flag.wait_for_set(10.0)
31
+ Sneakers.logger.debug("Heartbeat: running threads [#{Thread.list.count}]")
32
+ # report aggregated stats?
33
+ end
34
+
35
+ end
36
+
37
+ def stop
38
+ Sneakers.logger.info("Shutting down workers")
39
+ @workers.each do |worker|
40
+ worker.stop
41
+ end
42
+ @stop_flag.set!
43
+ end
44
+
45
+ end
46
+ end
47
+
@@ -0,0 +1,35 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'sneakers/version'
5
+
6
+ Gem::Specification.new do |gem|
7
+ gem.name = 'sneakers_custom_bunny'
8
+ gem.version = Sneakers::VERSION
9
+ gem.authors = ['Dotan Nahum']
10
+ gem.email = ['jondotan@gmail.com']
11
+ gem.description = %q( Fast background processing framework for Ruby and RabbitMQ )
12
+ gem.summary = %q( Fast background processing framework for Ruby and RabbitMQ )
13
+ gem.homepage = ''
14
+
15
+ gem.files = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' }
16
+ gem.executables = gem.files.grep(/^bin/).map { |f| File.basename(f) }
17
+ gem.test_files = gem.files.grep(/^(test|spec|features)\//)
18
+ gem.require_paths = ['lib']
19
+ gem.add_dependency 'serverengine', '~> 1.5.5'
20
+ gem.add_dependency 'bunny', '~> 1.7.0'
21
+ gem.add_dependency 'thread', '~> 0.1.7'
22
+ gem.add_dependency 'thor'
23
+
24
+ gem.add_development_dependency 'rr'
25
+ gem.add_development_dependency 'ruby-prof'
26
+ gem.add_development_dependency 'nokogiri'
27
+ gem.add_development_dependency 'guard-minitest'
28
+ gem.add_development_dependency 'metric_fu'
29
+ gem.add_development_dependency 'simplecov'
30
+ gem.add_development_dependency 'simplecov-rcov-text'
31
+ gem.add_development_dependency 'rake'
32
+ gem.add_development_dependency 'minitest'
33
+ gem.add_development_dependency 'guard'
34
+ end
35
+
@@ -0,0 +1,17 @@
1
+ require 'sneakers'
2
+ require 'open-uri'
3
+ require 'nokogiri'
4
+
5
+
6
+ class TitleScraper
7
+ include Sneakers::Worker
8
+
9
+ from_queue 'downloads'
10
+
11
+ def work(msg)
12
+ doc = Nokogiri::HTML(open(msg))
13
+ worker_trace "FOUND <#{doc.css('title').text}>"
14
+ ack!
15
+ end
16
+ end
17
+
@@ -0,0 +1,63 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+ require 'sneakers/cli'
4
+ require 'sneakers/runner'
5
+
6
+ describe Sneakers::CLI do
7
+ describe "#work" do
8
+ before do
9
+ any_instance_of(Sneakers::Runner) do |runner|
10
+ stub(runner).run{ true }
11
+ end
12
+ end
13
+
14
+ describe 'with dirty class loading' do
15
+ after do
16
+ # require cleanup
17
+ Object.send(:remove_const, :TitleScraper)
18
+ end
19
+
20
+ it "should perform a run" do
21
+ any_instance_of(Sneakers::Runner) do |runner|
22
+ mock(runner).run{ true }
23
+ end
24
+ out = capture_io{ Sneakers::CLI.start [
25
+ 'work',
26
+ "TitleScraper",
27
+ "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
28
+ ]}.join ''
29
+
30
+ out.must_match(/Workers.*:.*TitleScraper.*/)
31
+
32
+ end
33
+
34
+ it "should be able to run as front-running process" do
35
+ out = capture_io{ Sneakers::CLI.start [
36
+ 'work',
37
+ "TitleScraper",
38
+ "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
39
+ ]}.join ''
40
+
41
+ out.must_match(/Log.*Console/)
42
+ end
43
+
44
+ it "should be able to run as daemonized process" do
45
+ out = capture_io{ Sneakers::CLI.start [
46
+ 'work',
47
+ "TitleScraper",
48
+ "--daemonize",
49
+ "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}"
50
+ ]}.join ''
51
+
52
+ out.must_match(/sneakers.log/)
53
+ end
54
+ end
55
+
56
+ it "should fail when no workers found" do
57
+ out = capture_io{ Sneakers::CLI.start ['work', 'TitleScraper'] }.join ''
58
+ out.must_match(/Missing workers: TitleScraper/)
59
+ end
60
+
61
+ end
62
+ end
63
+
@@ -0,0 +1,39 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+ require 'logger'
4
+
5
+
6
+ class Foobar
7
+ include Sneakers::Concerns::Logging
8
+ end
9
+
10
+ describe Sneakers::Concerns::Logging do
11
+ describe ".configure" do
12
+ before do
13
+ Foobar.logger = nil
14
+ end
15
+
16
+ it "should configure a default logger when included" do
17
+ Foobar.logger.must_be_nil
18
+ Foobar.configure_logger
19
+ Foobar.logger.wont_be_nil
20
+ Foobar.logger.formatter.must_equal Sneakers::Support::ProductionFormatter
21
+ end
22
+
23
+ it "should supply accessible instance logger" do
24
+ Foobar.logger.must_be_nil
25
+ Foobar.configure_logger
26
+ f = Foobar.new
27
+ f.logger.must_equal Foobar.logger
28
+ f.logger.wont_be_nil
29
+ end
30
+
31
+ it "should configure a given logger when specified" do
32
+ Foobar.logger.must_be_nil
33
+ log = Logger.new(STDOUT)
34
+ Foobar.configure_logger(log)
35
+ Foobar.logger.must_equal log
36
+ end
37
+ end
38
+ end
39
+