chore-core 1.10.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +5 -13
  2. data/LICENSE.txt +1 -1
  3. data/README.md +172 -153
  4. data/chore-core.gemspec +3 -3
  5. data/lib/chore.rb +29 -5
  6. data/lib/chore/cli.rb +22 -4
  7. data/lib/chore/configuration.rb +1 -1
  8. data/lib/chore/consumer.rb +54 -12
  9. data/lib/chore/fetcher.rb +12 -7
  10. data/lib/chore/hooks.rb +2 -1
  11. data/lib/chore/job.rb +19 -0
  12. data/lib/chore/manager.rb +17 -2
  13. data/lib/chore/publisher.rb +18 -2
  14. data/lib/chore/queues/filesystem/consumer.rb +126 -64
  15. data/lib/chore/queues/filesystem/filesystem_queue.rb +19 -0
  16. data/lib/chore/queues/filesystem/publisher.rb +10 -16
  17. data/lib/chore/queues/sqs.rb +22 -13
  18. data/lib/chore/queues/sqs/consumer.rb +64 -51
  19. data/lib/chore/queues/sqs/publisher.rb +26 -17
  20. data/lib/chore/strategies/consumer/batcher.rb +6 -6
  21. data/lib/chore/strategies/consumer/single_consumer_strategy.rb +5 -5
  22. data/lib/chore/strategies/consumer/threaded_consumer_strategy.rb +7 -6
  23. data/lib/chore/strategies/consumer/throttled_consumer_strategy.rb +120 -0
  24. data/lib/chore/strategies/worker/forked_worker_strategy.rb +5 -6
  25. data/lib/chore/strategies/worker/helpers/ipc.rb +87 -0
  26. data/lib/chore/strategies/worker/helpers/preforked_worker.rb +163 -0
  27. data/lib/chore/strategies/worker/helpers/work_distributor.rb +65 -0
  28. data/lib/chore/strategies/worker/helpers/worker_info.rb +13 -0
  29. data/lib/chore/strategies/worker/helpers/worker_killer.rb +40 -0
  30. data/lib/chore/strategies/worker/helpers/worker_manager.rb +183 -0
  31. data/lib/chore/strategies/worker/preforked_worker_strategy.rb +150 -0
  32. data/lib/chore/unit_of_work.rb +2 -1
  33. data/lib/chore/util.rb +5 -1
  34. data/lib/chore/version.rb +2 -2
  35. data/lib/chore/worker.rb +30 -3
  36. data/spec/chore/cli_spec.rb +2 -2
  37. data/spec/chore/consumer_spec.rb +1 -5
  38. data/spec/chore/duplicate_detector_spec.rb +17 -5
  39. data/spec/chore/fetcher_spec.rb +0 -11
  40. data/spec/chore/manager_spec.rb +7 -0
  41. data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb +74 -16
  42. data/spec/chore/queues/sqs/consumer_spec.rb +117 -78
  43. data/spec/chore/queues/sqs/publisher_spec.rb +49 -60
  44. data/spec/chore/queues/sqs_spec.rb +32 -41
  45. data/spec/chore/strategies/consumer/single_consumer_strategy_spec.rb +3 -3
  46. data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb +6 -6
  47. data/spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb +165 -0
  48. data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb +6 -1
  49. data/spec/chore/strategies/worker/helpers/ipc_spec.rb +127 -0
  50. data/spec/chore/strategies/worker/helpers/preforked_worker_spec.rb +236 -0
  51. data/spec/chore/strategies/worker/helpers/work_distributor_spec.rb +131 -0
  52. data/spec/chore/strategies/worker/helpers/worker_info_spec.rb +14 -0
  53. data/spec/chore/strategies/worker/helpers/worker_killer_spec.rb +97 -0
  54. data/spec/chore/strategies/worker/helpers/worker_manager_spec.rb +304 -0
  55. data/spec/chore/strategies/worker/preforked_worker_strategy_spec.rb +183 -0
  56. data/spec/chore/strategies/worker/single_worker_strategy_spec.rb +1 -1
  57. data/spec/chore/worker_spec.rb +70 -15
  58. data/spec/spec_helper.rb +1 -1
  59. data/spec/support/queues/sqs/fake_objects.rb +18 -0
  60. metadata +53 -29
@@ -37,10 +37,10 @@ Gem::Specification.new do |s|
37
37
  s.summary = "Job processing... for the future!"
38
38
 
39
39
  s.add_runtime_dependency(%q<json>, [">= 0"])
40
- s.add_runtime_dependency(%q<aws-sdk-v1>, ["~> 1.56", ">= 1.56.0"])
40
+ s.add_runtime_dependency(%q<aws-sdk-sqs>, ["~> 1"])
41
41
  s.add_runtime_dependency(%q<thread>, ["~> 0.1.3"])
42
- s.add_development_dependency(%q<rspec>, ["~> 3.3.0"])
42
+ s.add_runtime_dependency('get_process_mem', ["~> 0.2.0"])
43
+ s.add_development_dependency(%q<rspec>, ["~> 3.3"])
43
44
  s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
44
45
  s.add_development_dependency(%q<bundler>, [">= 0"])
45
46
  end
46
-
@@ -41,7 +41,11 @@ module Chore #:nodoc:
41
41
  :shutdown_timeout => (2 * 60),
42
42
  :max_attempts => 1.0 / 0.0, # Infinity
43
43
  :dupe_on_cache_failure => false,
44
- :payload_handler => Chore::Job
44
+ :queue_polling_size => 10,
45
+ :payload_handler => Chore::Job,
46
+ :master_procline => "chore-master-#{Chore::VERSION}",
47
+ :worker_procline => "chore-worker-#{Chore::VERSION}",
48
+ :consumer_sleep_interval => 1
45
49
  }
46
50
 
47
51
  class << self
@@ -59,6 +63,24 @@ module Chore #:nodoc:
59
63
  end
60
64
  end
61
65
 
66
+ def self.log_level_to_sym
67
+ return self.config[:log_level] if self.config[:log_level].is_a?(Symbol)
68
+ case self.config[:log_level]
69
+ when 0
70
+ :debug
71
+ when 1
72
+ :info
73
+ when 2
74
+ :warn
75
+ when 3
76
+ :error
77
+ when 4
78
+ :fatal
79
+ else
80
+ :unknown
81
+ end
82
+ end
83
+
62
84
  # Reopens any open files. This will match any logfile that was opened by Chore,
63
85
  # Rails, or any other library.
64
86
  def self.reopen_logs
@@ -111,9 +133,9 @@ module Chore #:nodoc:
111
133
  # add_hook(:before_fork) {|worker| puts 1 }
112
134
  # add_hook(:before_fork) {|worker| puts 2 }
113
135
  # add_hook(:before_fork) {|worker| puts 3 }
114
- #
136
+ #
115
137
  # run_hooks_for(:before_fork, worker)
116
- #
138
+ #
117
139
  # # ...will produce the following output
118
140
  # => 1
119
141
  # => 2
@@ -130,9 +152,9 @@ module Chore #:nodoc:
130
152
  # add_hook(:around_fork) {|worker, &block| puts 'before 1'; block.call; puts 'after 1'}
131
153
  # add_hook(:around_fork) {|worker, &block| puts 'before 2'; block.call; puts 'after 2'}
132
154
  # add_hook(:around_fork) {|worker, &block| puts 'before 3'; block.call; puts 'after 3'}
133
- #
155
+ #
134
156
  # run_hooks_for(:around_fork, worker) { puts 'block' }
135
- #
157
+ #
136
158
  # # ...will produce the following output
137
159
  # => before 1
138
160
  # => before 2
@@ -214,6 +236,8 @@ module Chore #:nodoc:
214
236
  end
215
237
 
216
238
  # List of queue_names as configured via Chore::Job including their prefix, if set.
239
+ #
240
+ # @return [Array<String>]
217
241
  def self.prefixed_queue_names
218
242
  Chore::Job.job_classes.collect {|klass| c = constantize(klass); c.prefixed_queue_name}
219
243
  end
@@ -89,10 +89,11 @@ module Chore #:nodoc:
89
89
  detect_queues
90
90
  Chore.configure(options)
91
91
  Chore.configuring = false
92
+ validate_strategy!
92
93
  end
93
94
 
94
-
95
95
  private
96
+
96
97
  def setup_options #:nodoc:
97
98
  register_option "queues", "-q", "--queues QUEUE1,QUEUE2", "Names of queues to process (default: all known)" do |arg|
98
99
  # This will remove duplicates. We ultimately force this to be a Set further below
@@ -134,7 +135,7 @@ module Chore #:nodoc:
134
135
  options[:consumer_strategy] = constantize(arg)
135
136
  end
136
137
 
137
- register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages. Defaults vary depending on consumer implementation'
138
+ register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages (default: 1)'
138
139
 
139
140
  register_option 'payload_handler', '--payload_handler CLASS_NAME', 'Name of a class to use as the payload handler (default: Chore::Job)' do |arg|
140
141
  options[:payload_handler] = constantize(arg)
@@ -144,6 +145,7 @@ module Chore #:nodoc:
144
145
 
145
146
  register_option 'dupe_on_cache_failure', '--dupe-on-cache-failure BOOLEAN', 'Determines the deduping behavior when a cache connection error occurs. When set to false, the message is assumed not to be a duplicate. (default: false)'
146
147
 
148
+ register_option 'queue_polling_size', '--queue_polling_size NUM', Integer, 'Amount of messages to grab on each request (default: 10)'
147
149
  end
148
150
 
149
151
  def parse_opts(argv, ignore_errors = false) #:nodoc:
@@ -254,7 +256,6 @@ module Chore #:nodoc:
254
256
  end
255
257
 
256
258
  def validate! #:nodoc:
257
-
258
259
  missing_option!("--require [PATH|DIR]") unless options[:require]
259
260
 
260
261
  if !File.exist?(options[:require]) ||
@@ -266,8 +267,25 @@ module Chore #:nodoc:
266
267
  puts @parser
267
268
  exit(1)
268
269
  end
270
+ end
269
271
 
272
+ def validate_strategy!
273
+ consumer_strategy = Chore.config.consumer_strategy.to_s
274
+ worker_strategy = Chore.config.worker_strategy.to_s
275
+
276
+ throttled_consumer = 'Chore::Strategy::ThrottledConsumerStrategy'
277
+ preforked_worker = 'Chore::Strategy::PreForkedWorkerStrategy'
278
+
279
+ if consumer_strategy == throttled_consumer || worker_strategy == preforked_worker
280
+ unless consumer_strategy == throttled_consumer && worker_strategy == preforked_worker
281
+ puts "=================================================================="
282
+ puts " PreForkedWorkerStrategy may only be paired with "
283
+ puts " ThrottledConsumerStrategy or vice versa "
284
+ puts " Please check your configurations "
285
+ puts "=================================================================="
286
+ exit(1)
287
+ end
288
+ end
270
289
  end
271
290
  end
272
291
  end
273
-
@@ -1,6 +1,6 @@
1
1
  module Chore
2
2
  # Wrapper around an OpenStruct to define configuration data
3
- # (TODO): Add required opts, and validate that they're set
3
+ # TODO: Add required opts, and validate that they're set
4
4
  class Configuration < OpenStruct
5
5
  # Helper method to make merging Hashes into OpenStructs easier
6
6
  def merge_hash(hsh={})
@@ -1,16 +1,17 @@
1
1
  module Chore
2
2
  # Raised when Chore is booting up, but encounters a set of configuration that is impossible to boot from. Typically
3
- # you'll find additional information around the cause of the exception by examining the logfiles
3
+ # you'll find additional information around the cause of the exception by examining the logfiles.
4
+ # You can raise this exception if your queue is in a terrible state and must shut down.
4
5
  class TerribleMistake < Exception
5
- # You can raise this exception if your queue is in a terrible state and must shut down
6
6
  end
7
7
 
8
- # Base class for a Chore Consumer. Provides the basic interface to adhere to for building custom
9
- # Chore Consumers.
8
+ # Base class for a Chore Consumer. Provides the interface that a Chore::Consumer implementation should adhere to.
10
9
  class Consumer
11
10
 
12
11
  attr_accessor :queue_name
13
12
 
13
+ # @param [String] queue_name Name of queue to be consumed from
14
+ # @param [Hash] opts
14
15
  def initialize(queue_name, opts={})
15
16
  @queue_name = queue_name
16
17
  @running = true
@@ -21,26 +22,28 @@ module Chore
21
22
  def self.reset_connection!
22
23
  end
23
24
 
24
- # Cleans up any resources that were left behind from prior instances of the
25
- # chore process. By default, this is a no-op.
26
- def self.cleanup(queue)
27
- end
28
-
29
25
  # Consume takes a block with an arity of two. The two params are
30
26
  # |message_id,message_body| where message_id is any object that the
31
27
  # consumer will need to be able to act on a message later (reject, complete, etc)
32
- def consume(&block)
28
+ #
29
+ # @param [Block] &handler Message handler, used by the calling context (worker) to create & assigns a UnitOfWork
30
+ def consume(&handler)
33
31
  raise NotImplementedError
34
32
  end
35
33
 
36
34
  # Reject should put a message back on a queue to be processed again later. It takes
37
35
  # a message_id as returned via consume.
36
+ #
37
+ # @param [String] message_id Unique ID of the message
38
38
  def reject(message_id)
39
39
  raise NotImplementedError
40
40
  end
41
41
 
42
- # Complete should mark a message as finished. It takes a message_id as returned via consume
43
- def complete(message_id)
42
+ # Complete should mark a message as finished.
43
+ #
44
+ # @param [String] message_id Unique ID of the message
45
+ # @param [Hash] receipt_handle Unique ID of the consuming transaction in non-filesystem implementations
46
+ def complete(message_id, receipt_handle)
44
47
  raise NotImplementedError
45
48
  end
46
49
 
@@ -50,8 +53,47 @@ module Chore
50
53
  end
51
54
 
52
55
  # Returns true if the Consumer is currently running
56
+ #
57
+ # @return [TrueClass, FalseClass]
53
58
  def running?
54
59
  @running
55
60
  end
61
+
62
+ # Returns up to n work
63
+ #
64
+ # @param n
65
+ def provide_work(n)
66
+ raise NotImplementedError
67
+ end
68
+
69
+ # Determine whether or not we have already seen this message
70
+ #
71
+ # @param [String] dedupe_key
72
+ # @param [Class] klass
73
+ # @param [Integer] queue_timeout
74
+ #
75
+ # @return [TrueClass, FalseClass]
76
+ def duplicate_message?(dedupe_key, klass, queue_timeout)
77
+ dupe_detector.found_duplicate?(:id=>dedupe_key, :queue=>klass.to_s, :visibility_timeout=>queue_timeout)
78
+ end
79
+
80
+ # Instance of duplicate detection implementation class
81
+ #
82
+ # @return [DuplicateDetector]
83
+ def dupe_detector
84
+ @dupes ||= DuplicateDetector.new({:servers => Chore.config.dedupe_servers,
85
+ :dupe_on_cache_failure => false})
86
+ end
87
+
88
+ private
89
+
90
+ # Gets messages from queue implementation and invokes the provided block over each one. Afterwards, the :on_fetch
91
+ # hook will be invoked per message. This block call provides data necessary for the worker (calling context) to
92
+ # populate a UnitOfWork struct.
93
+ #
94
+ # @param [Block] &handler Message handler, passed along by #consume
95
+ def handle_messages(&handler)
96
+ raise NotImplementedError
97
+ end
56
98
  end
57
99
  end
@@ -11,21 +11,16 @@ module Chore
11
11
  # Starts the fetcher with the configured Consumer Strategy. This will begin consuming messages from your queue
12
12
  def start
13
13
  Chore.logger.info "Fetcher starting up"
14
-
15
- # Clean up configured queues in case there are any resources left behind
16
- Chore.config.queues.each do |queue|
17
- Chore.config.consumer.cleanup(queue)
18
- end
19
-
20
14
  @strategy.fetch
21
15
  end
22
16
 
23
17
  # Stops the fetcher, preventing any further messages from being pulled from the queue
24
18
  def stop!
25
19
  unless @stopping
26
- Chore.logger.info "Fetcher shutting down"
20
+ Chore.logger.info "Fetcher shutting down started"
27
21
  @stopping = true
28
22
  @strategy.stop!
23
+ Chore.logger.info "Fetcher shutting down completed"
29
24
  end
30
25
  end
31
26
 
@@ -33,5 +28,15 @@ module Chore
33
28
  def stopping?
34
29
  @stopping
35
30
  end
31
+
32
+ # returns upto n work units
33
+ def provide_work(n)
34
+ @strategy.provide_work(n)
35
+ end
36
+
37
+ # gives work back to the consumer in case it couldn't be assigned
38
+ def return_work(work_units)
39
+ @strategy.return_work(work_units)
40
+ end
36
41
  end
37
42
  end
@@ -15,7 +15,8 @@ module Chore
15
15
 
16
16
  private
17
17
  def hooks_for(event)
18
- candidate_methods.grep(/^#{event}/).sort
18
+ @_chore_hooks ||= {}
19
+ @_chore_hooks[event] ||= candidate_methods.grep(/^#{event}/).sort
19
20
  end
20
21
 
21
22
  # NOTE: Any hook methods defined after this is first referenced (i.e.,
@@ -60,6 +60,12 @@ module Chore
60
60
  raise ArgumentError, "#{self.to_s}: backoff must accept a single argument"
61
61
  end
62
62
  end
63
+
64
+ if @chore_options.key?(:dedupe_lambda)
65
+ if !@chore_options[:dedupe_lambda].is_a?(Proc)
66
+ raise ArgumentError, "#{self.to_s}: dedupe_lambda must be a lambda or Proc"
67
+ end
68
+ end
63
69
  end
64
70
 
65
71
  # This is a method so it can be overriden to create additional required
@@ -99,6 +105,8 @@ module Chore
99
105
  end
100
106
 
101
107
  # The name of the configured queue, combined with an optional prefix
108
+ #
109
+ # @return [String]
102
110
  def prefixed_queue_name
103
111
  "#{Chore.config.queue_prefix}#{self.options[:name]}"
104
112
  end
@@ -108,6 +116,17 @@ module Chore
108
116
  def has_backoff?
109
117
  self.options.key?(:backoff)
110
118
  end
119
+
120
+ def has_dedupe_lambda?
121
+ self.options.key?(:dedupe_lambda)
122
+ end
123
+
124
+ def dedupe_key(*args)
125
+ return unless has_dedupe_lambda?
126
+
127
+ # run the proc to get the key
128
+ self.options[:dedupe_lambda].call(*args).to_s
129
+ end
111
130
  end #ClassMethods
112
131
 
113
132
  # This is handy to override in an included job to be able to do job setup that requires
@@ -5,14 +5,15 @@ require 'chore/fetcher'
5
5
  module Chore
6
6
  # Manages the interactions between fetching messages (Consumer Strategy), and working over them (Worker Strategy)
7
7
  class Manager
8
+ include Util
8
9
 
9
10
  def initialize()
10
11
  Chore.logger.info "Booting Chore #{Chore::VERSION}"
11
12
  Chore.logger.debug { Chore.config.inspect }
13
+ procline("#{Chore.config.master_procline}:Started:#{Time.now}")
12
14
  @started_at = nil
13
15
  @worker_strategy = Chore.config.worker_strategy.new(self)
14
16
  @fetcher = Chore.config.fetcher.new(self)
15
- @processed = 0
16
17
  @stopping = false
17
18
  end
18
19
 
@@ -27,11 +28,12 @@ module Chore
27
28
  # Shut down the Manager, the Worker Strategy, and the Fetcher. This calls the +:before_shutdown+ hook.
28
29
  def shutdown!
29
30
  unless @stopping
30
- Chore.logger.info "Manager shutting down"
31
+ Chore.logger.info "Manager shutting down started"
31
32
  @stopping = true
32
33
  Chore.run_hooks_for(:before_shutdown)
33
34
  @fetcher.stop!
34
35
  @worker_strategy.stop!
36
+ Chore.logger.info "Manager shutting down completed"
35
37
  end
36
38
  end
37
39
 
@@ -42,7 +44,20 @@ module Chore
42
44
  # than they can be consumed.
43
45
  def assign(work)
44
46
  Chore.logger.debug { "Manager#assign: No. of UnitsOfWork: #{work.length})" }
47
+ work.each do | item |
48
+ Chore.run_hooks_for(:before_send_to_worker, item)
49
+ end
45
50
  @worker_strategy.assign(work) unless @stopping
46
51
  end
52
+
53
+ # returns up to n from the throttled consumer queue
54
+ def fetch_work(n)
55
+ @fetcher.provide_work(n)
56
+ end
57
+
58
+ # gives work back to the fetcher in case it couldn't be assigned
59
+ def return_work(work_units)
60
+ @fetcher.return_work(work_units)
61
+ end
47
62
  end
48
63
  end
@@ -1,26 +1,42 @@
1
1
  module Chore
2
- # Base class for Chore Publishers. Provides the bare interface one needs to adhere to when writing custom publishers
2
+ # Base class for a Chore Publisher. Provides the interface that a Chore::Publisher implementation should adhere to.
3
3
  class Publisher
4
4
  DEFAULT_OPTIONS = { :encoder => Encoder::JsonEncoder }
5
5
 
6
6
  attr_accessor :options
7
7
 
8
+ # @param [Hash] opts
8
9
  def initialize(opts={})
9
10
  self.options = DEFAULT_OPTIONS.merge(opts)
10
11
  end
11
12
 
12
13
  # Publishes the provided +job+ to the queue identified by the +queue_name+. Not designed to be used directly, this
13
14
  # method ferries to the publish method on an instance of your configured Publisher.
15
+ #
16
+ # @param [String] queue_name Name of queue to be consumed from
17
+ # @param [Hash] job Job instance definition, will be encoded to JSON
14
18
  def self.publish(queue_name,job)
15
19
  self.new.publish(queue_name,job)
16
20
  end
17
21
 
18
- # Raises a NotImplementedError. This method should be overridden in your descendent, custom publisher class
22
+ # Publishes a message to queue
23
+ #
24
+ # @param [String] queue_name Name of the SQS queue
25
+ # @param [Hash] job Job instance definition, will be encoded to JSON
19
26
  def publish(queue_name,job)
20
27
  raise NotImplementedError
21
28
  end
29
+
30
+ # Sets a flag that instructs the publisher to reset the connection the next time it's used.
31
+ # Should be overriden in publishers (but is not required)
32
+ def self.reset_connection!
33
+ end
34
+
22
35
  protected
23
36
 
37
+ # Encodes the job class to format provided by endoder implementation
38
+ #
39
+ # @param [Any] job
24
40
  def encode_job(job)
25
41
  options[:encoder].encode(job)
26
42
  end