chore-core 1.10.0 → 3.2.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +5 -13
  2. data/README.md +2 -0
  3. data/chore-core.gemspec +1 -0
  4. data/lib/chore.rb +9 -5
  5. data/lib/chore/cli.rb +21 -2
  6. data/lib/chore/consumer.rb +15 -5
  7. data/lib/chore/fetcher.rb +12 -7
  8. data/lib/chore/hooks.rb +2 -1
  9. data/lib/chore/job.rb +17 -0
  10. data/lib/chore/manager.rb +17 -2
  11. data/lib/chore/queues/filesystem/consumer.rb +116 -59
  12. data/lib/chore/queues/filesystem/filesystem_queue.rb +19 -0
  13. data/lib/chore/queues/filesystem/publisher.rb +9 -15
  14. data/lib/chore/queues/sqs/consumer.rb +6 -21
  15. data/lib/chore/strategies/consumer/threaded_consumer_strategy.rb +1 -0
  16. data/lib/chore/strategies/consumer/throttled_consumer_strategy.rb +121 -0
  17. data/lib/chore/strategies/worker/forked_worker_strategy.rb +5 -6
  18. data/lib/chore/strategies/worker/helpers/ipc.rb +88 -0
  19. data/lib/chore/strategies/worker/helpers/preforked_worker.rb +163 -0
  20. data/lib/chore/strategies/worker/helpers/work_distributor.rb +65 -0
  21. data/lib/chore/strategies/worker/helpers/worker_info.rb +13 -0
  22. data/lib/chore/strategies/worker/helpers/worker_killer.rb +40 -0
  23. data/lib/chore/strategies/worker/helpers/worker_manager.rb +183 -0
  24. data/lib/chore/strategies/worker/preforked_worker_strategy.rb +150 -0
  25. data/lib/chore/util.rb +5 -1
  26. data/lib/chore/version.rb +3 -3
  27. data/lib/chore/worker.rb +27 -0
  28. data/spec/chore/cli_spec.rb +2 -2
  29. data/spec/chore/consumer_spec.rb +0 -4
  30. data/spec/chore/duplicate_detector_spec.rb +17 -5
  31. data/spec/chore/fetcher_spec.rb +0 -11
  32. data/spec/chore/manager_spec.rb +7 -0
  33. data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb +71 -11
  34. data/spec/chore/queues/sqs/consumer_spec.rb +1 -3
  35. data/spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb +165 -0
  36. data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb +5 -0
  37. data/spec/chore/strategies/worker/helpers/ipc_spec.rb +127 -0
  38. data/spec/chore/strategies/worker/helpers/preforked_worker_spec.rb +236 -0
  39. data/spec/chore/strategies/worker/helpers/work_distributor_spec.rb +131 -0
  40. data/spec/chore/strategies/worker/helpers/worker_info_spec.rb +14 -0
  41. data/spec/chore/strategies/worker/helpers/worker_killer_spec.rb +97 -0
  42. data/spec/chore/strategies/worker/helpers/worker_manager_spec.rb +304 -0
  43. data/spec/chore/strategies/worker/preforked_worker_strategy_spec.rb +183 -0
  44. data/spec/chore/worker_spec.rb +55 -0
  45. metadata +49 -21
checksums.yaml CHANGED
@@ -1,15 +1,7 @@
1
1
  ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- OWVhNTgxZTYxMzNkNDVjODU1NTI5YmJjOGYxOWU4NmUzNDkzMDljYw==
5
- data.tar.gz: !binary |-
6
- ZTliZWFkNzRhZDZlZDBiZWZlNjg4MDcxNDEzMTlhNDkwNmJkOWU4NA==
2
+ SHA256:
3
+ metadata.gz: 5aeebbb3efb30c9047c67996272864003542d9bd91d9887d3e149b6840f0573a
4
+ data.tar.gz: dd210d2eda88ddfab6e97646cb69ecccfc01a57a4198a8c2fc50c30d260dd32f
7
5
  SHA512:
8
- metadata.gz: !binary |-
9
- MDJkYmVlYWY4YTljMTA4ZjkxNGFlNGYxMDRiYThhNTJiYTJjNjhkYzVkNWFl
10
- YTUzNjBlMDIxYjM1MTcxNjE4NzU1YjdkNGVmOWY1Y2IzODg1MTQ5YjIxZGUx
11
- ZWI5N2U4ZGQwZGRhYTVkZTRiZmMxYzY0NGM1MjQ1YzM5NmJiZTM=
12
- data.tar.gz: !binary |-
13
- Mjc2ODk3ZmFjZjc2ZDdiODlkMTA4ODQzM2I1N2JiZDI5M2QwZjU2ODNkOWRl
14
- NGY4ZDg3NDRiMDAzMTUxNzM0MGQxNTNiNDM4ZmMwMmVhOTQzZTgwOTc4ZTc2
15
- MTg1YjI4Y2M1ZjE2YTM5ZDJiNTMwY2NkMjNjNTlmMTAyZmM1YTQ=
6
+ metadata.gz: 0df90580322b71a356a1b7bed8d1480c0794e6fcd12995bc82edfb3a878f40981fb87fefdcb2ad97a9841cabad2a151268a0e098d08f697cd56b727fcb973948
7
+ data.tar.gz: 24ef7ff568162c5f173ef599d687cd768c9b8a85948d5c80df10855fe30eac9df118e6a15b741e348e064e51ab693b395d81bf4e3689ca9519d9c15b69d05521
data/README.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # Chore: Job processing... for the future!
2
2
 
3
+ [![Build Status](https://travis-ci.org/Tapjoy/chore.svg?branch=master)](https://travis-ci.org/Tapjoy/chore)
4
+
3
5
  ## About
4
6
 
5
7
  Chore is a pluggable, multi-backend job processor. It was built from the ground up to be extremely flexible. We hope that you
@@ -39,6 +39,7 @@ Gem::Specification.new do |s|
39
39
  s.add_runtime_dependency(%q<json>, [">= 0"])
40
40
  s.add_runtime_dependency(%q<aws-sdk-v1>, ["~> 1.56", ">= 1.56.0"])
41
41
  s.add_runtime_dependency(%q<thread>, ["~> 0.1.3"])
42
+ s.add_runtime_dependency('get_process_mem', ["~> 0.2.0"])
42
43
  s.add_development_dependency(%q<rspec>, ["~> 3.3.0"])
43
44
  s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
44
45
  s.add_development_dependency(%q<bundler>, [">= 0"])
@@ -41,7 +41,11 @@ module Chore #:nodoc:
41
41
  :shutdown_timeout => (2 * 60),
42
42
  :max_attempts => 1.0 / 0.0, # Infinity
43
43
  :dupe_on_cache_failure => false,
44
- :payload_handler => Chore::Job
44
+ :queue_polling_size => 10,
45
+ :payload_handler => Chore::Job,
46
+ :master_procline => "chore-master-#{Chore::VERSION}",
47
+ :worker_procline => "chore-worker-#{Chore::VERSION}",
48
+ :consumer_sleep_interval => 1
45
49
  }
46
50
 
47
51
  class << self
@@ -111,9 +115,9 @@ module Chore #:nodoc:
111
115
  # add_hook(:before_fork) {|worker| puts 1 }
112
116
  # add_hook(:before_fork) {|worker| puts 2 }
113
117
  # add_hook(:before_fork) {|worker| puts 3 }
114
- #
118
+ #
115
119
  # run_hooks_for(:before_fork, worker)
116
- #
120
+ #
117
121
  # # ...will produce the following output
118
122
  # => 1
119
123
  # => 2
@@ -130,9 +134,9 @@ module Chore #:nodoc:
130
134
  # add_hook(:around_fork) {|worker, &block| puts 'before 1'; block.call; puts 'after 1'}
131
135
  # add_hook(:around_fork) {|worker, &block| puts 'before 2'; block.call; puts 'after 2'}
132
136
  # add_hook(:around_fork) {|worker, &block| puts 'before 3'; block.call; puts 'after 3'}
133
- #
137
+ #
134
138
  # run_hooks_for(:around_fork, worker) { puts 'block' }
135
- #
139
+ #
136
140
  # # ...will produce the following output
137
141
  # => before 1
138
142
  # => before 2
@@ -89,6 +89,7 @@ module Chore #:nodoc:
89
89
  detect_queues
90
90
  Chore.configure(options)
91
91
  Chore.configuring = false
92
+ validate_strategy!
92
93
  end
93
94
 
94
95
 
@@ -134,7 +135,7 @@ module Chore #:nodoc:
134
135
  options[:consumer_strategy] = constantize(arg)
135
136
  end
136
137
 
137
- register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages. Defaults vary depending on consumer implementation'
138
+ register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages (default: 1)'
138
139
 
139
140
  register_option 'payload_handler', '--payload_handler CLASS_NAME', 'Name of a class to use as the payload handler (default: Chore::Job)' do |arg|
140
141
  options[:payload_handler] = constantize(arg)
@@ -144,6 +145,7 @@ module Chore #:nodoc:
144
145
 
145
146
  register_option 'dupe_on_cache_failure', '--dupe-on-cache-failure BOOLEAN', 'Determines the deduping behavior when a cache connection error occurs. When set to false, the message is assumed not to be a duplicate. (default: false)'
146
147
 
148
+ register_option 'queue_polling_size', '--queue_polling_size NUM', Integer, 'Amount of messages to grab on each request (default: 10)'
147
149
  end
148
150
 
149
151
  def parse_opts(argv, ignore_errors = false) #:nodoc:
@@ -254,7 +256,6 @@ module Chore #:nodoc:
254
256
  end
255
257
 
256
258
  def validate! #:nodoc:
257
-
258
259
  missing_option!("--require [PATH|DIR]") unless options[:require]
259
260
 
260
261
  if !File.exist?(options[:require]) ||
@@ -266,7 +267,25 @@ module Chore #:nodoc:
266
267
  puts @parser
267
268
  exit(1)
268
269
  end
270
+ end
269
271
 
272
+ def validate_strategy!
273
+ consumer_strategy = Chore.config.consumer_strategy.to_s
274
+ worker_strategy = Chore.config.worker_strategy.to_s
275
+
276
+ throttled_consumer = 'Chore::Strategy::ThrottledConsumerStrategy'
277
+ preforked_worker = 'Chore::Strategy::PreForkedWorkerStrategy'
278
+
279
+ if consumer_strategy == throttled_consumer || worker_strategy == preforked_worker
280
+ unless consumer_strategy == throttled_consumer && worker_strategy == preforked_worker
281
+ puts "=================================================================="
282
+ puts " PreForkedWorkerStrategy may only be paired with "
283
+ puts " ThrottledConsumerStrategy or vice versa "
284
+ puts " Please check your configurations "
285
+ puts "=================================================================="
286
+ exit(1)
287
+ end
288
+ end
270
289
  end
271
290
  end
272
291
  end
@@ -21,11 +21,6 @@ module Chore
21
21
  def self.reset_connection!
22
22
  end
23
23
 
24
- # Cleans up any resources that were left behind from prior instances of the
25
- # chore process. By default, this is a no-op.
26
- def self.cleanup(queue)
27
- end
28
-
29
24
  # Consume takes a block with an arity of two. The two params are
30
25
  # |message_id,message_body| where message_id is any object that the
31
26
  # consumer will need to be able to act on a message later (reject, complete, etc)
@@ -53,5 +48,20 @@ module Chore
53
48
  def running?
54
49
  @running
55
50
  end
51
+
52
+ # returns up to n work
53
+ def provide_work(n)
54
+ raise NotImplementedError
55
+ end
56
+
57
+ # now, given an arbitrary key and klass, have we seen the key already?
58
+ def duplicate_message?(dedupe_key, klass, queue_timeout)
59
+ dupe_detector.found_duplicate?(:id=>dedupe_key, :queue=>klass.to_s, :visibility_timeout=>queue_timeout)
60
+ end
61
+
62
+ def dupe_detector
63
+ @dupes ||= DuplicateDetector.new({:servers => Chore.config.dedupe_servers,
64
+ :dupe_on_cache_failure => false})
65
+ end
56
66
  end
57
67
  end
@@ -11,21 +11,16 @@ module Chore
11
11
  # Starts the fetcher with the configured Consumer Strategy. This will begin consuming messages from your queue
12
12
  def start
13
13
  Chore.logger.info "Fetcher starting up"
14
-
15
- # Clean up configured queues in case there are any resources left behind
16
- Chore.config.queues.each do |queue|
17
- Chore.config.consumer.cleanup(queue)
18
- end
19
-
20
14
  @strategy.fetch
21
15
  end
22
16
 
23
17
  # Stops the fetcher, preventing any further messages from being pulled from the queue
24
18
  def stop!
25
19
  unless @stopping
26
- Chore.logger.info "Fetcher shutting down"
20
+ Chore.logger.info "Fetcher shutting down started"
27
21
  @stopping = true
28
22
  @strategy.stop!
23
+ Chore.logger.info "Fetcher shutting down completed"
29
24
  end
30
25
  end
31
26
 
@@ -33,5 +28,15 @@ module Chore
33
28
  def stopping?
34
29
  @stopping
35
30
  end
31
+
32
+ # returns upto n work units
33
+ def provide_work(n)
34
+ @strategy.provide_work(n)
35
+ end
36
+
37
+ # gives work back to the consumer in case it couldn't be assigned
38
+ def return_work(work_units)
39
+ @strategy.return_work(work_units)
40
+ end
36
41
  end
37
42
  end
@@ -15,7 +15,8 @@ module Chore
15
15
 
16
16
  private
17
17
  def hooks_for(event)
18
- candidate_methods.grep(/^#{event}/).sort
18
+ @_chore_hooks ||= {}
19
+ @_chore_hooks[event] ||= candidate_methods.grep(/^#{event}/).sort
19
20
  end
20
21
 
21
22
  # NOTE: Any hook methods defined after this is first referenced (i.e.,
@@ -60,6 +60,12 @@ module Chore
60
60
  raise ArgumentError, "#{self.to_s}: backoff must accept a single argument"
61
61
  end
62
62
  end
63
+
64
+ if @chore_options.key?(:dedupe_lambda)
65
+ if !@chore_options[:dedupe_lambda].is_a?(Proc)
66
+ raise ArgumentError, "#{self.to_s}: dedupe_lambda must be a lambda or Proc"
67
+ end
68
+ end
63
69
  end
64
70
 
65
71
  # This is a method so it can be overriden to create additional required
@@ -108,6 +114,17 @@ module Chore
108
114
  def has_backoff?
109
115
  self.options.key?(:backoff)
110
116
  end
117
+
118
+ def has_dedupe_lambda?
119
+ self.options.key?(:dedupe_lambda)
120
+ end
121
+
122
+ def dedupe_key(*args)
123
+ return unless has_dedupe_lambda?
124
+
125
+ # run the proc to get the key
126
+ self.options[:dedupe_lambda].call(*args).to_s
127
+ end
111
128
  end #ClassMethods
112
129
 
113
130
  # This is handy to override in an included job to be able to do job setup that requires
@@ -5,14 +5,15 @@ require 'chore/fetcher'
5
5
  module Chore
6
6
  # Manages the interactions between fetching messages (Consumer Strategy), and working over them (Worker Strategy)
7
7
  class Manager
8
+ include Util
8
9
 
9
10
  def initialize()
10
11
  Chore.logger.info "Booting Chore #{Chore::VERSION}"
11
12
  Chore.logger.debug { Chore.config.inspect }
13
+ procline("#{Chore.config.master_procline}:Started:#{Time.now}")
12
14
  @started_at = nil
13
15
  @worker_strategy = Chore.config.worker_strategy.new(self)
14
16
  @fetcher = Chore.config.fetcher.new(self)
15
- @processed = 0
16
17
  @stopping = false
17
18
  end
18
19
 
@@ -27,11 +28,12 @@ module Chore
27
28
  # Shut down the Manager, the Worker Strategy, and the Fetcher. This calls the +:before_shutdown+ hook.
28
29
  def shutdown!
29
30
  unless @stopping
30
- Chore.logger.info "Manager shutting down"
31
+ Chore.logger.info "Manager shutting down started"
31
32
  @stopping = true
32
33
  Chore.run_hooks_for(:before_shutdown)
33
34
  @fetcher.stop!
34
35
  @worker_strategy.stop!
36
+ Chore.logger.info "Manager shutting down completed"
35
37
  end
36
38
  end
37
39
 
@@ -42,7 +44,20 @@ module Chore
42
44
  # than they can be consumed.
43
45
  def assign(work)
44
46
  Chore.logger.debug { "Manager#assign: No. of UnitsOfWork: #{work.length})" }
47
+ work.each do | item |
48
+ Chore.run_hooks_for(:before_send_to_worker, item)
49
+ end
45
50
  @worker_strategy.assign(work) unless @stopping
46
51
  end
52
+
53
+ # returns up to n from the throttled consumer queue
54
+ def fetch_work(n)
55
+ @fetcher.provide_work(n)
56
+ end
57
+
58
+ # gives work back to the fetcher in case it couldn't be assigned
59
+ def return_work(work_units)
60
+ @fetcher.return_work(work_units)
61
+ end
47
62
  end
48
63
  end
@@ -21,55 +21,105 @@ module Chore
21
21
 
22
22
  Chore::CLI.register_option 'fs_queue_root', '--fs-queue-root DIRECTORY', 'Root directory for fs based queue'
23
23
 
24
- FILE_QUEUE_MUTEXES = {}
25
-
26
24
  class << self
27
- # Cleans up the in-progress files by making them new again. This should only
28
- # happen once per process.
29
- def cleanup(queue)
30
- new_dir = self.new_dir(queue)
31
- in_progress_dir = self.in_progress_dir(queue)
32
-
33
- job_files(in_progress_dir).each do |file|
34
- make_new_again(file, new_dir, in_progress_dir)
25
+ # Cleans up expired in-progress files by making them new again.
26
+ def cleanup(expiration_time, new_dir, in_progress_dir)
27
+ each_file(in_progress_dir) do |job_file|
28
+ id, previous_attempts, timestamp = file_info(job_file)
29
+ next if timestamp > expiration_time
30
+
31
+ begin
32
+ make_new_again(job_file, new_dir, in_progress_dir)
33
+ rescue Errno::ENOENT
34
+ # File no longer exists; skip since it's been recovered by another
35
+ # consumer
36
+ rescue ArgumentError
37
+ # Move operation was attempted at same time as another consumer;
38
+ # skip since the other process succeeded where this one didn't
39
+ end
35
40
  end
36
41
  end
37
42
 
38
- def make_in_progress(job, new_dir, in_progress_dir)
39
- move_job(File.join(new_dir, job), File.join(in_progress_dir, job))
43
+ # Moves job file to inprogress directory and returns the full path
44
+ # if the job was successfully locked by this consumer
45
+ def make_in_progress(job, new_dir, in_progress_dir, queue_timeout)
46
+ basename, previous_attempts, * = file_info(job)
47
+
48
+ from = File.join(new_dir, job)
49
+ # Add a timestamp to mark when the job was started
50
+ to = File.join(in_progress_dir, "#{basename}.#{previous_attempts}.#{Time.now.to_i}.job")
51
+
52
+ # If the file is non-zero, this means it was successfully written to
53
+ # by a publisher and we can attempt to move it to "in progress".
54
+ #
55
+ # There is a small window of time where the file can be zero, but
56
+ # the publisher hasn't finished writing to the file yet.
57
+ if !File.zero?(from)
58
+ File.open(from, "r") do |f|
59
+ # If the lock can't be obtained, that means it's been locked
60
+ # by another consumer or the publisher of the file) -- don't
61
+ # block and skip it
62
+ if f.flock(File::LOCK_EX | File::LOCK_NB)
63
+ FileUtils.mv(from, to)
64
+ to
65
+ end
66
+ end
67
+ elsif (Time.now - File.ctime(from)) >= queue_timeout
68
+ # The file is empty (zero bytes) and enough time has passed since
69
+ # the file was written that we can safely assume it will never
70
+ # get written to be the publisher.
71
+ #
72
+ # The scenario where this happens is when the publisher created
73
+ # the file, but the process was killed before it had a chance to
74
+ # actually write the data.
75
+ File.delete(from)
76
+ nil
77
+ end
78
+ rescue Errno::ENOENT
79
+ # File no longer exists; skip it since it's been picked up by
80
+ # another consumer
40
81
  end
41
82
 
83
+ # Moves job file to new directory and returns the full path
42
84
  def make_new_again(job, new_dir, in_progress_dir)
43
85
  basename, previous_attempts = file_info(job)
44
- move_job(File.join(in_progress_dir, job), File.join(new_dir, "#{basename}.#{previous_attempts + 1}.job"))
45
- end
46
86
 
47
- # Moves job file to inprogress directory and returns the full path
48
- def move_job(from, to)
49
- f = File.open(from, "r")
50
- # wait on the lock a publisher in another process might have.
51
- # Once we get the lock the file is ours to move to mark it in progress
52
- f.flock(File::LOCK_EX)
53
- begin
54
- FileUtils.mv(f.path, to)
55
- ensure
56
- f.flock(File::LOCK_UN) # yes we can unlock it after its been moved, I checked
57
- end
87
+ from = File.join(in_progress_dir, job)
88
+ to = File.join(new_dir, "#{basename}.#{previous_attempts + 1}.job")
89
+ FileUtils.mv(from, to)
90
+
58
91
  to
59
92
  end
60
93
 
61
- def job_files(dir)
62
- Dir.entries(dir).select{|e| ! e.start_with?(".")}
94
+ def each_file(path, limit = nil)
95
+ count = 0
96
+
97
+ Dir.foreach(path) do |file|
98
+ next if file.start_with?('.')
99
+
100
+ yield file
101
+
102
+ count += 1
103
+ break if limit && count >= limit
104
+ end
63
105
  end
64
106
 
65
107
  # Grabs the unique identifier for the job filename and the number of times
66
108
  # it's been attempted (also based on the filename)
67
109
  def file_info(job_file)
68
- id, previous_attempts = File.basename(job_file, '.job').split('.')
69
- [id, previous_attempts.to_i]
110
+ id, previous_attempts, timestamp, * = job_file.split('.')
111
+ [id, previous_attempts.to_i, timestamp.to_i]
70
112
  end
71
113
  end
72
114
 
115
+ # The minimum number of seconds to allow to pass between checks for expired
116
+ # jobs on the filesystem.
117
+ #
118
+ # Since queue times are measured on the order of seconds, 1 second is the
119
+ # smallest duration. It also prevents us from burning a lot of CPU looking
120
+ # at expired jobs when the consumer sleep interval is less than 1 second.
121
+ EXPIRATION_CHECK_INTERVAL = 1
122
+
73
123
  # The amount of time units of work can run before the queue considers
74
124
  # them timed out. For filesystem queues, this is the global default.
75
125
  attr_reader :queue_timeout
@@ -77,38 +127,46 @@ module Chore
77
127
  def initialize(queue_name, opts={})
78
128
  super(queue_name, opts)
79
129
 
80
- # Even though putting these Mutexes in this hash is, by itself, not particularly threadsafe
81
- # as long as some Mutex ends up in the queue after all consumers are created we're good
82
- # as they are pulled from the queue and synchronized for file operations below
83
- FILE_QUEUE_MUTEXES[@queue_name] ||= Mutex.new
84
-
85
130
  @in_progress_dir = self.class.in_progress_dir(queue_name)
86
131
  @new_dir = self.class.new_dir(queue_name)
87
- @queue_timeout = Chore.config.default_queue_timeout
132
+ @queue_timeout = self.class.queue_timeout(queue_name)
88
133
  end
89
134
 
90
- def consume(&handler)
135
+ def consume
91
136
  Chore.logger.info "Starting consuming file system queue #{@queue_name} in #{self.class.queue_dir(queue_name)}"
92
137
  while running?
93
138
  begin
94
- #TODO move expired job files to new directory?
95
- handle_jobs(&handler)
139
+ # Move expired job files to new directory (so long as enough time has
140
+ # passed since we last did this check)
141
+ if !@last_cleaned_at || (Time.now - @last_cleaned_at).to_i >= EXPIRATION_CHECK_INTERVAL
142
+ self.class.cleanup(Time.now.to_i - @queue_timeout, @new_dir, @in_progress_dir)
143
+ @last_cleaned_at = Time.now
144
+ end
145
+
146
+ found_files = false
147
+ handle_jobs do |*args|
148
+ found_files = true
149
+ yield(*args)
150
+ end
96
151
  rescue => e
97
152
  Chore.logger.error { "#{self.class}#consume: #{e} #{e.backtrace * "\n"}" }
98
153
  ensure
99
- sleep 5
154
+ sleep(Chore.config.consumer_sleep_interval) unless found_files
100
155
  end
101
156
  end
102
157
  end
103
158
 
159
+ # Rejects the given message from the filesystem by +id+. Currently a noop
104
160
  def reject(id)
105
- Chore.logger.debug "Rejecting: #{id}"
106
- make_new_again(id)
161
+
107
162
  end
108
163
 
109
164
  def complete(id)
110
165
  Chore.logger.debug "Completing (deleting): #{id}"
111
- FileUtils.rm_f(File.join(@in_progress_dir, id))
166
+ File.delete(File.join(@in_progress_dir, id))
167
+ rescue Errno::ENOENT
168
+ # The job took too long to complete, was deemed expired, and moved
169
+ # back into "new". Ignore.
112
170
  end
113
171
 
114
172
  private
@@ -116,27 +174,26 @@ module Chore
116
174
  # finds all new job files, moves them to in progress and starts the job
117
175
  # Returns a list of the job files processed
118
176
  def handle_jobs(&block)
119
- # all consumers on a single queue share a lock on handling files.
120
- # Each consumer comes along, processes all present files and release the lock.
121
- # This isn't particularly useful but is here to allow the configuration of
122
- # ThreadedConsumerStrategy with mutiple threads on a queue safely although you
123
- # probably wouldn't want to do that.
124
- FILE_QUEUE_MUTEXES[@queue_name].synchronize do
125
- self.class.job_files(@new_dir).each do |job_file|
126
- Chore.logger.debug "Found a new job #{job_file}"
127
-
128
- job_json = File.read(make_in_progress(job_file))
129
- basename, previous_attempts = self.class.file_info(job_file)
130
-
131
- # job_file is just the name which is the job id
132
- block.call(job_file, queue_name, queue_timeout, job_json, previous_attempts)
133
- Chore.run_hooks_for(:on_fetch, job_file, job_json)
134
- end
177
+ self.class.each_file(@new_dir, Chore.config.queue_polling_size) do |job_file|
178
+ Chore.logger.debug "Found a new job #{job_file}"
179
+
180
+ in_progress_path = make_in_progress(job_file)
181
+ next unless in_progress_path
182
+
183
+ # The job filename may have changed, so update it to reflect the in progress path
184
+ job_file = File.basename(in_progress_path)
185
+
186
+ job_json = File.read(in_progress_path)
187
+ basename, previous_attempts, * = self.class.file_info(job_file)
188
+
189
+ # job_file is just the name which is the job id
190
+ block.call(job_file, queue_name, queue_timeout, job_json, previous_attempts)
191
+ Chore.run_hooks_for(:on_fetch, job_file, job_json)
135
192
  end
136
193
  end
137
194
 
138
195
  def make_in_progress(job)
139
- self.class.make_in_progress(job, @new_dir, @in_progress_dir)
196
+ self.class.make_in_progress(job, @new_dir, @in_progress_dir, @queue_timeout)
140
197
  end
141
198
 
142
199
  def make_new_again(job)