chore-core 1.5.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +15 -0
  2. data/LICENSE.txt +20 -0
  3. data/README.md +260 -0
  4. data/Rakefile +32 -0
  5. data/bin/chore +34 -0
  6. data/chore-core.gemspec +46 -0
  7. data/lib/chore/cli.rb +232 -0
  8. data/lib/chore/configuration.rb +13 -0
  9. data/lib/chore/consumer.rb +52 -0
  10. data/lib/chore/duplicate_detector.rb +56 -0
  11. data/lib/chore/fetcher.rb +31 -0
  12. data/lib/chore/hooks.rb +25 -0
  13. data/lib/chore/job.rb +103 -0
  14. data/lib/chore/json_encoder.rb +18 -0
  15. data/lib/chore/manager.rb +47 -0
  16. data/lib/chore/publisher.rb +29 -0
  17. data/lib/chore/queues/filesystem/consumer.rb +128 -0
  18. data/lib/chore/queues/filesystem/filesystem_queue.rb +49 -0
  19. data/lib/chore/queues/filesystem/publisher.rb +45 -0
  20. data/lib/chore/queues/sqs/consumer.rb +121 -0
  21. data/lib/chore/queues/sqs/publisher.rb +55 -0
  22. data/lib/chore/queues/sqs.rb +38 -0
  23. data/lib/chore/railtie.rb +18 -0
  24. data/lib/chore/signal.rb +175 -0
  25. data/lib/chore/strategies/consumer/batcher.rb +76 -0
  26. data/lib/chore/strategies/consumer/single_consumer_strategy.rb +34 -0
  27. data/lib/chore/strategies/consumer/threaded_consumer_strategy.rb +81 -0
  28. data/lib/chore/strategies/worker/forked_worker_strategy.rb +221 -0
  29. data/lib/chore/strategies/worker/single_worker_strategy.rb +39 -0
  30. data/lib/chore/tasks/queues.task +11 -0
  31. data/lib/chore/unit_of_work.rb +17 -0
  32. data/lib/chore/util.rb +18 -0
  33. data/lib/chore/version.rb +9 -0
  34. data/lib/chore/worker.rb +117 -0
  35. data/lib/chore-core.rb +1 -0
  36. data/lib/chore.rb +218 -0
  37. data/spec/chore/cli_spec.rb +182 -0
  38. data/spec/chore/consumer_spec.rb +36 -0
  39. data/spec/chore/duplicate_detector_spec.rb +62 -0
  40. data/spec/chore/fetcher_spec.rb +38 -0
  41. data/spec/chore/hooks_spec.rb +44 -0
  42. data/spec/chore/job_spec.rb +80 -0
  43. data/spec/chore/json_encoder_spec.rb +11 -0
  44. data/spec/chore/manager_spec.rb +39 -0
  45. data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb +71 -0
  46. data/spec/chore/queues/sqs/consumer_spec.rb +136 -0
  47. data/spec/chore/queues/sqs/publisher_spec.rb +74 -0
  48. data/spec/chore/queues/sqs_spec.rb +37 -0
  49. data/spec/chore/signal_spec.rb +244 -0
  50. data/spec/chore/strategies/consumer/batcher_spec.rb +93 -0
  51. data/spec/chore/strategies/consumer/single_consumer_strategy_spec.rb +23 -0
  52. data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb +105 -0
  53. data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb +281 -0
  54. data/spec/chore/strategies/worker/single_worker_strategy_spec.rb +36 -0
  55. data/spec/chore/worker_spec.rb +134 -0
  56. data/spec/chore_spec.rb +108 -0
  57. data/spec/spec_helper.rb +58 -0
  58. data/spec/test_job.rb +7 -0
  59. metadata +194 -0
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ ZDg1NjQ5Yzk0YzUwOGI5NDYxNWYyYmVhNTU4MmY5MTk2NWFjZjZmNA==
5
+ data.tar.gz: !binary |-
6
+ YmU5ZjI5YTFlNWUyODE3NzZlYzlkNjQ5NTFlZDViOWI4ZTZiYzU3Yg==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ OWRlNjA2ZTI2Mjg2ZGRiYjk5MzY4Nzk1ZjNlMTMzM2UyMzU5NTdhZDI0ZDI2
10
+ Nzg1NTRkNGViZWI4NjgxNzkwYWQ4ZDM1YWU4MmQyMTgwNzRhMmYwMDRlYmMx
11
+ OTM3MDY5NWY5YjllZmZiM2I2NDJjZGE4NWRlZmY3MDcyYTI3Njg=
12
+ data.tar.gz: !binary |-
13
+ ODk0ZjIxYTI2YTU1MWQ0M2RiYzIyOGE4MGJjNTJkOTI1MmNhYjk4ZDFhODZl
14
+ YTQ5ZDIwMjAzOTEyY2JjYzBjNTZlMmFhYmEyZjYwYTU3Y2ZiMWUwYjZiYmRh
15
+ ZDg1YTg4MjBlZTkyNjQyMmRkNTZmZDkwMDBmYjFiNWJmYThiYjE=
data/LICENSE.txt ADDED
@@ -0,0 +1,20 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2014, Tapjoy, Inc.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
6
+ this software and associated documentation files (the "Software"), to deal in
7
+ the Software without restriction, including without limitation the rights to
8
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9
+ the Software, and to permit persons to whom the Software is furnished to do so,
10
+ subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,260 @@
1
+ # Chore: Job processing... for the future!
2
+
3
+ ## About
4
+
5
+ Chore is a pluggable, multi-backend job processor. It was built from the ground up to be extremely flexible. We hope that you
6
+ will find integrating and using Chore to be as pleasant as we do.
7
+
8
+ The full docs for Chore can always be found at http://tapjoy.github.io/chore.
9
+
10
+ ## Configuration
11
+
12
+ Chore can be integrated with any Ruby-based project by following these instructions:
13
+
14
+ gem 'chore-core', '~> 1.5'
15
+
16
+ If you also plan on using SQS, you must also bring in dalli to use for memcached:
17
+
18
+ gem 'dalli'
19
+
20
+ Create a `Chorefile` file in the root of your project directory. While you can configure Chore itself from this file, it's primarly used to direct the Chore binary toward the root of your application, so that it can locate all of the depdendencies and required code.
21
+
22
+ --require=./<FILE_TO_LOAD>
23
+
24
+ Make sure that `--require` points to the main entry point for your app. If integrating with a Rails app, just point it to the directory of your application and it will handle loading the correct files on its own.
25
+
26
+ Other options include:
27
+
28
+ --concurrency 16 # number of concurrent worker processes, if using forked worker strategy
29
+ --worker-strategy Chore::Strategy::ForkedWorkerStrategy # which worker strategy class to use
30
+ --consumer Chore::Queues::SQS::Consumer # which consumer class to use Options are SQS::Consumer and Filesystem::Consumer. Filesystem is recommended for local and testing purposes only.
31
+ --consumer-strategy Chore::Queues::Strategies::Consumer::ThreadedConsumerStrategy # which consuming strategy to use. Options are SingleConsumerStrategy and ThreadedConsumerStrategy. Threaded is recommended for better tuning your consuming profile
32
+ --threads-per-queue 4 # number of threads per queue for consuming from a given queue.
33
+ --dedupe-servers # if using SQS or similiar queue with at-least once delivery and your memcache is running on something other than localhost
34
+ --batch-size 50 # how many messages are batched together before handing them to a worker
35
+ --queue_prefix prefixy # A prefix to prepend to queue names, mainly for development and qa testing purposes
36
+ --max-attempts 100 # The maximum number of times a job can be attempted
37
+ --dupe-on-cache-failure # Determines the deduping behavior when a cache connection error occurs. When set to `false`, the message is assumed not to be a duplicate. Defaults to `false`.
38
+ --queue-polling-size 10 # If your particular queueing system supports responding with messages in batches of a certain size, you can control that with this flag. SQS has a built in upper-limit of 10, but other systems will vary.
39
+
40
+ If you're using SQS, you'll want to add AWS keys so that Chore can authenticate with AWS.
41
+
42
+ --aws-access-key=<AWS KEY>
43
+ --aws-secret-key=<AWS SECRET>
44
+
45
+ By default, Chore will run over all queues it detects among the required files. If you wish to change this behavior, you can use:
46
+
47
+ --queues QUEUE1,QUEUE2... # a list of queues to process
48
+ --except-queues QUEUE1,QUEUE2... # a list of queues _not_ to process
49
+
50
+ Note that you can use one or the other but not both. Chore will quit and make fun of you if both options are specified.
51
+
52
+ ### Tips for configuring Chore
53
+
54
+ When it comes to configuring Chore, you have 2 main use cases - as a producer of messages, or as a consumer of messages (the consumer is also able to produce messages if need be, but is running as it's own isolated instance of your application).
55
+
56
+ For producers, you must do all of your Chore configuration in an intializer.
57
+
58
+ For consumers, you need to either Chorefile or Chorefile + an initializer.
59
+
60
+ Because you are likely to use the same app as the basis for both producing and consuming messages, you'll already have a considerable amount of configuration in your Producer - it makes sense to use Chorefile to simply provide the `require` option, and stick to the initializer for the rest of the configuration to keep things DRY.
61
+
62
+ However, like many aspects of Chore, it is ultimately up to the developer to decide which use case fits their needs best. Chore is happy to let you configure it in almost any way you want.
63
+
64
+ An example of how to configure chore via and initializer:
65
+
66
+ ```ruby
67
+ Chore.configure do |c|
68
+ c.concurrency = 16
69
+ c.worker_strategy = Chore::Strategy::ForkedWorkerStrategy
70
+ c.max_attempts = 100
71
+ ...
72
+ c.batch_size = 50
73
+ end
74
+ ```
75
+
76
+ ## Integration
77
+
78
+ Add an appropriate line to your `Procfile`:
79
+
80
+ jobs: bundle exec chore -c config/chore.config
81
+
82
+ If your queues do not exist, you must create them before you run the application:
83
+
84
+ ```ruby
85
+ require 'aws-sdk'
86
+ sqs = AWS::SQS.new
87
+ sqs.queues.create("test_queue")
88
+ ```
89
+
90
+ Finally, start foreman as usual
91
+
92
+ bundle exec foreman start
93
+
94
+ ## Chore::Job
95
+
96
+ A Chore::Job is any class that includes `Chore::Job` and implements `perform(*args)` Here is an example job class:
97
+
98
+ ```ruby
99
+ class TestJob
100
+ include Chore::Job
101
+ queue_options :name => 'test_queue'
102
+
103
+ def perform(args={})
104
+ Chore.logger.debug "My first async job"
105
+ end
106
+
107
+ end
108
+ ```
109
+
110
+ This job declares that the name of the queue it uses is `test_queue`, set in the queue_options method.
111
+
112
+ ### Chore::Job and perform signatures
113
+
114
+ The perform method signature can have explicit argument names, but in practice this makes changing the signature more difficult later on. Once a Job is in production and is being used at a constant rate, it becomes problematic to begin mixing versions of jobs which have non-matching signatures.
115
+
116
+ While this is able to be overcome with a number of techniques, such as versioning your jobs/queues, it increases the complexity of making changes.
117
+
118
+ The simplest way to structure job signatures is to treat the arguments as a hash. This will allow you to maintain forwards and backwards compatibility between signature changes with the same job class.
119
+
120
+ However, Chore is ultimately agnostic to your particular needs in this regard, and will let you use explicit arguments in your signatures as easily as you can use a simple hash - the choice is left to you, the developer.
121
+
122
+ ### Chore::Job and publishing Jobs
123
+
124
+ Now that you've got a test job, if you wanted to publish to that job it's as simple as:
125
+ ```ruby
126
+ TestJob.perform_async({"message"=>"YES, DO THAT THING."})
127
+ ```
128
+
129
+ It's advisable to specify the Publisher chore uses to send messages globally, so that you can change it easily for local and test environments. To do this, you can add a configuration block to an initializer like so:
130
+
131
+ ```ruby
132
+ Chore.configure do |c|
133
+ c.publisher = Some::Other::Publisher
134
+ end
135
+ ```
136
+
137
+ It is worth noting that any option that can be set via config file or command-line args can also be set in a configure block.
138
+
139
+ If a global publisher is set, it can be overridden on a per-job basis by specifying the publisher in `queue_options`.
140
+
141
+
142
+ ## Hooks
143
+
144
+ A number of hooks, both global and per-job, exist in Chore for your convenience.
145
+
146
+ Global Hooks:
147
+
148
+ * before_first_fork
149
+ * before_fork
150
+ * after_fork
151
+ * around_fork
152
+ * within_fork
153
+
154
+ ("within_fork" behaves similarly to around_fork, except that it is called after the worker process has been forked. In contrast, around_fork is called by the parent process.)
155
+
156
+ Filesystem Consumer/Publisher
157
+
158
+ * on_fetch(job_file, job_json)
159
+
160
+ SQS Consumer
161
+
162
+ * on_fetch(handle, body)
163
+
164
+ Per Job:
165
+
166
+ * before_publish
167
+ * after_publish
168
+ * before_perform(message)
169
+ * after_perform(message)
170
+ * on_rejected(message)
171
+ * on_failure(message, error)
172
+ * on_permanent_failure(queue_name, message, error)
173
+
174
+ All per-job hooks can also be global hooks.
175
+
176
+ Hooks can be added to a job class as so:
177
+
178
+ ```ruby
179
+ class TestJob
180
+ include Chore::Job
181
+ queue_options :name => 'test_queue'
182
+
183
+ def perform(args={})
184
+ Chore.logger.debug "My first sync job"
185
+ end
186
+ end
187
+ ```
188
+ Global hooks can also be registered like so:
189
+
190
+ ```ruby
191
+ Chore.add_hook :after_publish do
192
+ # your special handler here
193
+ end
194
+ ```
195
+
196
+ ## Signals
197
+
198
+ Signal handling can get complicated when you have multiple threads, process
199
+ forks, and both signal handlers and application code making use of mutexes.
200
+
201
+ To simplify the complexities around this, Chore introduces some additional
202
+ behaviors on top of Ruby's default Signal.trap implementation. This
203
+ functionality is primarily inspired by sidekiq's signal handling @
204
+ https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb.
205
+
206
+ In particular, Chore handles signals in a separate thread and does so
207
+ sequentially instead of interrupt-driven. See Chore::Signal for more details
208
+ on the differences between Ruby's `Signal.trap` and Chore's `Chore::Signal.trap`.
209
+
210
+ Chore will respond to the following Signals:
211
+
212
+ * INT , TERM, QUIT - Chore will begin shutting down, taking steps to safely terminate workers and not interrupt jobs in progress unless it believes they may be hung
213
+ * USR1 - Re-opens logfiles, useful for handling log rotations
214
+
215
+ ## Timeouts
216
+
217
+ When using the forked worker strategy for processing jobs, inevitably there are
218
+ cases in which child processes become stuck. This could result from deadlocks,
219
+ hung network calls, tight loops, etc. When these jobs hang, they consume
220
+ resources and can affect throughput.
221
+
222
+ To mitigate this, Chore has built-in monitoring of forked child processes.
223
+ When a fork is created to process a batch of work, that fork is assigned an
224
+ expiration time -- if it doesn't complete by that time, the process is sent
225
+ a KILL signal.
226
+
227
+ Fork expiration times are determined from one of two places:
228
+ 1. The timeout associated with the queue. For SQS, this is the visibility
229
+ timeout.
230
+ 2. The default queue timeout configured for Chore. For Filesystem queues,
231
+ this is the value used.
232
+
233
+ For example, if a worker is processing a batch of 5 jobs and each job's queue
234
+ has a timeout of 60s, then the expiration time will be 5 minutes for the worker.
235
+
236
+ To change the default queue timeout (when one can't be inferred), you can do
237
+ the following:
238
+
239
+ ```ruby
240
+ Chore.configure do |c|
241
+ c.default_queue_timeout = 3600
242
+ end
243
+ ```
244
+
245
+ A reasonable timeout would be based on the maximum amount of time you expect any
246
+ job in your system to run. Keep in mind that the process running the job may
247
+ get killed if the job is running for too long.
248
+
249
+ ## Plugins
250
+
251
+ Chore has several plugin gems available, which extend it's core functionality
252
+
253
+ [New Relic](https://github.com/Tapjoy/chore-new_relic) - Integrating Chore with New Relic
254
+
255
+ [Airbrake](https://github.com/Tapjoy/chore-airbrake) - Integrating Chore with Airbrake
256
+
257
+ ## Copyright
258
+
259
+ Copyright (c) 2013 - 2014 Tapjoy. See LICENSE.txt for
260
+ further details.
data/Rakefile ADDED
@@ -0,0 +1,32 @@
1
+ # encoding: utf-8
2
+
3
+ require 'rubygems'
4
+ require 'bundler'
5
+ begin
6
+ Bundler.setup(:default, :development)
7
+ rescue Bundler::BundlerError => e
8
+ $stderr.puts e.message
9
+ $stderr.puts "Run `bundle install` to install missing gems"
10
+ exit e.status_code
11
+ end
12
+ require 'rake'
13
+
14
+ require 'rspec/core'
15
+ require 'rspec/core/rake_task'
16
+ RSpec::Core::RakeTask.new(:spec) do |spec|
17
+ spec.pattern = FileList['spec/**/*_spec.rb']
18
+ end
19
+
20
+ #RSpec::Core::RakeTask.new(:rcov) do |spec|
21
+ # spec.pattern = 'spec/**/*_spec.rb'
22
+ # spec.rcov = true
23
+ #end
24
+
25
+ task :default => :spec
26
+
27
+ require 'yard'
28
+
29
+ YARD::Rake::YardocTask.new do |yard|
30
+ require 'chore/version'
31
+ yard.options = [ '--output=rdoc', "title=#{Chore::Version::STRING}", 'main=README.md']
32
+ end
data/bin/chore ADDED
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
4
+ require 'chore'
5
+ require 'chore/cli'
6
+ require 'chore/signal'
7
+
8
+ # This is a pure-ruby patch of something that resolves hostnames when making external calls
9
+ # Without it, we have a chance to fork while the lock is held, which results in dead forks
10
+ # This is not included anywhere else because this is the only Chore-specific code that is not
11
+ # included in other projects by requiring chore, where this patch may be undesirable.
12
+ require 'resolv-replace'
13
+
14
+ ["INT","TERM","QUIT"].each do |sig|
15
+ Chore::Signal.trap sig do
16
+ Chore::CLI.instance.shutdown
17
+ end
18
+ end
19
+
20
+ Chore::Signal.trap "USR1" do
21
+ Chore.reopen_logs
22
+ end
23
+
24
+ begin
25
+ cli = Chore::CLI.instance
26
+ cli.run!(ARGV)
27
+ rescue => e
28
+ raise e if $DEBUG
29
+ STDERR.puts e.message
30
+ STDERR.puts e.backtrace.join("\n")
31
+ exit 1
32
+ end
33
+
34
+
@@ -0,0 +1,46 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $: << File.expand_path('lib', File.dirname(__FILE__))
3
+
4
+ require 'chore/version'
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = "chore-core"
8
+ s.version = Chore::Version::STRING
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["Tapjoy"]
12
+ s.date = Time.new.strftime("%Y-%m-%d")
13
+ s.description = "Job processing with pluggable backends and strategies"
14
+ s.email = "eng-group-arch@tapjoy.com"
15
+
16
+ s.executables = Dir["bin/*"].map { |f| f.gsub(/bin\//, '') }
17
+ s.default_executable = "chore"
18
+
19
+ s.extra_rdoc_files = [
20
+ "LICENSE.txt",
21
+ "README.md"
22
+ ]
23
+ s.files = Dir[*%w(
24
+ chore-core.gemspec
25
+ LICENSE.txt
26
+ README.md
27
+ Rakefile
28
+ bin/*
29
+ lib/**/*
30
+ spec/**/*
31
+ )]
32
+
33
+ s.homepage = "http://github.com/Tapjoy/chore"
34
+ s.licenses = ["MIT"]
35
+ s.require_paths = ["lib"]
36
+ s.rubygems_version = "1.8.25"
37
+ s.summary = "Job processing... for the future!"
38
+
39
+ s.add_runtime_dependency(%q<json>, [">= 0"])
40
+ s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.12", ">= 1.12.0"])
41
+ s.add_runtime_dependency(%q<thread>, ["~> 0.1.3"])
42
+ s.add_development_dependency(%q<rspec>, ["~> 2.12.0"])
43
+ s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
44
+ s.add_development_dependency(%q<bundler>, [">= 0"])
45
+ end
46
+
data/lib/chore/cli.rb ADDED
@@ -0,0 +1,232 @@
1
+ require 'pp'
2
+ require 'singleton'
3
+ require 'optparse'
4
+ require 'chore'
5
+ require 'erb'
6
+ require 'set'
7
+
8
+ require 'chore/manager'
9
+
10
+ module Chore #:nodoc:
11
+
12
+ # Class that handles the command line interactions in Chore.
13
+ # It primarily is responsible for invoking the Chore process with the provided configuration
14
+ # to begin processing jobs.
15
+ class CLI
16
+ include Singleton
17
+ include Util
18
+
19
+ attr_reader :options, :registered_opts
20
+
21
+ def initialize
22
+ @options = {}
23
+ @registered_opts = {}
24
+ @stopping = false
25
+ end
26
+
27
+ #
28
+ # +register_option+ is a method for plugins or other components to register command-line config options.
29
+ # * <tt>key</tt> is the name for this option that can be referenced from Chore.config.+key+
30
+ # * <tt>*args</tt> is an <tt>OptionParser</tt> style list of options.
31
+ # * <tt>&blk</tt> is an option block, passed to <tt>OptionParser</tt>
32
+ #
33
+ # === Examples
34
+ # Chore::CLI.register_option 'sample', '-s', '--sample-key SOME_VAL', 'A description of this value'
35
+ #
36
+ # Chore::CLI.register_option 'something', '-g', '--something-complex VALUE', 'A description' do |arg|
37
+ # # make sure your key here matches the key you register
38
+ # options[:something] arg.split(',')
39
+ # end
40
+ def self.register_option(key,*args,&blk)
41
+ instance.register_option(key,*args,&blk)
42
+ end
43
+
44
+ def register_option(key,*args,&blk) #:nodoc:
45
+ registered_opts[key] = {:args => args}
46
+ registered_opts[key].merge!(:block => blk) if blk
47
+ end
48
+
49
+ # Start up the consuming side of the application. This calls Chore::Manager#start.
50
+ def run!(args=ARGV)
51
+ parse(args)
52
+ @manager = Chore::Manager.new
53
+ @manager.start
54
+ end
55
+
56
+ # Begins the Chore shutdown process. This will call Chore::Manager#shutdown if it is not already in the process of stopping
57
+ # Exits with code 0
58
+ def shutdown
59
+ unless @stopping
60
+ @stopping = true
61
+ @manager.shutdown! if @manager
62
+ exit(0)
63
+ end
64
+ end
65
+
66
+ def parse_config_file(file) #:nodoc:
67
+ data = File.read(file)
68
+ data = ERB.new(data).result
69
+ parse_opts(data.split(/\s/).map!(&:chomp).map!(&:strip))
70
+ end
71
+
72
+ def parse(args=ARGV) #:nodoc:
73
+ Chore.configuring = true
74
+ setup_options
75
+
76
+ # parse once to load the config file & require options
77
+ parse_opts(args)
78
+ parse_config_file(@options[:config_file]) if @options[:config_file]
79
+
80
+ validate!
81
+ boot_system
82
+
83
+ # parse again to pick up options required by loaded classes
84
+ parse_opts(args)
85
+ parse_config_file(@options[:config_file]) if @options[:config_file]
86
+ detect_queues
87
+ Chore.configure(options)
88
+ Chore.configuring = false
89
+ end
90
+
91
+
92
+ private
93
+ def setup_options #:nodoc:
94
+ register_option "queues", "-q", "--queues QUEUE1,QUEUE2", "Names of queues to process (default: all known)" do |arg|
95
+ options[:queues] = arg.split(",")
96
+ end
97
+
98
+ register_option "except_queues", "-x", "--except QUEUE1,QUEUE2", "Process all queues (cannot specify --queues), except for the ones listed here" do |arg|
99
+ options[:except_queues] = arg.split(",")
100
+ end
101
+
102
+ register_option "verbose", "-v", "--verbose", "Print more verbose output. Use twice to increase." do
103
+ options[:log_level] ||= Logger::WARN
104
+ options[:log_level] = options[:log_level] - 1 if options[:log_level] > 0
105
+ end
106
+
107
+ register_option "environment", '-e', '--environment ENV', "Application environment"
108
+
109
+ register_option "config_file", '-c', '--config-file FILE', "Location of a file specifying additional chore configuration"
110
+
111
+ register_option 'require', '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require"
112
+
113
+ register_option 'num_workers', '--concurrency NUM', Integer, 'Number of workers to run concurrently'
114
+
115
+ register_option 'queue_prefix', '--queue-prefix PREFIX', "Prefix to use on Queue names to prevent non-determinism in testing environments" do |arg|
116
+ options[:queue_prefix] = arg.downcase << "_"
117
+ end
118
+
119
+ register_option 'max_attempts', '--max-attempts NUM', Integer, 'Number of times to attempt failed jobs'
120
+
121
+ register_option 'worker_strategy', '--worker-strategy CLASS_NAME', 'Name of a class to use as the worker strategy (default: ForkedWorkerStrategy' do |arg|
122
+ options[:worker_strategy] = constantize(arg)
123
+ end
124
+
125
+ register_option 'consumer', '--consumer CLASS_NAME', 'Name of a class to use as the queue consumer (default: SqsConsumer)' do |arg|
126
+ options[:consumer] = constantize(arg)
127
+ end
128
+
129
+ register_option 'consumer_strategy', '--consumer-strategy CLASS_NAME', 'Name of a class to use as the consumer strategy (default: Chore::Strategy::ThreadedConsumerStrategy' do |arg|
130
+ options[:consumer_strategy] = constantize(arg)
131
+ end
132
+
133
+ register_option 'shutdown_timeout', '--shutdown-timeout SECONDS', Float, "Upon shutdown, the number of seconds to wait before force killing worker strategies (default: #{Chore::DEFAULT_OPTIONS[:shutdown_timeout]})"
134
+
135
+ register_option 'dupe_on_cache_failure', '--dupe-on-cache-failure BOOLEAN', 'Determines the deduping behavior when a cache connection error occurs. When set to false, the message is assumed not to be a duplicate. (default: false)'
136
+
137
+ end
138
+
139
+ def parse_opts(argv) #:nodoc:
140
+ @options ||= {}
141
+ @parser = OptionParser.new do |o|
142
+ registered_opts.each do |key,opt|
143
+ if opt[:block]
144
+ o.on(*opt[:args],&opt[:block])
145
+ else
146
+ o.on(*opt[:args]) do |arg|
147
+ options[key.to_sym] = arg
148
+ end
149
+ end
150
+ end
151
+ end
152
+
153
+ @parser.banner = "chore [options]"
154
+
155
+ @parser.on_tail "-h", "--help", "Show help" do
156
+ puts @parser
157
+ exit 1
158
+ end
159
+
160
+ @parser.parse!(argv)
161
+
162
+ @options
163
+ end
164
+
165
+
166
+ def detected_environment #:nodoc:
167
+ options[:environment] ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
168
+ end
169
+
170
+ def boot_system #:nodoc:
171
+ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = detected_environment
172
+
173
+ raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
174
+
175
+ if File.directory?(options[:require])
176
+ require 'rails'
177
+ require 'chore/railtie'
178
+ require File.expand_path("#{options[:require]}/config/environment.rb")
179
+ ::Rails.application.eager_load!
180
+ else
181
+ require File.expand_path(options[:require])
182
+ end
183
+ end
184
+
185
+ def detect_queues #:nodoc:
186
+ if (options[:queues] && options[:except_queues])
187
+ raise ArgumentError, "Cannot specify both --except and --queues"
188
+ end
189
+
190
+ if !options[:queues]
191
+ options[:queues] = Set.new
192
+ Chore::Job.job_classes.each do |j|
193
+ klazz = constantize(j)
194
+ options[:queues] << klazz.options[:name] if klazz.options[:name]
195
+ options[:queues] -= (options[:except_queues] || [])
196
+ end
197
+ end
198
+
199
+ original_queues = options[:queues].dup
200
+ # Now apply the prefixing
201
+ # Because the prefix could have been detected via the apps chore config file
202
+ # Lets see if that is present before we check for a CLI passed prefix
203
+ prefix = Chore.config.queue_prefix || options[:queue_prefix]
204
+ options[:queues] = Set.new.tap do |queue_set|
205
+ original_queues.each {|oq_name| queue_set << "#{prefix}#{oq_name}"}
206
+ end
207
+ raise ArgumentError, "No queues specified. Either include classes that include Chore::Job, or specify the --queues option" if options[:queues].empty?
208
+ end
209
+
210
+ def missing_option!(option) #:nodoc:
211
+ puts "Missing argument: #{option}"
212
+ exit(255)
213
+ end
214
+
215
+ def validate! #:nodoc:
216
+
217
+ missing_option!("--require [PATH|DIR]") unless options[:require]
218
+
219
+ if !File.exist?(options[:require]) ||
220
+ (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
221
+ puts "=================================================================="
222
+ puts " Please point chore to a Rails 3 application or a Ruby file "
223
+ puts " to load your worker classes with -r [DIR|FILE]."
224
+ puts "=================================================================="
225
+ puts @parser
226
+ exit(1)
227
+ end
228
+
229
+ end
230
+ end
231
+ end
232
+
@@ -0,0 +1,13 @@
1
+ module Chore
2
+ # Wrapper around an OpenStruct to define configuration data
3
+ # (TODO): Add required opts, and validate that they're set
4
+ class Configuration < OpenStruct
5
+ # Helper method to make merging Hashes into OpenStructs easier
6
+ def merge_hash(hsh={})
7
+ hsh.keys.each do |k|
8
+ self.send("#{k.to_sym}=",hsh[k])
9
+ end
10
+ self
11
+ end
12
+ end
13
+ end