chore-core 1.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +15 -0
  2. data/LICENSE.txt +20 -0
  3. data/README.md +260 -0
  4. data/Rakefile +32 -0
  5. data/bin/chore +34 -0
  6. data/chore-core.gemspec +46 -0
  7. data/lib/chore/cli.rb +232 -0
  8. data/lib/chore/configuration.rb +13 -0
  9. data/lib/chore/consumer.rb +52 -0
  10. data/lib/chore/duplicate_detector.rb +56 -0
  11. data/lib/chore/fetcher.rb +31 -0
  12. data/lib/chore/hooks.rb +25 -0
  13. data/lib/chore/job.rb +103 -0
  14. data/lib/chore/json_encoder.rb +18 -0
  15. data/lib/chore/manager.rb +47 -0
  16. data/lib/chore/publisher.rb +29 -0
  17. data/lib/chore/queues/filesystem/consumer.rb +128 -0
  18. data/lib/chore/queues/filesystem/filesystem_queue.rb +49 -0
  19. data/lib/chore/queues/filesystem/publisher.rb +45 -0
  20. data/lib/chore/queues/sqs/consumer.rb +121 -0
  21. data/lib/chore/queues/sqs/publisher.rb +55 -0
  22. data/lib/chore/queues/sqs.rb +38 -0
  23. data/lib/chore/railtie.rb +18 -0
  24. data/lib/chore/signal.rb +175 -0
  25. data/lib/chore/strategies/consumer/batcher.rb +76 -0
  26. data/lib/chore/strategies/consumer/single_consumer_strategy.rb +34 -0
  27. data/lib/chore/strategies/consumer/threaded_consumer_strategy.rb +81 -0
  28. data/lib/chore/strategies/worker/forked_worker_strategy.rb +221 -0
  29. data/lib/chore/strategies/worker/single_worker_strategy.rb +39 -0
  30. data/lib/chore/tasks/queues.task +11 -0
  31. data/lib/chore/unit_of_work.rb +17 -0
  32. data/lib/chore/util.rb +18 -0
  33. data/lib/chore/version.rb +9 -0
  34. data/lib/chore/worker.rb +117 -0
  35. data/lib/chore-core.rb +1 -0
  36. data/lib/chore.rb +218 -0
  37. data/spec/chore/cli_spec.rb +182 -0
  38. data/spec/chore/consumer_spec.rb +36 -0
  39. data/spec/chore/duplicate_detector_spec.rb +62 -0
  40. data/spec/chore/fetcher_spec.rb +38 -0
  41. data/spec/chore/hooks_spec.rb +44 -0
  42. data/spec/chore/job_spec.rb +80 -0
  43. data/spec/chore/json_encoder_spec.rb +11 -0
  44. data/spec/chore/manager_spec.rb +39 -0
  45. data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb +71 -0
  46. data/spec/chore/queues/sqs/consumer_spec.rb +136 -0
  47. data/spec/chore/queues/sqs/publisher_spec.rb +74 -0
  48. data/spec/chore/queues/sqs_spec.rb +37 -0
  49. data/spec/chore/signal_spec.rb +244 -0
  50. data/spec/chore/strategies/consumer/batcher_spec.rb +93 -0
  51. data/spec/chore/strategies/consumer/single_consumer_strategy_spec.rb +23 -0
  52. data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb +105 -0
  53. data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb +281 -0
  54. data/spec/chore/strategies/worker/single_worker_strategy_spec.rb +36 -0
  55. data/spec/chore/worker_spec.rb +134 -0
  56. data/spec/chore_spec.rb +108 -0
  57. data/spec/spec_helper.rb +58 -0
  58. data/spec/test_job.rb +7 -0
  59. metadata +194 -0
@@ -0,0 +1,52 @@
1
+ module Chore
2
+ # Raised when Chore is booting up, but encounters a set of configuration that is impossible to boot from. Typically
3
+ # you'll find additional information around the cause of the exception by examining the logfiles
4
+ class TerribleMistake < Exception
5
+ # You can raise this exception if your queue is in a terrible state and must shut down
6
+ end
7
+
8
+ # Base class for a Chore Consumer. Provides the basic interface to adhere to for building custom
9
+ # Chore Consumers.
10
+ class Consumer
11
+
12
+ attr_accessor :queue_name
13
+
14
+ def initialize(queue_name, opts={})
15
+ @queue_name = queue_name
16
+ @running = true
17
+ end
18
+
19
+ # Causes the underlying connection for all consumers of this class to be reset. Useful for the case where
20
+ # the consumer is being used across a fork. Should be overriden in consumers (but is not required).
21
+ def self.reset_connection!
22
+ end
23
+
24
+ # Consume takes a block with an arity of two. The two params are
25
+ # |message_id,message_body| where message_id is any object that the
26
+ # consumer will need to be able to act on a message later (reject, complete, etc)
27
+ def consume(&block)
28
+ raise NotImplementedError
29
+ end
30
+
31
+ # Reject should put a message back on a queue to be processed again later. It takes
32
+ # a message_id as returned via consume.
33
+ def reject(message_id)
34
+ raise NotImplementedError
35
+ end
36
+
37
+ # Complete should mark a message as finished. It takes a message_id as returned via consume
38
+ def complete(message_id)
39
+ raise NotImplementedError
40
+ end
41
+
42
+ # Perform any shutdown behavior and stop consuming messages
43
+ def stop
44
+ @running = false
45
+ end
46
+
47
+ # Returns true if the Consumer is currently running
48
+ def running?
49
+ @running
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,56 @@
1
+ module Chore
2
+ class DuplicateDetector #:nodoc:
3
+
4
+ def initialize(opts={})
5
+ # Make it optional. Only required when we use it.
6
+ begin
7
+ require 'dalli'
8
+ rescue LoadError => e
9
+ Chore.logger.error "Unable to load dalli gem. It is required if duplicate \
10
+ detection is enabled. Install it with 'gem install dalli'."
11
+ raise e
12
+ end
13
+
14
+ memcached_options = {
15
+ :auto_eject_hosts => false,
16
+ :cache_lookups => false,
17
+ :tcp_nodelay => true,
18
+ :socket_max_failures => 5,
19
+ :socket_timeout => 2
20
+ }
21
+
22
+ @timeouts = {}
23
+ @dupe_on_cache_failure = opts.fetch(:dupe_on_cache_failure) { false }
24
+ @timeout = opts.fetch(:timeout) { 0 }
25
+ @servers = opts.fetch(:servers) { nil }
26
+ @memcached_client = opts.fetch(:memcached_client) { Dalli::Client.new(@servers, memcached_options) }
27
+ end
28
+
29
+ # Checks the message against the configured dedupe server to see if the message is unique or not
30
+ # Unique messages will return false
31
+ # Duplicated messages will return true
32
+ def found_duplicate?(msg)
33
+ return false unless msg && msg.respond_to?(:queue) && msg.queue
34
+ timeout = self.queue_timeout(msg.queue)
35
+ begin
36
+ !@memcached_client.add(msg.id, "1",timeout)
37
+ rescue StandardError => e
38
+ if @dupe_on_cache_failure
39
+ Chore.logger.error "Error accessing duplicate cache server. Assuming message is a duplicate. #{e}\n#{e.backtrace * "\n"}"
40
+ true
41
+ else
42
+ Chore.logger.error "Error accessing duplicate cache server. Assuming message is not a duplicate. #{e}\n#{e.backtrace * "\n"}"
43
+ false
44
+ end
45
+ end
46
+ end
47
+
48
+ # Retrieves the timeout for the given queue. The timeout is the window of time in seconds that
49
+ # we would consider the message to be non-unique, before we consider it dead in the water
50
+ # After that timeout, we would consider the next copy of the message received to be unique, and process it.
51
+ def queue_timeout(queue)
52
+ @timeouts[queue.url] ||= queue.visibility_timeout || @timeout
53
+ end
54
+
55
+ end
56
+ end
@@ -0,0 +1,31 @@
1
+ module Chore
2
+ class Fetcher #:nodoc:
3
+ attr_reader :manager, :consumers
4
+
5
+ def initialize(manager)
6
+ @stopping = false
7
+ @manager = manager
8
+ @strategy = Chore.config.consumer_strategy.new(self)
9
+ end
10
+
11
+ # Starts the fetcher with the configured Consumer Strategy. This will begin consuming messages from your queue
12
+ def start
13
+ Chore.logger.info "Fetcher starting up"
14
+ @strategy.fetch
15
+ end
16
+
17
+ # Stops the fetcher, preventing any further messages from being pulled from the queue
18
+ def stop!
19
+ unless @stopping
20
+ Chore.logger.info "Fetcher shutting down"
21
+ @stopping = true
22
+ @strategy.stop!
23
+ end
24
+ end
25
+
26
+ # Determines in the fetcher is in the process of stopping
27
+ def stopping?
28
+ @stopping
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,25 @@
1
+ module Chore
2
+ # Abstracts the notion of registering and running hooks during certain points in the lifecycle of chore
3
+ # processing work.
4
+ module Hooks
5
+
6
+ # Helper method to look up, and execute hooks based on an event name.
7
+ # Hooks are assumed to be methods defined on `self` that are of the pattern
8
+ # hook_name_identifier. ex: before_perform_log
9
+ def run_hooks_for(event,*args)
10
+ results = global_hooks_for(event).map { |prc| prc.call(*args) } || [true]
11
+ results << hooks_for(event).map { |method| send(method,*args) }
12
+ results = false if results.any? {|r| false == r }
13
+ results
14
+ end
15
+
16
+ private
17
+ def hooks_for(event)
18
+ (self.methods - Object.methods).grep(/^#{event}/).sort
19
+ end
20
+
21
+ def global_hooks_for(event)
22
+ Chore.hooks_for(event)
23
+ end
24
+ end
25
+ end
data/lib/chore/job.rb ADDED
@@ -0,0 +1,103 @@
1
+ require 'chore/hooks'
2
+
3
+ module Chore
4
+
5
+ # <tt>Chore::Job</tt> is the module which gives your job classes the methods they need to be published
6
+ # and run within Chore. You cannot have a Job in Chore that does not include this module
7
+ module Job
8
+
9
+ # An exception to represent a job choosing to forcibly reject a given instance of itself.
10
+ # The reasoning behind rejecting the job and the message that spawned it are left to
11
+ # the developer to dedide to use or not to use.
12
+ class RejectMessageException < Exception
13
+ # Throw a RejectMessageException from your job to signal that the message should be rejected.
14
+ # The semantics of +reject+ are queue implementation dependent.
15
+ end
16
+
17
+ def self.job_classes #:nodoc:
18
+ @classes || []
19
+ end
20
+
21
+ def self.included(base) #:nodoc:
22
+ @classes ||= []
23
+ @classes << base.name
24
+ base.extend(ClassMethods)
25
+ base.extend(Hooks)
26
+ end
27
+
28
+ module ClassMethods
29
+ DEFAULT_OPTIONS = { }
30
+
31
+ # Pass a hash of options to queue_options the included class's use of Chore::Job
32
+ # +opts+ has just the one required option.
33
+ # * +:name+: which should map to the name of the queue this job should be published to.
34
+ def queue_options(opts = {})
35
+ @chore_options = (@chore_options || DEFAULT_OPTIONS).merge(opts_from_cli).merge(opts)
36
+ required_options.each do |k|
37
+ raise ArgumentError.new("#{self.to_s} :#{k} is a required option for Chore::Job") unless @chore_options[k]
38
+ end
39
+ end
40
+
41
+ # This is a method so it can be overriden to create additional required
42
+ # queue_options params. This also determines what options get pulled
43
+ # from the global Chore.config.
44
+ def required_options
45
+ [:name, :publisher, :max_attempts]
46
+ end
47
+
48
+ def options #:nodoc:#
49
+ @chore_options ||= queue_options
50
+ end
51
+
52
+ def opts_from_cli #:nodoc:#
53
+ @from_cli ||= (Chore.config.marshal_dump.select {|k,v| required_options.include? k } || {})
54
+ end
55
+
56
+ # Execute the current job. We create an instance of the job to do the perform
57
+ # as this allows the jobs themselves to do initialization that might require access
58
+ # to the parameters of the job.
59
+ def perform(*args)
60
+ job = self.new(args)
61
+ job.perform(*args)
62
+ end
63
+
64
+ # Publish a job using an instance of job. Similar to perform we do this so that a job
65
+ # can perform initialization logic before the perform_async is begun. This, in addition, to
66
+ # hooks allows for rather complex jobs to be written simply.
67
+ def perform_async(*args)
68
+ job = self.new(args)
69
+ job.perform_async(*args)
70
+ end
71
+
72
+ # Resque/Sidekiq compatible serialization. No reason to change what works
73
+ def job_hash(job_params)
74
+ {:class => self.to_s, :args => job_params}
75
+ end
76
+
77
+ # The name of the configured queue, combined with an optional prefix
78
+ def prefixed_queue_name
79
+ "#{Chore.config.queue_prefix}#{self.options[:name]}"
80
+ end
81
+ end #ClassMethods
82
+
83
+ # This is handy to override in an included job to be able to do job setup that requires
84
+ # access to a job's arguments to be able to perform any context specific initialization that may
85
+ # be required.
86
+ def initialize(args=nil)
87
+ end
88
+
89
+ # This needs to be overriden by the object that is including this module.
90
+ def perform(*args)
91
+ raise NotImplementedError
92
+ end
93
+
94
+ # Use the current configured publisher to send this job into a queue.
95
+ def perform_async(*args)
96
+ self.class.run_hooks_for(:before_publish,*args)
97
+ @chore_publisher ||= self.class.options[:publisher]
98
+ @chore_publisher.publish(self.class.prefixed_queue_name,self.class.job_hash(args))
99
+ self.class.run_hooks_for(:after_publish,*args)
100
+ end
101
+
102
+ end #Job
103
+ end #Chore
@@ -0,0 +1,18 @@
1
+ require 'json'
2
+
3
+ module Chore
4
+ # Json encoding for serializing jobs.
5
+ module JsonEncoder
6
+ class << self
7
+ # Encodes the +job+ into JSON using the standard ruby JSON parsing library
8
+ def encode(job)
9
+ JSON.generate(job.to_hash)
10
+ end
11
+
12
+ # Decodes the +job+ from JSON into a ruby Hash using the standard ruby JSON parsing library
13
+ def decode(job)
14
+ JSON.parse(job)
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,47 @@
1
+ require 'json'
2
+ require 'chore/worker'
3
+ require 'chore/fetcher'
4
+
5
+ module Chore
6
+ # Manages the interactions between fetching messages (Consumer Strategy), and working over them (Worker Strategy)
7
+ class Manager
8
+
9
+ def initialize()
10
+ Chore.logger.info "Booting Chore #{Chore::VERSION}"
11
+ Chore.logger.debug { Chore.config.inspect }
12
+ @started_at = nil
13
+ @worker_strategy = Chore.config.worker_strategy.new(self)
14
+ @fetcher = Chore.config.fetcher.new(self)
15
+ @processed = 0
16
+ @stopping = false
17
+ end
18
+
19
+ # Start the Manager. This calls both the #start method of the configured Worker Strategy, as well as Fetcher#start.
20
+ def start
21
+ @started_at = Time.now
22
+ @worker_strategy.start
23
+ @fetcher.start
24
+ end
25
+
26
+ # Shut down the Manager, the Worker Strategy, and the Fetcher. This calls the +:before_shutdown+ hook.
27
+ def shutdown!
28
+ unless @stopping
29
+ Chore.logger.info "Manager shutting down"
30
+ @stopping = true
31
+ Chore.run_hooks_for(:before_shutdown)
32
+ @fetcher.stop!
33
+ @worker_strategy.stop!
34
+ end
35
+ end
36
+
37
+ # Take in an amount of +work+ (either an Array of, or a single UnitOfWork), and pass it down for the
38
+ # worker strategy to process. <b>This method is blocking</b>. It will continue to attempt to assign the work via
39
+ # the worker strategy, until it accepts it. It is up to the strategy to determine what cases it is allowed to accept
40
+ # work. The blocking semantic of this method is to prevent the Fetcher from getting messages off of the queue faster
41
+ # than they can be consumed.
42
+ def assign(work)
43
+ Chore.logger.debug { "Manager#assign: No. of UnitsOfWork: #{work.length})" }
44
+ @worker_strategy.assign(work) unless @stopping
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,29 @@
1
+ module Chore
2
+ # Base class for Chore Publishers. Provides the bare interface one needs to adhere to when writing custom publishers
3
+ class Publisher
4
+ DEFAULT_OPTIONS = { :encoder => JsonEncoder }
5
+
6
+ attr_accessor :options
7
+
8
+ def initialize(opts={})
9
+ self.options = DEFAULT_OPTIONS.merge(opts)
10
+ end
11
+
12
+ # Publishes the provided +job+ to the queue identified by the +queue_name+. Not designed to be used directly, this
13
+ # method ferries to the publish method on an instance of your configured Publisher.
14
+ def self.publish(queue_name,job)
15
+ self.new.publish(queue_name,job)
16
+ end
17
+
18
+ # Raises a NotImplementedError. This method should be overridden in your descendent, custom publisher class
19
+ def publish(queue_name,job)
20
+ raise NotImplementedError
21
+ end
22
+ protected
23
+
24
+ def encode_job(job)
25
+ options[:encoder].encode(job)
26
+ end
27
+
28
+ end
29
+ end
@@ -0,0 +1,128 @@
1
+ require 'fileutils'
2
+ require 'chore/queues/filesystem/filesystem_queue'
3
+
4
+ module Chore
5
+ module Queues
6
+ module Filesystem
7
+
8
+ # This is the consuming side of the file system queue. This class consumes jobs created by
9
+ # FilesystemPublisher#publish. The root of the file system queue is configured in
10
+ # Chore.config.fs_queue_root. In there a directory will be created for each queue name.
11
+ # Each queue directory contains a directory called "new" and one called "inprogress".
12
+ # FilesystemPublisher#publish creates new job files in the "new" directory. This consumer
13
+ # polls that directory every 5 seconds for new jobs which are moved to "inprogress".
14
+ #
15
+ # Once complete job files are deleted.
16
+ # If rejected they are moved back into new and will be processed again. This may not be the
17
+ # desired behavior long term and we may want to add configuration to this class to allow more
18
+ # creating failure handling and retrying.
19
+ class Consumer < Chore::Consumer
20
+ include FilesystemQueue
21
+
22
+ Chore::CLI.register_option 'fs_queue_root', '--fs-queue-root DIRECTORY', 'Root directory for fs based queue'
23
+
24
+ FILE_QUEUE_MUTEXES = {}
25
+
26
+ # The amount of time units of work can run before the queue considers
27
+ # them timed out. For filesystem queues, this is the global default.
28
+ attr_reader :queue_timeout
29
+
30
+ def initialize(queue_name, opts={})
31
+ super(queue_name, opts)
32
+
33
+ # Even though putting these Mutexes in this hash is, by itself, not particularly threadsafe
34
+ # as long as some Mutex ends up in the queue after all consumers are created we're good
35
+ # as they are pulled from the queue and synchronized for file operations below
36
+ FILE_QUEUE_MUTEXES[@queue_name] ||= Mutex.new
37
+
38
+ @in_progress_dir = in_progress_dir(queue_name)
39
+ @new_dir = new_dir(queue_name)
40
+ @queue_timeout = Chore.config.default_queue_timeout
41
+ end
42
+
43
+ def consume(&handler)
44
+ Chore.logger.info "Starting consuming file system queue #{@queue_name} in #{queue_dir(queue_name)}"
45
+ while running?
46
+ begin
47
+ #TODO move expired job files to new directory?
48
+ handle_jobs(&handler)
49
+ rescue => e
50
+ Chore.logger.error { "#{self.class}#consume: #{e} #{e.backtrace * "\n"}" }
51
+ ensure
52
+ sleep 5
53
+ end
54
+ end
55
+ end
56
+
57
+ def reject(id)
58
+ Chore.logger.debug "Rejecting: #{id}"
59
+ make_new_again(id)
60
+ end
61
+
62
+ def complete(id)
63
+ Chore.logger.debug "Completing (deleting): #{id}"
64
+ FileUtils.rm(File.join(@in_progress_dir, id))
65
+ end
66
+
67
+ private
68
+
69
+ # finds all new job files, moves them to in progress and starts the job
70
+ # Returns a list of the job files processed
71
+ def handle_jobs(&block)
72
+ # all consumers on a single queue share a lock on handling files.
73
+ # Each consumer comes along, processes all present files and release the lock.
74
+ # This isn't particularly useful but is here to allow the configuration of
75
+ # ThreadedConsumerStrategy with mutiple threads on a queue safely although you
76
+ # probably wouldn't want to do that.
77
+ FILE_QUEUE_MUTEXES[@queue_name].synchronize do
78
+ job_files.each do |job_file|
79
+ Chore.logger.debug "Found a new job #{job_file}"
80
+
81
+ job_json = File.read(make_in_progress(job_file))
82
+ basename, previous_attempts = file_info(job_file)
83
+
84
+ # job_file is just the name which is the job id
85
+ block.call(job_file, queue_name, queue_timeout, job_json, previous_attempts)
86
+ Chore.run_hooks_for(:on_fetch, job_file, job_json)
87
+ end
88
+ end
89
+ end
90
+
91
+ def make_in_progress(job)
92
+ move_job(File.join(@new_dir, job), File.join(@in_progress_dir, job))
93
+ end
94
+
95
+ def make_new_again(job)
96
+ basename, previous_attempts = file_info(job)
97
+ move_job(File.join(@in_progress_dir, job), File.join(@new_dir, "#{basename}.#{previous_attempts + 1}.job"))
98
+ end
99
+
100
+ # Moves job file to inprogress directory and returns the full path
101
+ def move_job(from, to)
102
+ f = File.open(from, "r")
103
+ # wait on the lock a publisher in another process might have.
104
+ # Once we get the lock the file is ours to move to mark it in progress
105
+ f.flock(File::LOCK_EX)
106
+ begin
107
+ FileUtils.mv(f.path, to)
108
+ ensure
109
+ f.flock(File::LOCK_UN) # yes we can unlock it after its been moved, I checked
110
+ end
111
+ to
112
+ end
113
+
114
+ def job_files
115
+ Dir.entries(@new_dir).select{|e| ! e.start_with?(".")}
116
+ end
117
+
118
+ # Grabs the unique identifier for the job filename and the number of times
119
+ # it's been attempted (also based on the filename)
120
+ def file_info(job_file)
121
+ id, previous_attempts = File.basename(job_file, '.job').split('.')
122
+ [id, previous_attempts.to_i]
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
128
+
@@ -0,0 +1,49 @@
1
+ # Common methods used by FilesystemConsumer and FilesystemPublisher for dealing with the
2
+ # directories which implement the queue.
3
+ module Chore::FilesystemQueue
4
+
5
+ # Local directory for new jobs to be placed
6
+ NEW_JOB_DIR = "new"
7
+ # Local directory for jobs currently in-process to be moved
8
+ IN_PROGRESS_DIR = "inprogress"
9
+
10
+ # Retrieves the directory for in-process messages to go. If the directory for the +queue_name+ doesn't exist,
11
+ # it will be created for you. If the directory cannot be created, an IOError will be raised
12
+ def in_progress_dir(queue_name)
13
+ validate_dir(queue_name, IN_PROGRESS_DIR)
14
+ end
15
+
16
+ # Retrieves the directory for newly recieved messages to go. If the directory for the +queue_name+ doesn't exist,
17
+ # it will be created for you. If the directory cannot be created, an IOError will be raised
18
+ def new_dir(queue_name)
19
+ validate_dir(queue_name, NEW_JOB_DIR)
20
+ end
21
+
22
+ # Returns the root directory where messages are placed
23
+ def root_dir
24
+ @root_dir ||= prepare_dir(File.expand_path(Chore.config.fs_queue_root))
25
+ end
26
+
27
+ # Returns the fully qualified path to the directory for +queue_name+
28
+ def queue_dir(queue_name)
29
+ prepare_dir(File.join(root_dir, queue_name))
30
+ end
31
+
32
+ private
33
+ # Returns the directory for the given +queue_name+ and +task_state+. If the directory doesn't exist, it will be
34
+ # created for you. If the directory cannot be created, an IOError will be raised
35
+ def validate_dir(queue_name, task_state)
36
+ prepare_dir(File.join(queue_dir(queue_name), task_state))
37
+ end
38
+
39
+ # Creates a directory if it does not exist. Returns the directory
40
+ def prepare_dir(dir)
41
+ unless Dir.exists?(dir)
42
+ FileUtils.mkdir_p(dir)
43
+ end
44
+
45
+ raise IOError.new("directory for file system queue does not have write permission: #{dir}") unless File.writable?(dir)
46
+ dir
47
+ end
48
+
49
+ end
@@ -0,0 +1,45 @@
1
+ require 'chore/queues/filesystem/filesystem_queue'
2
+
3
+ module Chore
4
+ module Queues
5
+ module Filesystem
6
+
7
+ # Publisher for writing jobs to the local filesystem. Useful for testing in offline environments or
8
+ # when queuing implementations are irrelevent to the task at hand, such as local development of new jobs.
9
+ class Publisher < Chore::Publisher
10
+ # See the top of FilesystemConsumer for comments on how this works
11
+ include FilesystemQueue
12
+
13
+ # Mutex for holding a lock over the files for this queue while they are in process
14
+ FILE_MUTEX = Mutex.new
15
+
16
+ # use of mutex and file locking should make this both threadsafe and safe for multiple
17
+ # processes to use the same queue directory simultaneously.
18
+ def publish(queue_name,job)
19
+ FILE_MUTEX.synchronize do
20
+ while true
21
+ # keep trying to get a file with nothing in it meaning we just created it
22
+ # as opposed to us getting someone else's file that hasn't been processed yet.
23
+ f = File.open(filename(queue_name, job[:class].to_s), "w")
24
+ if f.flock(File::LOCK_EX | File::LOCK_NB) && f.size == 0
25
+ begin
26
+ f.write(job.to_json)
27
+ ensure
28
+ f.flock(File::LOCK_UN)
29
+ break
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
35
+
36
+ # create a unique filename for a job in a queue based on queue name, job name and date
37
+ def filename(queue_name, job_name)
38
+ now = Time.now.strftime "%Y%m%d-%H%M%S-%6N"
39
+ previous_attempts = 0
40
+ File.join(new_dir(queue_name), "#{queue_name}-#{job_name}-#{now}.#{previous_attempts}.job")
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end