chore-core 1.7.2 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 74242dc2e704cdf2cf5baf9dd9b25822bbe20066
-  data.tar.gz: 0082dc5ab748faff7ba73a8bb6f01773aaf6b35c
+  metadata.gz: fd6b4ede9ed3d138dde2225105cb0bb1d28e5411
+  data.tar.gz: 1aee13a468b20323eefa1b854469c867ce676895
 SHA512:
-  metadata.gz: de6ce3260f50d5a324843f6bbc9511974da8d69731e893aad11a5404ee4107f167a4cb02b9579e372f822f414551ce4ee754139a316845282a2465170f5c03a1
-  data.tar.gz: fdd9bb38cb35ff082e055bf76a897571bdde176cb1fa20cd9880ed19f57c55d908f0f3c6c830fceae62cec88f761c0228dcb1bcdb051f1e561a48ea5d3f2bc49
+  metadata.gz: c76ac25d8026e1a61fb3169bbefaf4151be6a586c60161624031b48761c83f4690a2e0e6d60ec6a8541d008323e7a64f64aa902114e899430682e5521c802a11
+  data.tar.gz: fa19122b3e452507848b6670fbe98b008e52439cabc427a57f96a850c30c45843eddee4bf230b5cd85875896f7ac89d6591fb2cd4bab1976e99091362720f677

data/bin/chore

@@ -22,9 +22,6 @@ Chore::Signal.trap "USR1" do
 end
 
 begin
-  # Pre-load any Bundler dependencies now, so that the CLI parser has them loaded
-  # prior to intrpretting the command line args for things like consumers/producers
-  Bundler.require if defined?(Bundler)
   cli = Chore::CLI.instance
   cli.run!(ARGV)
 rescue => e

data/lib/chore/cli.rb

@@ -63,10 +63,10 @@ module Chore #:nodoc:
       end
     end
 
-    def parse_config_file(file) #:nodoc:
+    def parse_config_file(file, ignore_errors = false) #:nodoc:
       data = File.read(file)
       data = ERB.new(data).result
-      parse_opts(data.split(/\s/).map!(&:chomp).map!(&:strip))
+      parse_opts(data.split(/\s/).map!(&:chomp).map!(&:strip), ignore_errors)
     end
 
     def parse(args=ARGV) #:nodoc:
@@ -74,13 +74,16 @@ module Chore #:nodoc:
       setup_options
 
       # parse once to load the config file & require options
-      parse_opts(args)
-      parse_config_file(@options[:config_file]) if @options[:config_file]
+      # any invalid options are ignored the first time around since booting the
+      # system may register additional options from 3rd-party libs
+      parse_opts(args, true)
+      parse_config_file(@options[:config_file], true) if @options[:config_file]
 
       validate!
       boot_system
 
       # parse again to pick up options required by loaded classes
+      # any invalid options will raise an exception this time
       parse_opts(args)
       parse_config_file(@options[:config_file]) if @options[:config_file]
       detect_queues
@@ -143,7 +146,7 @@ module Chore #:nodoc:
 
     end
 
-    def parse_opts(argv) #:nodoc:
+    def parse_opts(argv, ignore_errors = false) #:nodoc:
       @options ||= {}
       @parser = OptionParser.new do |o|
         registered_opts.each do |key,opt|
@@ -164,7 +167,22 @@ module Chore #:nodoc:
         exit 1
       end
 
-      @parser.parse!(argv)
+      # This will parse arguments in order, continuing even if invalid options
+      # are encountered
+      argv = argv.dup
+      begin
+        @parser.parse(argv)
+      rescue OptionParser::InvalidOption => ex
+        if ignore_errors
+          # Drop everything up to (and including) the invalid argument
+          # and start parsing again
+          invalid_arg = ex.args[0]
+          argv = argv.drop(argv.index(invalid_arg) + 1)
+          retry
+        else
+          raise
+        end
+      end
 
       @options
     end
@@ -185,6 +203,9 @@ module Chore #:nodoc:
         require File.expand_path("#{options[:require]}/config/environment.rb")
         ::Rails.application.eager_load!
       else
+        # Pre-load any Bundler dependencies now, so that the CLI parser has them loaded
+        # prior to intrpretting the command line args for things like consumers/producers
+        Bundler.require if defined?(Bundler)
         require File.expand_path(options[:require])
       end
     end

data/lib/chore/consumer.rb

@@ -21,6 +21,11 @@ module Chore
     def self.reset_connection!
     end
 
+    # Cleans up any resources that were left behind from prior instances of the
+    # chore process.  By default, this is a no-op.
+    def self.cleanup(queue)
+    end
+
     # Consume takes a block with an arity of two. The two params are
     # |message_id,message_body| where message_id is any object that the
     # consumer will need to be able to act on a message later (reject, complete, etc)

data/lib/chore/fetcher.rb

@@ -11,6 +11,12 @@ module Chore
     # Starts the fetcher with the configured Consumer Strategy. This will begin consuming messages from your queue
     def start
       Chore.logger.info "Fetcher starting up"
+
+      # Clean up configured queues in case there are any resources left behind
+      Chore.config.queues.each do |queue|
+        Chore.config.consumer.cleanup(queue)
+      end
+
       @strategy.fetch
     end
 

data/lib/chore/queues/filesystem/consumer.rb

@@ -17,12 +17,59 @@ module Chore
       # desired behavior long term and we may want to add configuration to this class to allow more
       # creating failure handling and retrying. 
       class Consumer < Chore::Consumer
-        include FilesystemQueue
+        extend FilesystemQueue
 
         Chore::CLI.register_option 'fs_queue_root', '--fs-queue-root DIRECTORY', 'Root directory for fs based queue'
         
         FILE_QUEUE_MUTEXES = {}
-        
+
+        class << self
+          # Cleans up the in-progress files by making them new again.  This should only
+          # happen once per process.
+          def cleanup(queue)
+            new_dir = self.new_dir(queue)
+            in_progress_dir = self.in_progress_dir(queue)
+
+            job_files(in_progress_dir).each do |file|
+              make_new_again(file, new_dir, in_progress_dir)
+            end
+          end
+
+          def make_in_progress(job, new_dir, in_progress_dir)
+            move_job(File.join(new_dir, job), File.join(in_progress_dir, job))
+          end
+
+          def make_new_again(job, new_dir, in_progress_dir)
+            basename, previous_attempts = file_info(job)
+            move_job(File.join(in_progress_dir, job), File.join(new_dir, "#{basename}.#{previous_attempts + 1}.job"))
+          end
+
+          # Moves job file to inprogress directory and returns the full path
+          def move_job(from, to)
+            f = File.open(from, "r")
+            # wait on the lock a publisher in another process might have.
+            # Once we get the lock the file is ours to move to mark it in progress
+            f.flock(File::LOCK_EX)
+            begin
+              FileUtils.mv(f.path, to)
+            ensure
+              f.flock(File::LOCK_UN) # yes we can unlock it after its been moved, I checked
+            end
+            to
+          end
+
+          def job_files(dir)
+            Dir.entries(dir).select{|e| ! e.start_with?(".")}
+          end
+
+          # Grabs the unique identifier for the job filename and the number of times
+          # it's been attempted (also based on the filename)
+          def file_info(job_file)
+            id, previous_attempts = File.basename(job_file, '.job').split('.')
+            [id, previous_attempts.to_i]
+          end
+        end
+
         # The amount of time units of work can run before the queue considers
         # them timed out.  For filesystem queues, this is the global default.
         attr_reader :queue_timeout
@@ -35,13 +82,13 @@ module Chore
           # as they are pulled from the queue and synchronized for file operations below
           FILE_QUEUE_MUTEXES[@queue_name] ||= Mutex.new
 
-          @in_progress_dir = in_progress_dir(queue_name)
-          @new_dir = new_dir(queue_name)
+          @in_progress_dir = self.class.in_progress_dir(queue_name)
+          @new_dir = self.class.new_dir(queue_name)
           @queue_timeout = Chore.config.default_queue_timeout
         end
 
         def consume(&handler)
-          Chore.logger.info "Starting consuming file system queue #{@queue_name} in #{queue_dir(queue_name)}"
+          Chore.logger.info "Starting consuming file system queue #{@queue_name} in #{self.class.queue_dir(queue_name)}"
           while running?
             begin
               #TODO move expired job files to new directory?
@@ -75,11 +122,11 @@ module Chore
           # ThreadedConsumerStrategy with mutiple threads on a queue safely although you
           # probably wouldn't want to do that.
           FILE_QUEUE_MUTEXES[@queue_name].synchronize do
-            job_files.each do |job_file|
+            self.class.job_files(@new_dir).each do |job_file|
               Chore.logger.debug "Found a new job #{job_file}"
               
               job_json = File.read(make_in_progress(job_file))
-              basename, previous_attempts = file_info(job_file)
+              basename, previous_attempts = self.class.file_info(job_file)
 
               # job_file is just the name which is the job id
               block.call(job_file, queue_name, queue_timeout, job_json, previous_attempts)
@@ -89,37 +136,11 @@ module Chore
         end
 
         def make_in_progress(job)
-          move_job(File.join(@new_dir, job), File.join(@in_progress_dir, job))
+          self.class.make_in_progress(job, @new_dir, @in_progress_dir)
         end
 
         def make_new_again(job)
-          basename, previous_attempts = file_info(job)
-          move_job(File.join(@in_progress_dir, job), File.join(@new_dir, "#{basename}.#{previous_attempts + 1}.job"))
-        end
-        
-        # Moves job file to inprogress directory and returns the full path
-        def move_job(from, to)
-          f = File.open(from, "r")
-          # wait on the lock a publisher in another process might have.
-          # Once we get the lock the file is ours to move to mark it in progress
-          f.flock(File::LOCK_EX)
-          begin
-            FileUtils.mv(f.path, to)
-          ensure
-            f.flock(File::LOCK_UN) # yes we can unlock it after its been moved, I checked
-          end
-          to
-        end
-
-        def job_files
-          Dir.entries(@new_dir).select{|e| ! e.start_with?(".")}
-        end
-
-        # Grabs the unique identifier for the job filename and the number of times
-        # it's been attempted (also based on the filename)
-        def file_info(job_file)
-          id, previous_attempts = File.basename(job_file, '.job').split('.')
-          [id, previous_attempts.to_i]
+          self.class.make_new_again(job, @new_dir, @in_progress_dir)
         end
       end
     end

data/lib/chore/version.rb

@@ -1,8 +1,8 @@
 module Chore
   module Version #:nodoc:
     MAJOR = 1
-    MINOR = 7
-    PATCH = 2
+    MINOR = 8
+    PATCH = 0
 
     STRING = [ MAJOR, MINOR, PATCH ].join('.')
   end

data/spec/chore/consumer_spec.rb

@@ -22,6 +22,10 @@ describe Chore::Consumer do
     Chore::Consumer.should respond_to :reset_connection!
   end
 
+  it 'should have a class level cleanup method' do
+    Chore::Consumer.should respond_to :cleanup
+  end
+
   it 'should not have an implemented consume method' do
     expect { consumer.consume }.to raise_error(NotImplementedError)
   end

data/spec/chore/fetcher_spec.rb

@@ -35,4 +35,15 @@ describe Chore::Fetcher do
       fetcher.start
     end
   end
+
+  describe "cleaning up" do
+    before(:each) do
+      manager.stub(:assign)
+    end
+    
+    it "should run cleanup on each queue" do
+      consumer.should_receive(:cleanup).with('test')
+      fetcher.start
+    end
+  end
 end

data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb

@@ -19,52 +19,96 @@ describe Chore::Queues::Filesystem::Consumer do
     FileUtils.rm_rf(test_queues_dir)
   end
 
-  let!(:consumer_run_for_one_message) { expect(consumer).to receive(:running?).and_return(true, false) }
   let(:test_job_hash) {{:class => "TestClass", :args => "test-args"}}
+  let(:new_dir) { described_class.new_dir(test_queue) }
+  let(:in_progress_dir) { described_class.in_progress_dir(test_queue) }
 
-  context "founding a published job" do
-    before do
-      publisher.publish(test_queue, test_job_hash)
+  describe ".cleanup" do
+    it "should move in_progress jobs to new dir" do
+      FileUtils.touch("#{in_progress_dir}/foo.1.job")
+      described_class.cleanup(test_queue)
+      expect(File.exist?("#{new_dir}/foo.2.job")).to eq(true)
     end
+  end
 
-    it "should consume a published job and yield the job to the handler block" do
-      expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 0)
+  describe ".make_in_progress" do
+    it "should move job to in_progress dir" do
+      FileUtils.touch("#{new_dir}/foo.1.job")
+      described_class.make_in_progress("foo.1.job", new_dir, in_progress_dir)
+      expect(File.exist?("#{in_progress_dir}/foo.1.job")).to eq(true)
     end
+  end
 
-    context "rejecting a job" do
-      let!(:consumer_run_for_two_messages) { allow(consumer).to receive(:running?).and_return(true, false,true,false) }
+  describe ".make_new_again" do
+    it "should move job to new dir" do
+      FileUtils.touch("#{in_progress_dir}/foo.1.job")
+      described_class.make_new_again("foo.1.job", new_dir, in_progress_dir)
+      expect(File.exist?("#{new_dir}/foo.2.job")).to eq(true)
+    end
+  end
 
-      it "should requeue a job that gets rejected" do
-        rejected = false
-        consumer.consume do |job_id, queue_name, job_hash|
-          consumer.reject(job_id)
-          rejected = true
-        end
-        expect(rejected).to be true
+  describe ".job_files" do
+    it "should list jobs in dir" do
+      FileUtils.touch("#{new_dir}/foo.1.job")
+      expect(described_class.job_files(new_dir)).to eq(["foo.1.job"])
+    end
+  end
 
-        expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 1)
-      end
+  describe ".file_info" do
+    it "should split name and attempt number" do
+      name, attempt = described_class.file_info("foo.1.job")
+      expect(name).to eq("foo")
+      expect(attempt).to eq(1)
     end
+  end
 
-    context "completing a job" do
-      let!(:consumer_run_for_two_messages) { allow(consumer).to receive(:running?).and_return(true, false,true,false) }
+  describe 'consumption' do
+    let!(:consumer_run_for_one_message) { expect(consumer).to receive(:running?).and_return(true, false) }
 
-      it "should remove job on completion" do
-        completed = false
-        consumer.consume do |job_id, queue_name, job_hash|
-          consumer.complete(job_id)
-          completed = true
+    context "founding a published job" do
+      before do
+        publisher.publish(test_queue, test_job_hash)
+      end
+
+      it "should consume a published job and yield the job to the handler block" do
+        expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 0)
+      end
+
+      context "rejecting a job" do
+        let!(:consumer_run_for_two_messages) { allow(consumer).to receive(:running?).and_return(true, false,true,false) }
+
+        it "should requeue a job that gets rejected" do
+          rejected = false
+          consumer.consume do |job_id, queue_name, job_hash|
+            consumer.reject(job_id)
+            rejected = true
+          end
+          expect(rejected).to be true
+
+          expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 1)
         end
-        expect(completed).to be true
+      end
 
-        expect { |b| consumer.consume(&b) }.to_not yield_control
+      context "completing a job" do
+        let!(:consumer_run_for_two_messages) { allow(consumer).to receive(:running?).and_return(true, false,true,false) }
+
+        it "should remove job on completion" do
+          completed = false
+          consumer.consume do |job_id, queue_name, job_hash|
+            consumer.complete(job_id)
+            completed = true
+          end
+          expect(completed).to be true
+
+          expect { |b| consumer.consume(&b) }.to_not yield_control
+        end
       end
     end
-  end
 
-  context "not finding a published job" do
-    it "should consume a published job and yield the job to the handler block" do
-      expect { |b| consumer.consume(&b) }.to_not yield_control
+    context "not finding a published job" do
+      it "should consume a published job and yield the job to the handler block" do
+        expect { |b| consumer.consume(&b) }.to_not yield_control
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chore-core
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.8.0
 platform: ruby
 authors:
 - Tapjoy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-08 00:00:00.000000000 Z
+date: 2015-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json