rocketjob 3.3.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1974e77ea4b6aa126dfa57155d15bd49a659ae1e
-  data.tar.gz: 900333f68eb5ee6e378fc19d494b7408944fa83b
+  metadata.gz: 73879d59ee0d46f407f77acfc5b443b693e16727
+  data.tar.gz: 27d8a5a990b39d19c00ff086d57b3df2263eb727
 SHA512:
-  metadata.gz: 741923205b5e7caa759db125afe234fe4c237e80bfdadbb58db512b2ef2269f42fd9fc58207c4e9eb97a0447f164b9179472ccc0e1ba3ec64aaa155753cf0257
-  data.tar.gz: b44d77f843c6e6342a2be309ec50fbf25eae7f9db26e074eb96143d18283a321981050d70287189cc5c855bc3f9600da5d11328e388ac9b2f5a3aea3420170cf
+  metadata.gz: 3fb4043fb6aca63ecb3968393aada59cd81101d69453592cc5032722d1751ce3716c095dee54baa0be77136df05f36ce49687d1a4f1a66a37c68feeae22870b9
+  data.tar.gz: ff2c14ea7b456f59f86b85f34174a2ab0d90d72799c7e839b90def2a288b44969f8a558a915129df0dce726d9b0d28252c91d1f9d5aefaf331dcf43d9500cd51

data/lib/rocket_job/dirmon_entry.rb

@@ -187,15 +187,7 @@ module RocketJob
     #   # => {}
     def self.counts_by_state
       counts = {}
-      collection.aggregate([
-        {
-          '$group' => {
-            _id:   '$state',
-            count: {'$sum' => 1}
-          }
-        }
-      ]
-      ).each do |result|
+      collection.aggregate([{'$group' => {_id: '$state', count: {'$sum' => 1}}}]).each do |result|
         counts[result['_id'].to_sym] = result['count']
       end
       counts
@@ -229,14 +221,20 @@ module RocketJob
         # Case insensitive filename matching
         Pathname.glob(pattern, File::FNM_CASEFOLD).each do |pathname|
           next if pathname.directory?
-          pathname  = pathname.realpath
+          pathname = begin
+            pathname.realpath
+          rescue Errno::ENOENT
+            logger.warn("Unable to expand the realpath for #{pathname.inspect}. Skipping file.")
+            next
+          end
+
           file_name = pathname.to_s
 
           # Skip archive directories
           next if file_name.include?(self.class.default_archive_directory)
 
           # Security check?
-          if (whitelist_paths.size > 0) && whitelist_paths.none? { |whitepath| file_name.to_s.start_with?(whitepath) }
+          if (whitelist_paths.size > 0) && whitelist_paths.none? {|whitepath| file_name.to_s.start_with?(whitepath)}
             logger.error "Skipping file: #{file_name} since it is not in any of the whitelisted paths: #{whitelist_paths.join(', ')}"
             next
           end

data/lib/rocket_job/plugins/job/model.rb

@@ -39,7 +39,7 @@ module RocketJob
           field :priority, type: Integer, default: 50, class_attribute: true, user_editable: true, copy_on_restart: true
 
           # When the job completes destroy it from both the database and the UI
-          field :destroy_on_complete, type: Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+          field :destroy_on_complete, type: Boolean, default: true, class_attribute: true, copy_on_restart: true
 
           # Whether to store the results from this job
           field :collect_output, type: Boolean, default: false, class_attribute: true

data/lib/rocket_job/plugins/job/state_machine.rb

@@ -58,12 +58,14 @@ module RocketJob
             end
 
             event :pause do
-              transitions from: :running, to: :paused
+              transitions from: :running, to: :paused, if: :pausable?
+              # All jobs are pausable prior to starting the job.
               transitions from: :queued, to: :paused
             end
 
             event :resume do
-              transitions from: :paused, to: :running, if: -> { started_at }
+              transitions from: :paused, to: :running, if: -> { pausable? && started_at }
+              # All jobs paused before processing started are pausable.
               transitions from: :paused, to: :queued, unless: -> { started_at }
             end
 
@@ -82,6 +84,10 @@ module RocketJob
           end
           # @formatter:on
 
+          # By default all jobs are not pausable / resumable
+          class_attribute(:pausable)
+          self.pausable = false
+
           # Define a before and after callback method for each event
           state_machine_define_event_callbacks(*aasm.state_machine.events.keys)
 

data/lib/rocket_job/plugins/job/throttle_running_jobs.rb

@@ -16,10 +16,11 @@ module RocketJob
       #   end
       #
       # Notes:
-      # - The actual number will be around this value, it can go over slightly and
-      #   can drop depending on check interval can drop slightly below this value.
-      # - By avoiding hard locks and counters performance can be maintained while still
-      #   supporting good enough quantity throttling.
+      # - The number of running jobs will not exceed this value.
+      # - It may appear that a job is running briefly over this limit, but then is immediately back into queued state.
+      #   This is expected behavior and is part of the check to ensure this value is not exceeded.
+      #   The worker grabs the job and only then verifies the throttle, this is to prevent any other worker
+      #   from attempting to grab the job, which would have exceeded the throttle.
       module ThrottleRunningJobs
         extend ActiveSupport::Concern
 
@@ -37,7 +38,8 @@ module RocketJob
         def throttle_running_jobs_exceeded?
           throttle_running_jobs &&
             (throttle_running_jobs != 0) &&
-              (self.class.running.where(:id.ne => id).count >= throttle_running_jobs)
+              # Cannot use class since it will include instances of parent job classes.
+              (RocketJob::Job.running.where('_type' => self.class.name, :id.ne => id).count >= throttle_running_jobs)
         end
       end
     end

data/lib/rocket_job/plugins/transaction.rb

@@ -2,7 +2,7 @@ require 'active_support/concern'
 
 module RocketJob
   module Plugins
-    # Perform under a single Active Record transaction / unit or work.
+    # Wraps every #perform call with an Active Record transaction / unit or work.
     #
     # If the perform raises an exception it will cause any database changes to be rolled back.
     #
@@ -23,14 +23,19 @@ module RocketJob
     #       Audit.create!(table: 'user', description: 'Changed age to 21')
     #     end
     #   end
+    #
+    # Performance
+    # - On Ruby (MRI) an empty transaction block call takes about 1ms.
+    # - On JRuby an empty transaction block call takes about 55ms.
+    #
+    # Note:
+    # - This plugin will only be activated if ActiveRecord has been loaded first.
     module Transaction
       extend ActiveSupport::Concern
 
       included do
-        if respond_to?(:around_slice)
-          around_slice :rocket_job_transaction
-        else
-          around_perform :rocket_job_transaction
+        if defined?(ActiveRecord::Base)
+          respond_to?(:around_slice) ? around_slice(:rocket_job_transaction) : around_perform(:rocket_job_transaction)
         end
       end
 

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob #:nodoc
-  VERSION = '3.3.0'
+  VERSION = '3.3.1'
 end

data/lib/rocketjob.rb

@@ -40,6 +40,7 @@ module RocketJob
     autoload :Restart,          'rocket_job/plugins/restart'
     autoload :Singleton,        'rocket_job/plugins/singleton'
     autoload :StateMachine,     'rocket_job/plugins/state_machine'
+    autoload :Transaction,      'rocket_job/plugins/transaction'
   end
 
   module Jobs

data/test/config/database.yml

@@ -0,0 +1,5 @@
+test:
+  adapter:  sqlite3
+  database: test/test_db.sqlite3
+  pool:     5
+  timeout:  5000

data/test/plugins/job/throttle_test.rb

@@ -8,6 +8,7 @@ module Plugins
       class ThrottleJob < RocketJob::Job
         # Only allow one to be processed at a time
         self.throttle_running_jobs = 1
+        self.pausable              = true
 
         def perform
           21
@@ -85,15 +86,15 @@ module Plugins
             ThrottleJob.create!(state: :failed)
             ThrottleJob.create!(state: :complete)
             ThrottleJob.create!(state: :paused)
-            assert job = RocketJob::Job.rocket_job_next_job(@worker_name), -> { ThrottleJob.all.to_a.ai }
-            assert_equal @job.id, job.id, -> { ThrottleJob.all.to_a.ai }
+            assert job = RocketJob::Job.rocket_job_next_job(@worker_name), -> {ThrottleJob.all.to_a.ai}
+            assert_equal @job.id, job.id, -> {ThrottleJob.all.to_a.ai}
           end
 
           it 'return nil when other jobs are running' do
             ThrottleJob.create!
             @job = ThrottleJob.new
             @job.start!
-            assert_nil RocketJob::Job.rocket_job_next_job(@worker_name), -> { ThrottleJob.all.to_a.ai }
+            assert_nil RocketJob::Job.rocket_job_next_job(@worker_name), -> {ThrottleJob.all.to_a.ai}
           end
 
           it 'add job to filter when other jobs are running' do
@@ -101,7 +102,7 @@ module Plugins
             @job = ThrottleJob.new
             @job.start!
             filter = {}
-            assert_nil RocketJob::Job.rocket_job_next_job(@worker_name, filter), -> { ThrottleJob.all.to_a.ai }
+            assert_nil RocketJob::Job.rocket_job_next_job(@worker_name, filter), -> {ThrottleJob.all.to_a.ai}
             assert_equal 1, filter.size
           end
         end

data/test/plugins/restart_test.rb

@@ -16,11 +16,26 @@ module Plugins
       end
     end
 
+    class RestartablePausableJob < RocketJob::Job
+      include RocketJob::Plugins::Restart
+
+      field :start_at, type: Date
+      field :end_at, type: Date
+
+      # Job will reload itself during process to check if it was paused.
+      self.pausable = true
+
+      def perform
+        self.start_at = Date.today
+        self.end_at   = Date.today
+        'DONE'
+      end
+    end
+
     describe RocketJob::Plugins::Restart do
       before do
-        # destroy_all could create new instances
         RestartableJob.delete_all
-        assert_equal 0, RestartableJob.count
+        RestartablePausableJob.delete_all
       end
 
       after do
@@ -89,12 +104,12 @@ module Plugins
       end
 
       describe '#pause' do
-        it 'does not enqueues a new job when paused' do
-          @job = RestartableJob.new
+        it 'does not enqueue a new job when paused' do
+          @job = RestartablePausableJob.new
           @job.start
           @job.pause!
           assert @job.paused?
-          assert_equal 1, RestartableJob.count
+          assert_equal 1, RestartablePausableJob.count
         end
       end
 
@@ -115,7 +130,7 @@ module Plugins
         end
 
         it 'aborts from paused' do
-          @job = RestartableJob.new
+          @job = RestartablePausableJob.new
           @job.start
           @job.pause
           assert @job.paused?

data/test/plugins/transaction_test.rb

@@ -0,0 +1,86 @@
+require_relative '../test_helper'
+require 'active_record'
+
+ActiveRecord::Base.configurations = YAML::load(ERB.new(IO.read('test/config/database.yml')).result)
+ActiveRecord::Base.establish_connection(:test)
+
+ActiveRecord::Schema.define version: 0 do
+  create_table :users, force: true do |t|
+    t.string :login
+  end
+end
+
+class User < ActiveRecord::Base
+end
+
+module Plugins
+  module Job
+    class TransactionTest < Minitest::Test
+
+      class CommitTransactionJob < RocketJob::Job
+        # Wrap perform with a transaction, so that it is rolled back on exception.
+        include RocketJob::Plugins::Transaction
+
+        field :login, type: String
+
+        def perform
+          User.create!(login: login)
+        end
+      end
+
+      class RollbackTransactionJob < RocketJob::Job
+        # Wrap perform with a transaction, so that it is rolled back on exception.
+        include RocketJob::Plugins::Transaction
+
+        field :login, type: String
+
+        def perform
+          User.create!(login: login)
+          raise "This must fail and rollback the transaction"
+        end
+      end
+
+      describe RocketJob::Plugins::Job::Logger do
+        before do
+          User.delete_all
+          CommitTransactionJob.delete_all
+          RollbackTransactionJob.delete_all
+        end
+
+        after do
+          @job.destroy if @job && !@job.new_record?
+        end
+
+        describe '#rocket_job_transaction' do
+          it 'is registered' do
+            assert CommitTransactionJob.send(:get_callbacks, :perform).find {|c| c.filter == :rocket_job_transaction}
+            assert RollbackTransactionJob.send(:get_callbacks, :perform).find {|c| c.filter == :rocket_job_transaction}
+            refute RocketJob::Job.send(:get_callbacks, :perform).find {|c| c.filter == :rocket_job_transaction}
+          end
+        end
+
+        describe '#perform' do
+          it 'commits on success' do
+            assert_equal 0, User.count
+            job = CommitTransactionJob.new(login: 'Success')
+            job.perform_now
+            assert job.completed?
+            assert_equal 1, User.count
+            assert_equal 'Success', User.first.login
+          end
+
+          it 'rolls back on exception' do
+            assert_equal 0, User.count
+            job = RollbackTransactionJob.new(login: 'Bad')
+            assert_raises RuntimeError do
+              job.perform_now
+            end
+            assert job.failed?
+            assert_equal 0, User.count
+          end
+        end
+
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.3.1
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-19 00:00:00.000000000 Z
+date: 2017-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -42,16 +42,16 @@ dependencies:
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4.1'
 - !ruby/object:Gem::Dependency
   name: aasm
   requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,6 @@ extra_rdoc_files: []
 files:
 - LICENSE.txt
 - README.md
-- Rakefile
 - bin/rocketjob
 - bin/rocketjob_perf
 - lib/rocket_job/active_worker.rb
@@ -117,6 +116,7 @@ files:
 - lib/rocket_job/version.rb
 - lib/rocket_job/worker.rb
 - lib/rocketjob.rb
+- test/config/database.yml
 - test/config/mongoid.yml
 - test/config_test.rb
 - test/dirmon_entry_test.rb
@@ -138,6 +138,8 @@ files:
 - test/plugins/singleton_test.rb
 - test/plugins/state_machine_event_callbacks_test.rb
 - test/plugins/state_machine_test.rb
+- test/plugins/transaction_test.rb
+- test/test_db.sqlite3
 - test/test_helper.rb
 homepage: http://rocketjob.io
 licenses:
@@ -164,6 +166,7 @@ signing_key:
 specification_version: 4
 summary: Ruby's missing batch system.
 test_files:
+- test/config/database.yml
 - test/config/mongoid.yml
 - test/config_test.rb
 - test/dirmon_entry_test.rb
@@ -185,4 +188,6 @@ test_files:
 - test/plugins/singleton_test.rb
 - test/plugins/state_machine_event_callbacks_test.rb
 - test/plugins/state_machine_test.rb
+- test/plugins/transaction_test.rb
+- test/test_db.sqlite3
 - test/test_helper.rb

data/Rakefile

@@ -1,31 +0,0 @@
-# Setup bundler to avoid having to run bundle exec all the time.
-require 'rubygems'
-require 'bundler/setup'
-
-require 'rake/testtask'
-require_relative 'lib/rocket_job/version'
-
-task :gem do
-  system 'gem build rocketjob.gemspec'
-end
-
-task publish: :gem do
-  system "git tag -a v#{RocketJob::VERSION} -m 'Tagging #{RocketJob::VERSION}'"
-  system 'git push --tags'
-  system "gem push rocketjob-#{RocketJob::VERSION}.gem"
-  system "rm rocketjob-#{RocketJob::VERSION}.gem"
-end
-
-Rake::TestTask.new(:test) do |t|
-  t.pattern = 'test/**/*_test.rb'
-  t.verbose = true
-  t.warning = false
-end
-
-# By default run tests against all appraisals
-if !ENV["APPRAISAL_INITIALIZED"] && !ENV["TRAVIS"]
-  require 'appraisal'
-  task default: :appraisal
-else
-  task default: :test
-end