rocketjob 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 201f922d37f2533e56109c053e3c9f4bdef30108
-  data.tar.gz: 00598e8fb3039d7ce629f629bf270d3cc06f2e87
+  metadata.gz: 538bb39f185138c3fc5c4e007ef33470da129cfd
+  data.tar.gz: d0edc858039527f0fb33707bd563c3dacce0a968
 SHA512:
-  metadata.gz: 447f63cf687c4e7ac10189aff59730874ea25f3e57b7d6ca301fd4a34178e91d68d4428f5be095c2a6a4a5426d603262c3d82c78e9f3988a9ed35d76bf75bddf
-  data.tar.gz: e6a54a1b72cb2c7192894140eea4e49e42dc29049a85a77338de773e539c23a5e1ed09a5691a0a5c238a881e99398cb07b99eddf5abdda8e044971cf433766f3
+  metadata.gz: 73581f734996afa564614e434015ed6400461e755607b8af6652a29eee120d318415110058b9014e6224b15b594933a12d79ee023ec8feaf8c59b6335f173392
+  data.tar.gz: 896bc0dbfece93ff239aa56475a733bf1b310792e47d89be8bc17dce4c00e04ab426ba3532ad7cc1a864dc0f4a0eb280f8a177899a0ff1ec77f33d160adb0b3d

data/README.md

@@ -1,6 +1,10 @@
 # rocketjob [![Gem Version](https://badge.fury.io/rb/rocketjob.svg)](http://badge.fury.io/rb/rocketjob) [![Build Status](https://secure.travis-ci.org/rocketjob/rocketjob.png?branch=master)](http://travis-ci.org/rocketjob/rocketjob) ![](http://ruby-gem-downloads-badge.herokuapp.com/rocketjob?type=total)
 
-High volume, priority based, distributed, background job processing solution for Ruby.
+Enterprise Batch Processing System focused on performance, scalability, reliability, and visibility of every job in the system.
+
+Outgrown existing solutions? Or, start small and scale up later.
+
+Works with or without Rails.
 
 ## Status
 

data/lib/rocket_job/concerns/worker.rb

@@ -22,19 +22,7 @@ module RocketJob
 
         # Create a job and process it immediately in-line by this thread
         def now(method, *args, &block)
-          job = build(method, *args, &block)
-          # Call validations
-          if job.respond_to?(:validate!)
-            job.validate!
-          elsif job.invalid?
-            raise(MongoMapper::DocumentNotValid, "Validation failed: #{job.errors.messages.join(', ')}")
-          end
-          worker = RocketJob::Worker.new(name: 'inline')
-          worker.started
-          job.start
-          while job.running? && !job.work(worker)
-          end
-          job
+          build(method, *args, &block).work_now
         end
 
         # Build a Rocket Job instance
@@ -134,6 +122,8 @@ module RocketJob
         begin
           # before_perform
           call_method(perform_method, arguments, event: :before, log_level: log_level)
+          # Allow before perform to explicitly fail this job
+          return unless running?
 
           # perform
           ret = call_method(perform_method, arguments, log_level: log_level)
@@ -141,18 +131,39 @@ module RocketJob
             self.result = (ret.is_a?(Hash) || ret.is_a?(BSON::OrderedHash)) ? ret : {result: ret}
           end
 
+          # Only run after perform if perform did not explicitly fail the job
+          return unless running?
+
           # after_perform
           call_method(perform_method, arguments, event: :after, log_level: log_level)
 
-          complete!
+          new_record? ? complete : complete!
         rescue StandardError => exc
-          fail!(worker.name, exc) unless failed?
+          fail(worker.name, exc) if may_fail?
           logger.error("Exception running #{self.class.name}##{perform_method}", exc)
+          save! unless new_record?
           raise exc if RocketJob::Config.inline_mode
         end
         false
       end
 
+      # Validates and runs the work on this job now in the current thread
+      # Returns this job once it has finished running
+      def work_now
+        # Call validations
+        if respond_to?(:validate!)
+          validate!
+        elsif invalid?
+          raise(MongoMapper::DocumentNotValid, "Validation failed: #{errors.messages.join(', ')}")
+        end
+        worker = RocketJob::Worker.new(name: 'inline')
+        worker.started
+        start if may_start?
+        while running? && !work(worker)
+        end
+        self
+      end
+
       protected
 
       # Calls a method on this job, if it is defined

data/lib/rocket_job/config.rb

@@ -1,18 +1,16 @@
 # encoding: UTF-8
-require 'sync_attr'
 module RocketJob
   # Centralized Configuration for Rocket Jobs
   class Config
     include MongoMapper::Document
-    include SyncAttr
 
     # Prevent data in MongoDB from re-defining the model behavior
     #self.static_keys = true
 
     # Returns the single instance of the Rocket Job Configuration for this site
     # in a thread-safe way
-    sync_cattr_reader(:instance) do
-      begin
+    def self.instance
+      @@instance ||= begin
         first || create
       rescue StandardError
         # In case another process has already created the first document
@@ -22,7 +20,7 @@ module RocketJob
 
     # By enabling inline_mode jobs will be called in-line
     # No worker processes will be created, nor threads created
-    sync_cattr_accessor(:inline_mode) { false }
+    cattr_accessor(:inline_mode) { false }
 
     # @formatter:off
     # The maximum number of worker threads to create on any one worker

data/lib/rocket_job/dirmon_entry.rb

@@ -280,13 +280,16 @@ module RocketJob
 
     # Queues the job for the supplied pathname
     def later(pathname)
-      job_class.perform_later(*arguments) do |job|
-        job.perform_method = perform_method
-        # Set properties
-        properties.each_pair { |k, v| job.send("#{k}=".to_sym, v) }
-
-        upload_file(job, pathname)
-      end
+      job = job_class.new(
+        properties.merge(
+          arguments:      arguments,
+          properties:     properties,
+          perform_method: perform_method
+        )
+      )
+      upload_file(job, pathname)
+      job.save!
+      job
     end
 
     protected
@@ -315,9 +318,14 @@ module RocketJob
 
     # Archives the file for a job where there was no #file_store_upload or #upload method
     def upload_default(job, pathname)
-      # The first argument must be a hash
-      job.arguments << {} if job.arguments.size == 0
-      job.arguments.first[:full_file_name] = archive_file(job, pathname)
+      full_file_name = archive_file(job, pathname)
+      if job.respond_to?(:full_file_name=)
+        job.full_file_name = full_file_name
+      elsif job.arguments.first.is_a?(Hash)
+        job.arguments.first[:full_file_name] = full_file_name
+      else
+        raise(ArgumentError, "#{job_class_name} must either have attribute 'full_file_name' or the first argument must be a Hash")
+      end
     end
 
     # Move the file to the archive directory

data/lib/rocket_job/job.rb

@@ -120,6 +120,20 @@ module RocketJob
 
     validates_presence_of :state, :failure_count, :created_at, :perform_method
     validates :priority, inclusion: 1..100
+    validates :log_level, inclusion: SemanticLogger::LEVELS + [nil]
+
+    # User definable properties in Dirmon Entry
+    def self.rocket_job_properties
+      @rocket_job_properties ||= (self == RocketJob::Job ? [] : superclass.rocket_job_properties)
+    end
+
+    # Add to user definable properties in Dirmon Entry
+    def self.public_rocket_job_properties(*properties)
+      rocket_job_properties.concat(properties).uniq!
+    end
+
+    # User definable properties in Dirmon Entry
+    public_rocket_job_properties :description, :priority, :perform_method, :log_level, :arguments
 
     # State Machine events and transitions
     #

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -41,27 +41,29 @@ module RocketJob
 
       # Number of seconds between directory scans. Default 5 mins
       key :check_seconds, Float, default: 300.0
+      key :previous_file_names, Hash # Hash[file_name, size]
 
       # Iterate over each Dirmon entry looking for new files
       # If a new file is found, it is not processed immediately, instead
       # it is passed to the next run of this job along with the file size.
       # If the file size has not changed, the Job is kicked off.
-      def perform(previous_file_names={})
-        new_file_names = check_directories(previous_file_names)
+      def perform
+        check_directories
       ensure
         # Run again in the future, even if this run fails with an exception
-        self.class.perform_later(new_file_names || previous_file_names) do |job|
-          job.priority      = priority
-          job.check_seconds = check_seconds
-          job.run_at        = Time.now + check_seconds
-        end
+        self.class.create!(
+          previous_file_names: previous_file_names,
+          priority:            priority,
+          check_seconds:       check_seconds,
+          run_at:              Time.now + check_seconds
+        )
       end
 
       protected
 
       # Checks the directories for new files, starting jobs if files have not changed
       # since the last run
-      def check_directories(previous_file_names)
+      def check_directories
         new_file_names = {}
         DirmonEntry.where(state: :enabled).each do |entry|
           entry.each do |pathname|
@@ -73,7 +75,7 @@ module RocketJob
             end
           end
         end
-        new_file_names
+        self.previous_file_names = new_file_names
       end
 
       # Checks if a file should result in starting a job

data/lib/rocket_job/version.rb

@@ -1,4 +1,4 @@
 # encoding: UTF-8
 module RocketJob #:nodoc
-  VERSION = '1.2.1'
+  VERSION = '1.3.0'
 end

data/lib/rocket_job/worker.rb

@@ -1,6 +1,5 @@
 # encoding: UTF-8
 require 'socket'
-require 'sync_attr'
 require 'aasm'
 module RocketJob
   # Worker
@@ -30,7 +29,6 @@ module RocketJob
   class Worker
     include MongoMapper::Document
     include AASM
-    include SyncAttr
     include SemanticLogger::Loggable
 
     # Prevent data in MongoDB from re-defining the model behavior
@@ -164,7 +162,7 @@ module RocketJob
       @thread_pool ||= []
     end
 
-    # Run this instance of the worker
+    # Management Thread
     def run
       Thread.current.name = 'rocketjob main'
       build_heartbeat unless heartbeat
@@ -182,7 +180,7 @@ module RocketJob
           'heartbeat.current_threads' => thread_pool_count
         )
 
-        # Reload the worker model every 10 heartbeats in case its config was changed
+        # Reload the worker model every few heartbeats in case its config was changed
         # TODO make 3 configurable
         if count >= 3
           reload
@@ -316,11 +314,13 @@ module RocketJob
       RocketJob::Job.requeue_dead_worker(name)
     end
 
-    # Mutex protected shutdown indicator
-    sync_cattr_accessor :shutdown do
-      false
+    # Shutdown indicator
+    def self.shutdown
+      @@shutdown
     end
 
+    @@shutdown = false
+
     # Register handlers for the various signals
     # Term:
     #   Perform clean shutdown

data/test/dirmon_entry_test.rb

@@ -3,6 +3,16 @@ require_relative 'jobs/test_job'
 
 # Unit Test for RocketJob::Job
 class DirmonEntryTest < Minitest::Test
+  class WithFullFileNameJob < RocketJob::Job
+    # Dirmon will store the filename in this property when starting the job
+    key :full_file_name, String
+
+    def perform
+      # Do something with the file name stored in :full_file_name
+    end
+  end
+
+
   describe RocketJob::DirmonEntry do
     describe '.config' do
       it 'support multiple databases' do
@@ -186,11 +196,17 @@ class DirmonEntryTest < Minitest::Test
         @entry        = RocketJob::DirmonEntry.new(
           pattern:           'test/files/**/*',
           job_class_name:    'Jobs::TestJob',
-          arguments:         [{input: 'yes'}],
+          arguments:         [{}],
           properties:        {priority: 23, perform_method: :event},
           archive_directory: @archive_directory
         )
-        @job          = Jobs::TestJob.new
+        @job          = Jobs::TestJob.new(
+          @entry.properties.merge(
+            arguments:      @entry.arguments,
+            properties:     @entry.properties,
+            perform_method: @entry.perform_method
+          )
+        )
         @file         = Tempfile.new('archive')
         @file_name    = @file.path
         @pathname     = Pathname.new(@file_name)
@@ -222,10 +238,30 @@ class DirmonEntryTest < Minitest::Test
       end
 
       describe '#upload_default' do
-        it 'upload' do
+        it 'sets full_file_name in Hash argument' do
           @entry.send(:upload_default, @job, @pathname)
           assert_equal @archive_real_name, @job.arguments.first[:full_file_name], @job.arguments
         end
+
+        it 'sets full_file_name property' do
+          @entry = RocketJob::DirmonEntry.new(
+            pattern:           'test/files/**/*',
+            job_class_name:    'DirmonEntryTest::WithFullFileNameJob',
+            archive_directory: @archive_directory
+          )
+          assert @entry.valid?, @entry.errors.messages
+          job = @entry.job_class.new
+          @entry.send(:upload_default, job, @pathname)
+          archive_real_name = @archive_path.join("#{job.id}_#{File.basename(@file_name)}").to_s
+          assert_equal archive_real_name, job.full_file_name, job.arguments
+        end
+
+        it 'handles non hash argument and missing property' do
+          @job.arguments = [1]
+          assert_raises ArgumentError do
+            @entry.send(:upload_default, @job, @pathname)
+          end
+        end
       end
 
       describe '#upload_file' do

data/test/dirmon_job_test.rb

@@ -73,8 +73,7 @@ class DirmonJobTest < Minitest::Test
       end
 
       it 'no files' do
-        previous_file_names = {}
-        result              = @dirmon_job.send(:check_directories, previous_file_names)
+        result = @dirmon_job.send(:check_directories)
         assert_equal 0, result.count
       end
 
@@ -82,37 +81,40 @@ class DirmonJobTest < Minitest::Test
         create_file("#{@directory}/abc/file1", 5)
         create_file("#{@directory}/abc/file2", 10)
 
-        previous_file_names = {}
-        result              = @dirmon_job.send(:check_directories, previous_file_names)
-        assert_equal 2, result.count, result.inspect
-        assert_equal 5, result.values.first, result.inspect
-        assert_equal 10, result.values.second, result.inspect
+        result = @dirmon_job.send(:check_directories)
+        assert_equal 2, result.count, result
+        assert_equal 5, result.values.first, result
+        assert_equal 10, result.values.second, result
       end
 
       it 'allow files to grow' do
         create_file("#{@directory}/abc/file1", 5)
         create_file("#{@directory}/abc/file2", 10)
-        previous_file_names = {}
-        @dirmon_job.send(:check_directories, previous_file_names)
+        @dirmon_job.send(:check_directories)
         create_file("#{@directory}/abc/file1", 10)
         create_file("#{@directory}/abc/file2", 15)
-        result = @dirmon_job.send(:check_directories, previous_file_names)
-        assert_equal 2, result.count, result.inspect
-        assert_equal 10, result.values.first, result.inspect
-        assert_equal 15, result.values.second, result.inspect
+        result = @dirmon_job.send(:check_directories)
+        assert_equal 2, result.count, result
+        assert_equal 10, result.values.first, result
+        assert_equal 15, result.values.second, result
       end
 
       it 'start all files' do
         create_file("#{@directory}/abc/file1", 5)
         create_file("#{@directory}/abc/file2", 10)
-        previous_file_names = @dirmon_job.send(:check_directories, {})
+        files = @dirmon_job.send(:check_directories)
+        assert_equal 2, files.count, files
+        assert_equal 2, @dirmon_job.previous_file_names.count, files
+
+        # files = @dirmon_job.send(:check_directories)
+        # assert_equal 0, files.count, files
 
         count  = 0
         result = RocketJob::DirmonEntry.stub_any_instance(:later, -> path { count += 1 }) do
-          @dirmon_job.send(:check_directories, previous_file_names)
+          @dirmon_job.send(:check_directories)
         end
+        assert_equal 0, result.count, result
         assert 2, count
-        assert_equal 0, result.count, result.inspect
       end
 
       it 'skip files in archive directory' do
@@ -125,11 +127,11 @@ class DirmonJobTest < Minitest::Test
         FileUtils.makedirs(@entry.archive_pathname(file_pathname))
         create_file("#{@entry.archive_pathname(file_pathname)}/file3", 10)
 
-        result = @dirmon_job.send(:check_directories, {})
+        result = @dirmon_job.send(:check_directories)
 
-        assert_equal 2, result.count, result.inspect
-        assert_equal 5, result.values.first, result.inspect
-        assert_equal 10, result.values.second, result.inspect
+        assert_equal 2, result.count, result
+        assert_equal 5, result.values.first, result
+        assert_equal 10, result.values.second, result
       end
     end
 
@@ -147,10 +149,12 @@ class DirmonJobTest < Minitest::Test
         RocketJob::Jobs::DirmonJob.destroy_all
         RocketJob::Jobs::DirmonJob.stub_any_instance(:check_directories, new_file_names) do
           # perform_now does not save the job, just runs it
-          dirmon_job = RocketJob::Jobs::DirmonJob.perform_now(previous_file_names) do |job|
-            job.priority      = 11
-            job.check_seconds = 30
-          end
+          dirmon_job = RocketJob::Jobs::DirmonJob.new(
+            previous_file_names: previous_file_names,
+            priority:            11,
+            check_seconds:       30
+          )
+          dirmon_job.work_now
         end
         assert dirmon_job.completed?, dirmon_job.status.inspect
 
@@ -171,14 +175,15 @@ class DirmonJobTest < Minitest::Test
         RocketJob::Jobs::DirmonJob.destroy_all
         RocketJob::Jobs::DirmonJob.stub_any_instance(:check_directories, -> previous { raise RuntimeError.new("Oh no") }) do
           # perform_now does not save the job, just runs it
-          dirmon_job = RocketJob::Jobs::DirmonJob.perform_now do |job|
-            job.priority      = 11
-            job.check_seconds = 30
-          end
+          dirmon_job = RocketJob::Jobs::DirmonJob.create!(
+            priority:      11,
+            check_seconds: 30
+          )
+          dirmon_job.work_now
         end
         assert dirmon_job.failed?, dirmon_job.status.inspect
 
-        # It it have enqueued another instance to run in the future
+        # Must have enqueued another instance to run in the future
         assert_equal 2, RocketJob::Jobs::DirmonJob.count
         assert new_dirmon_job = RocketJob::Jobs::DirmonJob.last
         assert new_dirmon_job.run_at

data/test/job_test.rb

@@ -154,7 +154,8 @@ class JobTest < Minitest::Test
         @job.destroy_on_complete = true
         @job.start!
         assert_equal false, @job.work(@worker)
-        assert_equal nil, RocketJob::Job.find_by_id(@job.id)
+        assert @job.completed?, @job.state
+        assert_equal 0, RocketJob::Job.where(id: @job.id).count
       end
 
       it 'silence logging when log_level is set' do
@@ -254,8 +255,9 @@ class JobTest < Minitest::Test
       it 'requeue jobs from dead workers' do
         worker_name      = 'server:12345'
         @job.worker_name = worker_name
+        assert @job.valid?, @job.errors.messages
         @job.start!
-        assert @job.running?
+        assert @job.running?, @job.state
 
         @job.requeue
         assert @job.queued?
@@ -273,7 +275,7 @@ class JobTest < Minitest::Test
         worker_name      = 'server:12345'
         @job.worker_name = worker_name
         @job.start!
-        assert @job.running?
+        assert @job.running?, @job.state
 
         worker_name2      = 'server:76467'
         @job2.worker_name = worker_name2

data/test/test_helper.rb

@@ -8,7 +8,13 @@ require 'rocketjob'
 require 'awesome_print'
 require 'symmetric-encryption'
 
-Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
+if ENV['DETAILED_TESTS'].present?
+  # See every test and how long it took
+  MiniTest::Reporters.use! MiniTest::Reporters::SpecReporter.new
+else
+  # Only show failed tests
+  MiniTest::Reporters.use! MiniTest::Reporters::ProgressReporter.new
+end
 
 SemanticLogger.add_appender('test.log', &SemanticLogger::Appender::Base.colorized_formatter)
 SemanticLogger.default_level = :debug

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-15 00:00:00.000000000 Z
+date: 2015-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -94,22 +94,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: sync_attr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-description: Next generation, high performance, priority based, distributed, background
-  job processing solution
+description: Enterprise Batch Processing System focused on performance, scalability,
+  reliability, and visibility of every job in the system.
 email:
 - reidmo@gmail.com
 executables:
@@ -165,7 +151,7 @@ rubyforge_project:
 rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
-summary: Next generation background job processing system for Ruby, JRuby and Rubinius
+summary: Enterprise Batch Processing System for Ruby, JRuby, and Rubinius
 test_files:
 - test/config/mongo.yml
 - test/dirmon_entry_test.rb