rocketjob 4.1.0 → 4.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d14befce83747b072cf398356d5a5a798630d65c9aa50ec5f5de1e1f4d0e8d69
-  data.tar.gz: 17c5e295968836458ef1b998d1ffb79228ab6b5971c6339ec1bdc74d429d2511
+  metadata.gz: a88179130b3f8b1570a98c02b22b1daa93d1cfff9e91231f12c08af3eb47c750
+  data.tar.gz: e6fc5a6cf000dc9faa61b61bb232aaef6bf77dec144959e6bfb2114e193845db
 SHA512:
-  metadata.gz: 07b25f25ca1fd804e6c3d98ebcf3724acb8bcb564281f1b5ffd21bd815e8a05d7a99988721c5e3a25063da44f800ea3c998fcbecd7751a8b10755b214804feab
-  data.tar.gz: 48093ed5e152571a097e07f8e606df99fc48fcf1f8364e78b77d371e5e668acf384c972a002c078930b1af809dd7e34483ba28194fbcd52a9ae06bfb60f2dfce
+  metadata.gz: cef575bafd6e780beb0c235bfae3efab3c79ae456cf907c1335372f664ccab1ea125cfd9cbcca17a7791b328b97f78750bc039adf80786bf48834d9d1e6079af
+  data.tar.gz: dd88230b68ecf82433c641f903fcec276b0e873c8849dd1873ed4ee14cd88d3e99fe93ca4ffaf3f973efca16d6837bb1daf3cb91202176ec909a8c19bf28fc08

data/lib/rocket_job/batch/io.rb

@@ -65,6 +65,18 @@ module RocketJob
         count
       end
 
+      def upload_arel(arel, *column_names, category: :main, &block)
+        count             = input(category).upload_arel(arel, *column_names, &block)
+        self.record_count = (record_count || 0) + count
+        count
+      end
+
+      def upload_mongo_query(criteria, *column_names, category: :main, &block)
+        count             = input(category).upload_mongo_query(criteria, *column_names, &block)
+        self.record_count = (record_count || 0) + count
+        count
+      end
+
       # Upload the supplied slices for processing by workers
       #
       # Updates the record_count after adding the records

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -1,4 +1,4 @@
-# Generalized Batch Job.
+# Job to dynamically perform ruby code on demand as a Batch,
 #
 # Often used for data correction or cleansing.
 #

data/lib/rocket_job/jobs/on_demand_job.rb

@@ -1,4 +1,4 @@
-# Generalized Job.
+# Job to dynamically perform ruby code on demand,
 #
 # Create or schedule a generalized job for one off fixes or cleanups.
 #

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -1,4 +1,6 @@
 require 'fileutils'
+require 'uri'
+
 begin
   require 'iostreams'
 rescue LoadError
@@ -91,9 +93,12 @@ module RocketJob
       end
 
       def file_exists
-        return if upload_file_name.nil? || File.exist?(upload_file_name)
+        return if upload_file_name.nil? 
+        uri = URI.parse(upload_file_name)
+        return unless uri.scheme.nil? || uri.scheme == 'file'
+        return if File.exist?(upload_file_name) 
         errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
       end
-    end
+   end
   end
 end

data/lib/rocket_job/plugins/retry.rb

@@ -4,27 +4,12 @@ module RocketJob
   module Plugins
     # Automatically retry the job on failure.
     #
-    # The following retry algorithm is used to automatically retry a job when it fails:
-    # Failed jobs are aborted so that they cannot be restarted since a new instance has there are workers
-    # available to run. For example if workers are busy working on higher priority jobs, then the job
-    # will only run once those jobs have completed, or their priority lowered. Additionally, while the
-    # job is queued no additional instances will be enqueued, even if the next cron interval has been reached.
-    #
-    # Note:
-    # - After failure the job is scheduled to run again in the future.
-    # - The job will not be retried if:
-    #   - The job has expired.
-    #   - The job fails validations.
-    #   - The number of retry counts has been exceeded.
-    # - To see the number of times a job has failed so far:
-    #     job.failure_count
-    #
     # Example:
     #
     # class MyJob < RocketJob::Job
     #   include RocketJob::Plugins::Retry
     #
-    #   # Set the default retry_count
+    #   # Set the maximum number of times a job should be retried before giving up.
     #   self.retry_limit = 3
     #
     #   def perform
@@ -35,10 +20,10 @@ module RocketJob
     # # Queue the job for processing using the default cron_schedule specified above
     # MyJob.create!
     #
-    # # Replace the default retry_count
+    # # Override the default retry_limit for a specific job instance.
     # MyCronJob.create!(retry_limit: 10)
     #
-    # # Disable retries for this job instance
+    # # Disable retries for this job instance.
     # MyCronJob.create!(retry_limit: 0)
     #
     module Retry

data/lib/rocket_job/sliced/input.rb

@@ -107,7 +107,6 @@ module RocketJob
           IOStreams.public_send(iterator, file_name_or_io, encoding: encoding, **args) { |line| io << line }
         end
 
-        create_indexes
         Writer::Input.collect(self, on_first: on_first, &block)
       end
 
@@ -141,7 +140,6 @@ module RocketJob
       #   criteria = User.where(state: 'FL')
       #   job.record_count = job.upload_mongo_query(criteria, :zip_code)
       def upload_mongo_query(criteria, *column_names, &block)
-        create_indexes
         options = criteria.options
 
         # Without a block extract the fields from the supplied criteria
@@ -198,7 +196,6 @@ module RocketJob
       #   arel = User.where(country_code: 'US')
       #   job.record_count = job.upload_arel(arel, :user_name, :zip_code)
       def upload_arel(arel, *column_names, &block)
-        create_indexes
         unless block
           column_names = column_names.collect(&:to_sym)
           column_names << :id if column_names.size.zero?

data/lib/rocket_job/sliced/slices.rb

@@ -21,7 +21,9 @@ module RocketJob
         @slice_class     = slice_class
         @slice_size      = slice_size
         @collection_name = collection_name
-        @all             = slice_class.with_collection(collection_name)
+
+        # Using `Sliced::Slice` avoids having to add `_type` as an index when all slices are the same type anyway.
+        @all = Sliced::Slice.with_collection(collection_name)
       end
 
       def new(params = {})

data/lib/rocket_job/sliced/writer/input.rb

@@ -14,6 +14,8 @@ module RocketJob
         #     Default: nil
         def self.collect(input, **args, &block)
           writer = new(input, **args)
+          # Create indexes before uploading
+          input.create_indexes if input.respond_to?(:create_indexes)
           block.call(writer)
           writer.record_count
         rescue Exception => exc

data/lib/rocket_job/supervisor/shutdown.rb

@@ -41,7 +41,7 @@ module RocketJob
             Thread.new do
               shutdown!
               message = 'Shutdown signal (SIGTERM) received. Will shutdown as soon as active jobs/slices have completed.'
-              logger.warn(message)
+              logger.info(message)
             end
           end
 
@@ -49,7 +49,7 @@ module RocketJob
             Thread.new do
               shutdown!
               message = 'Shutdown signal (INT) received. Will shutdown as soon as active jobs/slices have completed.'
-              logger.warn(message)
+              logger.info(message)
             end
           end
         rescue StandardError

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = '4.1.0'.freeze
+  VERSION = '4.1.1'.freeze
 end

data/lib/rocket_job/worker_pool.rb

@@ -78,7 +78,7 @@ module RocketJob
       true
     end
 
-    # Returns [Fixnum] number of workers (threads) that are alive
+    # Returns [Integer] number of workers (threads) that are alive
     def living_count
       workers.count(&:alive?)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.1.1
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-08 00:00:00.000000000 Z
+date: 2019-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -193,7 +193,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Ruby's missing batch system.