rocketjob 5.4.1 → 6.0.0

data/lib/rocket_job/worker_pool.rb

@@ -69,12 +69,10 @@ module RocketJob
     # Return [false] on timeout
     def join(timeout = 5)
       while (worker = workers.first)
-        if worker.join(timeout)
-          # Worker thread is dead
-          workers.shift
-        else
-          return false
-        end
+        return false unless worker.join(timeout)
+
+        # Worker thread is dead
+        workers.shift
       end
       true
     end
@@ -91,7 +89,7 @@ module RocketJob
     private
 
     def add_one
-      workers << Worker.new(id: next_worker_id, server_name: server_name)
+      workers << ThreadWorker.new(id: next_worker_id, server_name: server_name)
     rescue StandardError => e
       logger.fatal("Cannot start worker", e)
     end

data/lib/rocketjob.rb

@@ -3,6 +3,8 @@ require "semantic_logger"
 require "symmetric-encryption"
 require "mongoid"
 require "rocket_job/extensions/mongo/logging"
+require "rocket_job/extensions/iostreams/path"
+require "rocket_job/extensions/psych/yaml_tree"
 require "rocket_job/version"
 require "rocket_job/rocket_job"
 require "rocket_job/config"
@@ -13,28 +15,37 @@ require "rocket_job/extensions/mongoid/clients/options"
 require "rocket_job/extensions/mongoid/contextual/mongo"
 require "rocket_job/extensions/mongoid/factory"
 
-# Apply patches for deprecated Symbol type
-require "rocket_job/extensions/mongoid/remove_warnings"
+# Backport New StringifiedSymbol type in Mongoid v7.2
+require "rocket_job/extensions/mongoid/stringified_symbol" unless defined?(Mongoid::StringifiedSymbol)
 
 # @formatter:off
 module RocketJob
-  autoload :ActiveWorker,       "rocket_job/active_worker"
-  autoload :Batch,              "rocket_job/batch"
-  autoload :CLI,                "rocket_job/cli"
-  autoload :DirmonEntry,        "rocket_job/dirmon_entry"
-  autoload :Event,              "rocket_job/event"
-  autoload :Heartbeat,          "rocket_job/heartbeat"
-  autoload :Job,                "rocket_job/job"
-  autoload :JobException,       "rocket_job/job_exception"
-  autoload :Worker,             "rocket_job/worker"
-  autoload :Performance,        "rocket_job/performance"
-  autoload :Server,             "rocket_job/server"
-  autoload :Sliced,             "rocket_job/sliced"
-  autoload :Subscriber,         "rocket_job/subscriber"
-  autoload :Supervisor,         "rocket_job/supervisor"
-  autoload :ThrottleDefinition, "rocket_job/throttle_definition"
-  autoload :ThrottleDefinitions, "rocket_job/throttle_definitions"
-  autoload :WorkerPool, "rocket_job/worker_pool"
+  autoload :ActiveWorker,            "rocket_job/active_worker"
+  autoload :Batch,                   "rocket_job/batch"
+  autoload :CLI,                     "rocket_job/cli"
+  autoload :DirmonEntry,             "rocket_job/dirmon_entry"
+  autoload :Event,                   "rocket_job/event"
+  autoload :Heartbeat,               "rocket_job/heartbeat"
+  autoload :Job,                     "rocket_job/job"
+  autoload :JobException,            "rocket_job/job_exception"
+  autoload :LookupCollection,        "rocket_job/lookup_collection"
+  autoload :Worker,                  "rocket_job/worker"
+  autoload :Performance,             "rocket_job/performance"
+  autoload :RactorWorker,            "rocket_job/ractor_worker"
+  autoload :Server,                  "rocket_job/server"
+  autoload :Sliced,                  "rocket_job/sliced"
+  autoload :Subscriber,              "rocket_job/subscriber"
+  autoload :Supervisor,              "rocket_job/supervisor"
+  autoload :ThreadWorker,            "rocket_job/thread_worker"
+  autoload :ThrottleDefinition,      "rocket_job/throttle_definition"
+  autoload :ThrottleDefinitions,     "rocket_job/throttle_definitions"
+  autoload :WorkerPool,              "rocket_job/worker_pool"
+
+  module Category
+    autoload :Base,                  "rocket_job/category/base"
+    autoload :Input,                 "rocket_job/category/input"
+    autoload :Output,                "rocket_job/category/output"
+  end
 
   module Plugins
     module Job
@@ -49,38 +60,37 @@ module RocketJob
       autoload :Transaction,         "rocket_job/plugins/job/transaction"
       autoload :Worker,              "rocket_job/plugins/job/worker"
     end
-    autoload :Cron,             "rocket_job/plugins/cron"
-    autoload :Document,         "rocket_job/plugins/document"
-    autoload :ProcessingWindow, "rocket_job/plugins/processing_window"
-    autoload :Restart,          "rocket_job/plugins/restart"
-    autoload :Retry,            "rocket_job/plugins/retry"
-    autoload :Singleton,        "rocket_job/plugins/singleton"
-    autoload :StateMachine,     "rocket_job/plugins/state_machine"
-    autoload :Transaction,      "rocket_job/plugins/transaction"
+    autoload :Cron,                  "rocket_job/plugins/cron"
+    autoload :Document,              "rocket_job/plugins/document"
+    autoload :ProcessingWindow,      "rocket_job/plugins/processing_window"
+    autoload :Retry,                 "rocket_job/plugins/retry"
+    autoload :Singleton,             "rocket_job/plugins/singleton"
+    autoload :StateMachine,          "rocket_job/plugins/state_machine"
+    autoload :Transaction,           "rocket_job/plugins/transaction"
+    autoload :ThrottleDependentJobs, "rocket_job/plugins/throttle_dependent_jobs"
   end
 
   module Jobs
-    autoload :ActiveJob,               "rocket_job/jobs/active_job"
-    autoload :CopyFileJob,             "rocket_job/jobs/copy_file_job"
-    autoload :DirmonJob,               "rocket_job/jobs/dirmon_job"
-    autoload :OnDemandBatchJob,        "rocket_job/jobs/on_demand_batch_job"
-    autoload :OnDemandBatchTabularJob, "rocket_job/jobs/on_demand_batch_tabular_job"
-    autoload :OnDemandJob,             "rocket_job/jobs/on_demand_job"
-    autoload :HousekeepingJob,         "rocket_job/jobs/housekeeping_job"
-    autoload :PerformanceJob,          "rocket_job/jobs/performance_job"
-    autoload :SimpleJob,               "rocket_job/jobs/simple_job"
-    autoload :UploadFileJob,           "rocket_job/jobs/upload_file_job"
+    autoload :ActiveJob,             "rocket_job/jobs/active_job"
+    autoload :ConversionJob,         "rocket_job/jobs/conversion_job"
+    autoload :CopyFileJob,           "rocket_job/jobs/copy_file_job"
+    autoload :DirmonJob,             "rocket_job/jobs/dirmon_job"
+    autoload :HousekeepingJob,       "rocket_job/jobs/housekeeping_job"
+    autoload :OnDemandBatchJob,      "rocket_job/jobs/on_demand_batch_job"
+    autoload :OnDemandJob,           "rocket_job/jobs/on_demand_job"
+    autoload :PerformanceJob,        "rocket_job/jobs/performance_job"
+    autoload :SimpleJob,             "rocket_job/jobs/simple_job"
+    autoload :UploadFileJob,         "rocket_job/jobs/upload_file_job"
+
     module ReEncrypt
-      if defined?(ActiveRecord) && defined?(SyncAttr)
-        autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
-      end
+      autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
     end
   end
 
   module Subscribers
-    autoload :Logger,           "rocket_job/subscribers/logger"
-    autoload :Server,           "rocket_job/subscribers/server"
-    autoload :Worker,           "rocket_job/subscribers/worker"
+    autoload :Logger,                "rocket_job/subscribers/logger"
+    autoload :Server,                "rocket_job/subscribers/server"
+    autoload :Worker,                "rocket_job/subscribers/worker"
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 5.4.1
+  version: 6.0.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-09 00:00:00.000000000 Z
+date: 2021-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.12'
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.12'
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -39,75 +39,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: iostreams
+  name: fugit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: mongoid
+  name: iostreams
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '1.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '1.9'
 - !ruby/object:Gem::Dependency
-  name: semantic_logger
+  name: mongoid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
-  name: symmetric-encryption
+  name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.7'
 - !ruby/object:Gem::Dependency
-  name: fugit
+  name: symmetric-encryption
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '4.3'
 description:
 email:
 executables:
@@ -124,6 +124,7 @@ files:
 - lib/rocket_job/active_worker.rb
 - lib/rocket_job/batch.rb
 - lib/rocket_job/batch/callbacks.rb
+- lib/rocket_job/batch/categories.rb
 - lib/rocket_job/batch/io.rb
 - lib/rocket_job/batch/logger.rb
 - lib/rocket_job/batch/lower_priority.rb
@@ -133,37 +134,40 @@ files:
 - lib/rocket_job/batch/results.rb
 - lib/rocket_job/batch/state_machine.rb
 - lib/rocket_job/batch/statistics.rb
-- lib/rocket_job/batch/tabular.rb
-- lib/rocket_job/batch/tabular/input.rb
-- lib/rocket_job/batch/tabular/output.rb
 - lib/rocket_job/batch/throttle.rb
 - lib/rocket_job/batch/throttle_running_workers.rb
 - lib/rocket_job/batch/throttle_windows.rb
 - lib/rocket_job/batch/worker.rb
+- lib/rocket_job/category/base.rb
+- lib/rocket_job/category/input.rb
+- lib/rocket_job/category/output.rb
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
 - lib/rocket_job/dirmon_entry.rb
 - lib/rocket_job/event.rb
+- lib/rocket_job/extensions/iostreams/path.rb
 - lib/rocket_job/extensions/mongo/logging.rb
 - lib/rocket_job/extensions/mongoid/clients/options.rb
 - lib/rocket_job/extensions/mongoid/contextual/mongo.rb
 - lib/rocket_job/extensions/mongoid/factory.rb
-- lib/rocket_job/extensions/mongoid/remove_warnings.rb
+- lib/rocket_job/extensions/mongoid/stringified_symbol.rb
+- lib/rocket_job/extensions/psych/yaml_tree.rb
 - lib/rocket_job/extensions/rocket_job_adapter.rb
 - lib/rocket_job/heartbeat.rb
 - lib/rocket_job/job.rb
 - lib/rocket_job/job_exception.rb
 - lib/rocket_job/jobs/active_job.rb
+- lib/rocket_job/jobs/conversion_job.rb
 - lib/rocket_job/jobs/copy_file_job.rb
 - lib/rocket_job/jobs/dirmon_job.rb
 - lib/rocket_job/jobs/housekeeping_job.rb
 - lib/rocket_job/jobs/on_demand_batch_job.rb
-- lib/rocket_job/jobs/on_demand_batch_tabular_job.rb
 - lib/rocket_job/jobs/on_demand_job.rb
 - lib/rocket_job/jobs/performance_job.rb
 - lib/rocket_job/jobs/re_encrypt/relational_job.rb
 - lib/rocket_job/jobs/simple_job.rb
 - lib/rocket_job/jobs/upload_file_job.rb
+- lib/rocket_job/lookup_collection.rb
 - lib/rocket_job/performance.rb
 - lib/rocket_job/plugins/cron.rb
 - lib/rocket_job/plugins/document.rb
@@ -180,7 +184,9 @@ files:
 - lib/rocket_job/plugins/retry.rb
 - lib/rocket_job/plugins/singleton.rb
 - lib/rocket_job/plugins/state_machine.rb
+- lib/rocket_job/plugins/throttle_dependent_jobs.rb
 - lib/rocket_job/plugins/transaction.rb
+- lib/rocket_job/ractor_worker.rb
 - lib/rocket_job/railtie.rb
 - lib/rocket_job/rocket_job.rb
 - lib/rocket_job/server.rb
@@ -189,6 +195,7 @@ files:
 - lib/rocket_job/sliced.rb
 - lib/rocket_job/sliced/bzip2_output_slice.rb
 - lib/rocket_job/sliced/compressed_slice.rb
+- lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb
 - lib/rocket_job/sliced/encrypted_slice.rb
 - lib/rocket_job/sliced/input.rb
 - lib/rocket_job/sliced/output.rb
@@ -202,13 +209,14 @@ files:
 - lib/rocket_job/subscribers/worker.rb
 - lib/rocket_job/supervisor.rb
 - lib/rocket_job/supervisor/shutdown.rb
+- lib/rocket_job/thread_worker.rb
 - lib/rocket_job/throttle_definition.rb
 - lib/rocket_job/throttle_definitions.rb
 - lib/rocket_job/version.rb
 - lib/rocket_job/worker.rb
 - lib/rocket_job/worker_pool.rb
 - lib/rocketjob.rb
-homepage: http://rocketjob.io
+homepage: https://rocketjob.io
 licenses:
 - Apache-2.0
 metadata: {}
@@ -227,7 +235,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Ruby's missing batch processing system.

data/lib/rocket_job/batch/tabular/input.rb

@@ -1,131 +0,0 @@
-require "active_support/concern"
-
-module RocketJob
-  module Batch
-    class Tabular
-      # For the simple case where all `input_categories` have the same format,
-      # If multiple input categories are used with different formats, then use IOStreams::Tabular directly
-      # instead of this plugin.
-      module Input
-        extend ActiveSupport::Concern
-
-        included do
-          field :tabular_input_header, type: Array, class_attribute: true, user_editable: true
-          field :tabular_input_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true
-          field :tabular_input_options, type: Hash, class_attribute: true
-
-          # tabular_input_mode: [:line | :array | :hash]
-          #   :line
-          #     Uploads the file a line (String) at a time for processing by workers.
-          #   :array
-          #     Parses each line from the file as an Array and uploads each array for processing by workers.
-          #   :hash
-          #     Parses each line from the file into a Hash and uploads each hash for processing by workers.
-          #   See IOStreams#each.
-          field :tabular_input_mode, type: Symbol, default: :line, class_attribute: true, user_editable: true, copy_on_restart: true
-
-          validates_inclusion_of :tabular_input_format, in: IOStreams::Tabular.registered_formats
-          validates_inclusion_of :tabular_input_mode, in: %i[line array hash row record]
-          validate :tabular_input_header_present
-
-          class_attribute :tabular_input_white_list
-          class_attribute :tabular_input_required
-          class_attribute :tabular_input_skip_unknown
-
-          # Cleanse all uploaded data by removing non-printable characters
-          # and any characters that cannot be converted to UTF-8
-          class_attribute :tabular_input_type
-
-          self.tabular_input_white_list   = nil
-          self.tabular_input_required     = nil
-          self.tabular_input_skip_unknown = true
-          self.tabular_input_type         = :text
-
-          before_perform :tabular_input_render
-        end
-
-        # Extract the header line during the upload.
-        #
-        # Overrides: RocketJob::Batch::IO#upload
-        #
-        # Notes:
-        # - When supplying a block the header must be set manually
-        def upload(stream = nil, **args, &block)
-          input_stream = stream.nil? ? nil : IOStreams.new(stream)
-
-          if stream && (tabular_input_type == :text)
-            # Cannot change the length of fixed width lines
-            replace = tabular_input_format == :fixed ? " " : ""
-            input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
-          end
-
-          # If an input header is not required, then we don't extract it'
-          return super(input_stream, stream_mode: tabular_input_mode, **args, &block) unless tabular_input.header?
-
-          # If the header is already set then it is not expected in the file
-          if tabular_input_header.present?
-            tabular_input_cleanse_header
-            return super(input_stream, stream_mode: tabular_input_mode, **args, &block)
-          end
-
-          case tabular_input_mode
-          when :line
-            parse_header = lambda do |line|
-              tabular_input.parse_header(line)
-              tabular_input_cleanse_header
-              self.tabular_input_header = tabular_input.header.columns
-            end
-            super(input_stream, on_first: parse_header, stream_mode: :line, **args, &block)
-          when :array, :row
-            set_header = lambda do |row|
-              tabular_input.header.columns = row
-              tabular_input_cleanse_header
-              self.tabular_input_header = tabular_input.header.columns
-            end
-            super(input_stream, on_first: set_header, stream_mode: :array, **args, &block)
-          when :hash, :record
-            super(input_stream, stream_mode: :hash, **args, &block)
-          else
-            raise(ArgumentError, "Invalid tabular_input_mode: #{stream_mode.inspect}")
-          end
-        end
-
-        private
-
-        # Shared instance used for this slice, by a single worker (thread)
-        def tabular_input
-          @tabular_input ||= IOStreams::Tabular.new(
-            columns:          tabular_input_header,
-            allowed_columns:  tabular_input_white_list,
-            required_columns: tabular_input_required,
-            skip_unknown:     tabular_input_skip_unknown,
-            format:           tabular_input_format,
-            format_options:   tabular_input_options&.deep_symbolize_keys
-          )
-        end
-
-        def tabular_input_render
-          return if tabular_input_header.blank? && tabular_input.header?
-
-          @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
-        end
-
-        # Cleanse custom input header if supplied.
-        def tabular_input_cleanse_header
-          ignored_columns = tabular_input.header.cleanse!
-          logger.warn("Stripped out invalid columns from custom header", ignored_columns) unless ignored_columns.empty?
-
-          self.tabular_input_header = tabular_input.header.columns
-        end
-
-        def tabular_input_header_present
-          if tabular_input_header.present? || !tabular_input.header? || (tabular_input_mode == :hash || tabular_input_mode == :record)
-            return
-          end
-
-          errors.add(:tabular_input_header, "is required when tabular_input_format is #{tabular_input_format.inspect}")
-        end
-      end
-    end
-  end
-end