rocketjob 3.5.2 → 4.0.0

data/lib/rocket_job/sliced/writer/output.rb

@@ -0,0 +1,82 @@
+module RocketJob
+  module Sliced
+    module Writer
+      # Internal class for writing categorized results into output slices
+      class Output
+        attr_reader :job, :categorized_records
+        attr_accessor :input_slice
+
+        # Collect output results and write to output collections
+        # iff job is collecting output
+        # Notes:
+        #   Nothing is saved if an exception is raised inside the block
+        def self.collect(job, input_slice = nil)
+          if job.collect_output?
+            writer = new(job, input_slice)
+            yield(writer)
+            writer.close
+          else
+            writer = NullWriter.new(job, input_slice)
+            yield(writer)
+          end
+        end
+
+        def initialize(job, input_slice = nil)
+          @job                 = job
+          @input_slice         = input_slice
+          @categorized_records = {}
+        end
+
+        # Writes the supplied result, RocketJob::Batch::Result or RocketJob::Batch::Results
+        # to the relevant collections
+        def <<(result)
+          if result.is_a?(RocketJob::Batch::Results)
+            result.each { |single| extract_categorized_result(single) }
+          else
+            extract_categorized_result(result)
+          end
+        end
+
+        # Write categorized results to their relevant collections
+        def close
+          categorized_records.each_pair do |category, results|
+            job.output(category).insert(results, input_slice)
+          end
+        end
+
+        private
+
+        # Stores the categorized result from one result
+        def extract_categorized_result(result)
+          category = :main
+          value    = result
+          if result.is_a?(RocketJob::Batch::Result)
+            category = result.category
+            value    = result.value
+            raise(ArgumentError, "Invalid RocketJob Output Category: #{category}") if job.output_categories.exclude?(category)
+          end
+          (categorized_records[category] ||= []) << value unless value.nil? && !job.collect_nil_output?
+        end
+      end
+
+      class NullWriter
+        attr_reader :job, :categorized_records
+        attr_accessor :input_slice
+
+        def initialize(job, input_slice = nil)
+          @job                 = job
+          @input_slice         = input_slice
+          @categorized_records = {}
+        end
+
+        def <<(_)
+          # noop
+        end
+
+        def close
+          # noop
+        end
+      end
+    end
+  end
+end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = '3.5.2'.freeze
+  VERSION = '4.0.0'.freeze
 end

data/lib/rocket_job/worker.rb

@@ -47,7 +47,7 @@ module RocketJob
       @name             = "#{server_name}:#{id}"
       @re_check_seconds = (re_check_seconds || 60).to_f
       @re_check_start   = Time.now
-      @filter           = filter || {}
+      @filter           = filter.nil? ? {} : filter.dup
       @current_filter   = @filter.dup
       @thread           = Thread.new { run } unless inline
     end
@@ -121,7 +121,7 @@ module RocketJob
       return unless (time - @re_check_start) > re_check_seconds
 
       @re_check_start     = time
-      self.current_filter = filter.dup
+      self.current_filter = filter.dup if current_filter != filter
     end
   end
 end

data/lib/rocketjob.rb

@@ -1,12 +1,25 @@
+require 'iostreams'
 require 'semantic_logger'
 require 'mongoid'
 require 'rocket_job/extensions/mongo/logging'
 require 'rocket_job/version'
 require 'rocket_job/rocket_job'
 
+# Apply patches to implement `with_collection`
+if Mongoid::VERSION.to_i >= 6
+  require 'rocket_job/extensions/mongoid/clients/options'
+  require 'rocket_job/extensions/mongoid/contextual/mongo'
+  require 'rocket_job/extensions/mongoid/factory'
+else
+  require 'rocket_job/extensions/mongoid_5/clients/options'
+  require 'rocket_job/extensions/mongoid_5/contextual/mongo'
+  require 'rocket_job/extensions/mongoid_5/factory'
+end
+
 # @formatter:off
 module RocketJob
   autoload :ActiveWorker,       'rocket_job/active_worker'
+  autoload :Batch,              'rocket_job/batch'
   autoload :CLI,                'rocket_job/cli'
   autoload :Config,             'rocket_job/config'
   autoload :DirmonEntry,        'rocket_job/dirmon_entry'
@@ -47,11 +60,26 @@ module RocketJob
   module Jobs
     autoload :ActiveJob,        'rocket_job/jobs/active_job'
     autoload :DirmonJob,        'rocket_job/jobs/dirmon_job'
+    autoload :OnDemandBatchJob, 'rocket_job/jobs/on_demand_batch_job'
     autoload :OnDemandJob,      'rocket_job/jobs/on_demand_job'
     autoload :HousekeepingJob,  'rocket_job/jobs/housekeeping_job'
+    autoload :PerformanceJob,   'rocket_job/jobs/performance_job'
     autoload :SimpleJob,        'rocket_job/jobs/simple_job'
     autoload :UploadFileJob,    'rocket_job/jobs/upload_file_job'
   end
+
+  module Sliced
+    autoload :Input,               'rocket_job/sliced/input'
+    autoload :Output,              'rocket_job/sliced/output'
+    autoload :Slice,               'rocket_job/sliced/slice'
+    autoload :Slices,              'rocket_job/sliced/slices'
+    autoload :Store,               'rocket_job/sliced/store'
+
+    module Writer
+      autoload :Input,             'rocket_job/sliced/writer/input'
+      autoload :Output,            'rocket_job/sliced/writer/output'
+    end
+  end
 end
 
 # Add Active Job adapter for Rails

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 3.5.2
+  version: 4.0.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-31 00:00:00.000000000 Z
+date: 2018-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -38,20 +38,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: iostreams
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.4'
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
@@ -78,13 +92,36 @@ files:
 - LICENSE.txt
 - README.md
 - bin/rocketjob
+- bin/rocketjob_batch_perf
 - bin/rocketjob_perf
 - lib/rocket_job/active_worker.rb
+- lib/rocket_job/batch.rb
+- lib/rocket_job/batch/callbacks.rb
+- lib/rocket_job/batch/io.rb
+- lib/rocket_job/batch/logger.rb
+- lib/rocket_job/batch/lower_priority.rb
+- lib/rocket_job/batch/model.rb
+- lib/rocket_job/batch/performance.rb
+- lib/rocket_job/batch/result.rb
+- lib/rocket_job/batch/results.rb
+- lib/rocket_job/batch/state_machine.rb
+- lib/rocket_job/batch/statistics.rb
+- lib/rocket_job/batch/tabular.rb
+- lib/rocket_job/batch/tabular/input.rb
+- lib/rocket_job/batch/tabular/output.rb
+- lib/rocket_job/batch/throttle.rb
+- lib/rocket_job/batch/throttle_running_slices.rb
+- lib/rocket_job/batch/worker.rb
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
 - lib/rocket_job/dirmon_entry.rb
 - lib/rocket_job/extensions/mongo/logging.rb
+- lib/rocket_job/extensions/mongoid/clients/options.rb
+- lib/rocket_job/extensions/mongoid/contextual/mongo.rb
 - lib/rocket_job/extensions/mongoid/factory.rb
+- lib/rocket_job/extensions/mongoid_5/clients/options.rb
+- lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb
+- lib/rocket_job/extensions/mongoid_5/factory.rb
 - lib/rocket_job/extensions/rocket_job_adapter.rb
 - lib/rocket_job/heartbeat.rb
 - lib/rocket_job/job.rb
@@ -92,7 +129,9 @@ files:
 - lib/rocket_job/jobs/active_job.rb
 - lib/rocket_job/jobs/dirmon_job.rb
 - lib/rocket_job/jobs/housekeeping_job.rb
+- lib/rocket_job/jobs/on_demand_batch_job.rb
 - lib/rocket_job/jobs/on_demand_job.rb
+- lib/rocket_job/jobs/performance_job.rb
 - lib/rocket_job/jobs/simple_job.rb
 - lib/rocket_job/jobs/upload_file_job.rb
 - lib/rocket_job/performance.rb
@@ -116,37 +155,15 @@ files:
 - lib/rocket_job/plugins/transaction.rb
 - lib/rocket_job/rocket_job.rb
 - lib/rocket_job/server.rb
+- lib/rocket_job/sliced/input.rb
+- lib/rocket_job/sliced/output.rb
+- lib/rocket_job/sliced/slice.rb
+- lib/rocket_job/sliced/slices.rb
+- lib/rocket_job/sliced/writer/input.rb
+- lib/rocket_job/sliced/writer/output.rb
 - lib/rocket_job/version.rb
 - lib/rocket_job/worker.rb
 - lib/rocketjob.rb
-- test/config/database.yml
-- test/config/mongoid.yml
-- test/config_test.rb
-- test/dirmon_entry_test.rb
-- test/dirmon_job_test.rb
-- test/files/text.txt
-- test/job_test.rb
-- test/jobs/housekeeping_job_test.rb
-- test/jobs/on_demand_job_test.rb
-- test/jobs/upload_file_job_test.rb
-- test/plugins/cron_test.rb
-- test/plugins/job/callbacks_test.rb
-- test/plugins/job/defaults_test.rb
-- test/plugins/job/logger_test.rb
-- test/plugins/job/model_test.rb
-- test/plugins/job/persistence_test.rb
-- test/plugins/job/state_machine_test.rb
-- test/plugins/job/throttle_test.rb
-- test/plugins/job/worker_test.rb
-- test/plugins/processing_window_test.rb
-- test/plugins/restart_test.rb
-- test/plugins/retry_test.rb
-- test/plugins/singleton_test.rb
-- test/plugins/state_machine_event_callbacks_test.rb
-- test/plugins/state_machine_test.rb
-- test/plugins/transaction_test.rb
-- test/test_db.sqlite3
-- test/test_helper.rb
 homepage: http://rocketjob.io
 licenses:
 - Apache-2.0
@@ -167,36 +184,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Ruby's missing batch system.
-test_files:
-- test/config_test.rb
-- test/config/mongoid.yml
-- test/config/database.yml
-- test/test_db.sqlite3
-- test/plugins/state_machine_test.rb
-- test/plugins/restart_test.rb
-- test/plugins/state_machine_event_callbacks_test.rb
-- test/plugins/transaction_test.rb
-- test/plugins/cron_test.rb
-- test/plugins/job/logger_test.rb
-- test/plugins/job/state_machine_test.rb
-- test/plugins/job/worker_test.rb
-- test/plugins/job/throttle_test.rb
-- test/plugins/job/defaults_test.rb
-- test/plugins/job/persistence_test.rb
-- test/plugins/job/callbacks_test.rb
-- test/plugins/job/model_test.rb
-- test/plugins/singleton_test.rb
-- test/plugins/processing_window_test.rb
-- test/plugins/retry_test.rb
-- test/job_test.rb
-- test/dirmon_job_test.rb
-- test/dirmon_entry_test.rb
-- test/files/text.txt
-- test/test_helper.rb
-- test/jobs/housekeeping_job_test.rb
-- test/jobs/on_demand_job_test.rb
-- test/jobs/upload_file_job_test.rb
+test_files: []

data/test/config/database.yml

@@ -1,5 +0,0 @@
-test:
-  adapter:  sqlite3
-  database: test/test_db.sqlite3
-  pool:     5
-  timeout:  5000

data/test/config/mongoid.yml

@@ -1,88 +0,0 @@
-# See: https://docs.mongodb.com/ruby-driver/master/tutorials/5.1.0/mongoid-installation/
-client_options: &client_options
-  read:
-    mode:             :primary
-  write:
-    w:                0
-  max_pool_size:      50
-  min_pool_size:      10
-  connect_timeout:    5
-  socket_timeout:     300
-  wait_queue_timeout: 5
-
-mongoid_options: &mongoid_options
-  # Includes the root model name in json serialization. (default: false)
-  # include_root_in_json: false
-
-  # Include the _type field in serialization. (default: false)
-  # include_type_for_serialization: false
-
-  # Preload all models in development, needed when models use
-  # inheritance. (default: false)
-  preload_models: true
-
-  # Raise an error when performing a #find and the document is not found.
-  # (default: true)
-  # raise_not_found_error: true
-
-  # Raise an error when defining a scope with the same name as an
-  # existing method. (default: false)
-  scope_overwrite_exception: true
-
-  # Use Active Support's time zone in conversions. (default: true)
-  # use_activesupport_time_zone: true
-
-  # Ensure all times are UTC in the app side. (default: false)
-  use_utc: true
-
-development:
-  clients:
-    default: &default_development
-      uri: mongodb://127.0.0.1:27017/rocketjob_development
-      options:
-        <<: *client_options
-        write:
-          w:   0
-        max_pool_size: 5
-        min_pool_size: 1
-    rocketjob:
-      <<: *default_development
-    rocketjob_slices:
-      <<: *default_development
-  options:
-    <<: *mongoid_options
-
-test:
-  clients:
-    default: &default_test
-      uri: mongodb://127.0.0.1:27017/rocketjob_test
-      options:
-        <<: *client_options
-        write:
-          w:           1
-        max_pool_size: 5
-        min_pool_size: 1
-    rocketjob:
-      <<: *default_test
-    rocketjob_slices:
-      <<: *default_test
-  options:
-    <<: *mongoid_options
-
-production:
-  clients:
-    default: &default_production
-      uri: mongodb://user:secret@server.example.org:27017,server2.example.org:27017/rocketjob_production
-      options:
-        <<: *client_options
-        write:
-          w:   0
-    rocketjob:
-      <<: *default_production
-    rocketjob_slices:
-      <<: *default_production
-      # Optionally Specify a different database or even server to store slices on
-      # uri: mongodb://user:secret@server3.example.org:27017/slices_production
-  options:
-    <<: *mongoid_options
-

data/test/config_test.rb

@@ -1,10 +0,0 @@
-require_relative 'test_helper'
-class ConfigTest < Minitest::Test
-  describe RocketJob::Config do
-    describe '.config' do
-      it 'support multiple databases' do
-        assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
-      end
-    end
-  end
-end