rocketjob 6.0.0.rc2 → 6.0.0.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6568c1307b7d42a0968df0335e3177967a8223838eceecbe0b3c0cab72c398af
-  data.tar.gz: 1109611ffce2fe4aed881f5bd9d017681546f06d62330ffc21873adbef12c179
+  metadata.gz: 6a04a33b0cd03bdf0a7cb948fc87dd6c7d7bb3b392e566a8c15df50b73e27459
+  data.tar.gz: fc62e740a0a92bae8daf1f4ffbe199af1debcb84f8859aed10ea5954dc44c7b6
 SHA512:
-  metadata.gz: 073adf2196d6d0cfd5c06ad8776374a1a14618b5fbd49a550e9d8df587b0a17fd95b4dad34a60d8812a2c5f4046974e22bcf616512d77442ea28145d9bd374d2
-  data.tar.gz: ee9b6d35149f7d7799071485f2e4032d47303d3452a4b7c70c5422019a66615938dba90e1e27090a4f33b635df6f565871efd3152f5a2f2a5cb03a61168b3755
+  metadata.gz: 74cac01d253cf21a856e1ca4a5cf63d5e90320303bdf310cf90325c9cca242c4ed1b7a0a1c43ca00764f2f40d29822df6e6bee499c1bff56c9ddaa2401bc3862
+  data.tar.gz: 1bbc47c7d869ef28fd578a7b2575f62957aa2f83f9fc927af1d6fba7866270b15fd21cef30007b78d58847137357c75ccae2d03545560d2ffe0d674fe34c1d0e

data/README.md

@@ -17,21 +17,151 @@ Checkout https://rocketjob.io/
 * Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
 * [Report bugs](https://github.com/rocketjob/rocketjob/issues)
 
-## Rocket Job v5
+## Rocket Job v6
 
 - Support for Ruby v3 and Rails 6.
-- Multiple output file support through extended `output_categories` capability.
-    - File output formats for each category. For example: CSV, PSV, JSON, etc.
-- Support for AWS DocumentDB as the data store.
+- Major enhancements in Batch job support:
+    - Direct built-in Tabular support for all input and output categories.
+    - Multiple output file support, each with its own settings for:
+        - Compression
+            - GZip, Zip, BZip2 (Chunked for much faster loading into Apache Spark).
+        - Encryption
+            - PGP, Symmetric Encryption.
+        - File format
+            - CSV, PSV, JSON, Fixed Format, xlsx.
+- Significant error handling improvements, especially around throttle failures
+  that used to result in "hanging" jobs.
+- Support AWS DocumentDB in addition to MongoDB as the data store.
 - Removed use of Symbols to meet Symbol deprecation in MongoDB and Mongoid.
 
-The following plugins have been deprecated and will be removed in Rocket Job v5.1
-- RocketJob::Batch::Tabular::Input
-- RocketJob::Batch::Tabular::Output
+### Upgrading to Rocket Job v6
+
+The following plugins have been deprecated and are no longer loaded by default.
+- `RocketJob::Batch::Tabular::Input`
+- `RocketJob::Batch::Tabular::Output`
+
+If your code relies on these plugins and you still want to upgrade to Rocket Job v6,
+add the following require statement to any jobs that still use them:
+
+~~~ruby
+require "rocket_job/batch/tabular"
+~~~
+
+It is important to migrate away from these plugins, since they will be removed in a future release.
+
+#### Upgrading Batch Jobs to Rocket Job v6
+
+Rocket Job v6 replaces the array of symbol type for `input_categories` and `output_categories`
+with an array of `RocketJob::Category::Input` and `RocketJob::Category::Output`.
+
+Jobs that added or modified the input or output categories need to be upgraded. For example:
+~~~ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+  
+  self.output_categories = [:main, :errors, :ignored]
+end
+~~~
+
+Needs to be changed to:
+~~~ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+  
+  output_category name: :main
+  output_category name: :errors
+  output_category name: :ignored
+end
+~~~
+
+##### slice_size, encrypt, compress
+
+These fields have been removed from the job itself:
+~~~ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+  
+  self.slice_sice = 1_000
+  self.encrypt    = true
+  self.compress   = true
+end
+~~~
+
+They are now specified on the `input_category` as follows:
+- `slice_size` just moves under `input_category`. 
+- `encrypt` becomes an option to `serializer`.
+- `compress` is now the default for all batch jobs so is not needed.
+
+If the serializer is set to `encrypt` then it is automatically compressed.
+
+~~~ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+  
+  input_category slice_sice: 1_000, serializer: :encrypt
+end
+~~~
+
+##### collect_output, collect_nil_output
+
+The following fields have been moved from the job itself:
+~~~ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+  
+  self.collect_output     = true
+  self.collect_nil_output = true
+end
+~~~
+
+Into the corresponding `output_category`:
+- `collect_output` no longer has any meaning. Output is collected anytime an `output_category` is defined.
+- `collect_nil_output` is now the option `nils` on the `output_category. 
+  It defaults to `false` so that by default any `nil` output from the `perform` method is not collected. 
+~~~ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+
+  output_category nils: true
+end
+~~~
+
+##### name
+
+For both `input_category` and `output_category`, when the `name` argument is not supplied
+it defaults to `:main`.
+
+For Example:
+~~~ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+
+  input_category name: :main, serializer: :encrypt
+  output_category name: :main
+end
+~~~
+
+Is the same as:
+~~~ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+
+  input_category serializer: :encrypt
+  output_category
+end
+~~~
+
+##### Existing and inflight jobs
+
+When migrating to Rocket Job 6, it is recommended to load every job and then save it back again as part of the 
+deployment. When the job loads it will automatically convert itself from the old schema to the new v6 schema.
+
+In flight jobs should not be affected, other than it is important to shutdown all running batch 
+servers _before_ running any new instances.
 
 ## Rocket Job v4
 
-Rocket Job Pro is now open source and included in Rocket Job. 
+Rocket Job Pro is now fully open source and included in Rocket Job under the Apache License. 
 
 The `RocketJob::Batch` plugin now adds batch processing capabilities to break up a single task into many
 concurrent workers processing slices of the entire job at the same time. 

data/lib/rocket_job/batch.rb

@@ -30,6 +30,5 @@ module RocketJob
     autoload :ThrottleWindows, "rocket_job/batch/throttle_windows"
     autoload :Result, "rocket_job/batch/result"
     autoload :Results, "rocket_job/batch/results"
-    autoload :Tabular, "rocket_job/batch/tabular"
   end
 end

data/lib/rocket_job/jobs/conversion_job.rb

@@ -0,0 +1,39 @@
+# Convert to and from CSV, JSON, xlsx, and PSV files.
+#
+# Example, Convert CSV file to JSON.
+#   job = RocketJob::ConversionJob.new
+#   job.upload("data.csv")
+#   job.output_category.file_name = "data.json"
+#   job.save!
+#
+# Example, Convert JSON file to PSV and compress it with GZip.
+#   job = RocketJob::ConversionJob.new
+#   job.upload("data.json")
+#   job.output_category.file_name = "data.psv.gz"
+#   job.save!
+#
+# Example, Read a CSV file that has been zipped from a remote website and the convert it to a GZipped json file.
+#   job = RocketJob::ConversionJob.new
+#   job.upload("https://example.org/file.zip")
+#   job.output_category.file_name = "data.json.gz"
+#   job.save!
+#
+module RocketJob
+  class ConversionJob < RocketJob::Job
+    include RocketJob::Batch
+
+    self.destroy_on_complete = false
+
+    # Detects file extension for its type
+    input_category format: :auto
+    output_category format: :auto
+
+    # When the job completes it will write the result to the output_category.file_name
+    after_batch :download
+
+    def perform(hash)
+      # For this job return the input hash record as-is. Could be transformed here as needed.
+      hash
+    end
+  end
+end

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -96,12 +96,14 @@ module RocketJob
       before_batch :run_before_code
       after_batch :run_after_code
 
-      # Make this job collect its output
-      # :nils [true|false]
-      #   Whether to skip the output from `code` when it is nil
-      #   Default: false
-      def collect_output(nils: false)
-        self.output_categories = [RocketJob::Category::Output.new(nils: nils)]
+      # Shortcut for setting the slice_size
+      def slice_size=(slice_size)
+        input_category.slice_size = slice_size
+      end
+
+      # Add a new output category and collect output for it.
+      def add_output_category(**args)
+        self.output_categories << RocketJob::Category::Output.new(**args)
       end
 
       private

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "6.0.0.rc2".freeze
+  VERSION = "6.0.0.rc3".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 6.0.0.rc2
+  version: 6.0.0.rc3
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-16 00:00:00.000000000 Z
+date: 2021-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -160,6 +160,7 @@ files:
 - lib/rocket_job/job.rb
 - lib/rocket_job/job_exception.rb
 - lib/rocket_job/jobs/active_job.rb
+- lib/rocket_job/jobs/conversion_job.rb
 - lib/rocket_job/jobs/copy_file_job.rb
 - lib/rocket_job/jobs/dirmon_job.rb
 - lib/rocket_job/jobs/housekeeping_job.rb