rocketjob 5.4.0.beta1 → 6.0.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55ab6ca2b3f76cdb4ddf679a2d3e88e1d6a6f3106a69349129a267b629ef4a53
-  data.tar.gz: 4e0d07878fb4265179b4a270650cc9b89ca4bca55f1d8f9a3451cb3064062c35
+  metadata.gz: 6568c1307b7d42a0968df0335e3177967a8223838eceecbe0b3c0cab72c398af
+  data.tar.gz: 1109611ffce2fe4aed881f5bd9d017681546f06d62330ffc21873adbef12c179
 SHA512:
-  metadata.gz: fdc7ae3827d987404f431410cb81161fbfa269dfe7575a4a86a6abf362c59c6acd0d2c7e9856273ced396eeaf20e50561f1be10a7de0d5c4ae45e5648d15d083
-  data.tar.gz: 7bb9e9ac90569e78e135293efcd357a0d75037b5417f5f530f18ae3f17f44acf28d7de5b4aeef880a0315117f2992623958a4fb4c9df93ca7273369ac052e759
+  metadata.gz: 073adf2196d6d0cfd5c06ad8776374a1a14618b5fbd49a550e9d8df587b0a17fd95b4dad34a60d8812a2c5f4046974e22bcf616512d77442ea28145d9bd374d2
+  data.tar.gz: ee9b6d35149f7d7799071485f2e4032d47303d3452a4b7c70c5422019a66615938dba90e1e27090a4f33b635df6f565871efd3152f5a2f2a5cb03a61168b3755

data/README.md

@@ -1,5 +1,5 @@
 # Rocket Job
-[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Build Status](https://travis-ci.org/rocketjob/rocketjob.svg?branch=master)](https://travis-ci.org/rocketjob/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/semantic_logger) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Support](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
+[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Support](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
 
 Ruby's missing batch system
 
@@ -17,11 +17,23 @@ Checkout https://rocketjob.io/
 * Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
 * [Report bugs](https://github.com/rocketjob/rocketjob/issues)
 
-## Rocket Job 4
+## Rocket Job v5
 
-Rocket Job Pro is now open sourced and included within Rocket Job. 
+- Support for Ruby v3 and Rails 6.
+- Multiple output file support through extended `output_categories` capability.
+    - File output formats for each category. For example: CSV, PSV, JSON, etc.
+- Support for AWS DocumentDB as the data store.
+- Removed use of Symbols to meet Symbol deprecation in MongoDB and Mongoid.
 
-The `RocketJob::Batch` plugin now adds batch processing capabilites to break up a single task into many
+The following plugins have been deprecated and will be removed in Rocket Job v5.1
+- RocketJob::Batch::Tabular::Input
+- RocketJob::Batch::Tabular::Output
+
+## Rocket Job v4
+
+Rocket Job Pro is now open source and included in Rocket Job. 
+
+The `RocketJob::Batch` plugin now adds batch processing capabilities to break up a single task into many
 concurrent workers processing slices of the entire job at the same time. 
 
 
@@ -33,7 +45,9 @@ class MyJob < RocketJob::Job
   
   self.description         = "Reverse names"
   self.destroy_on_complete = false
-  self.collect_output      = true
+  
+  # Collect the output for this job in the default output category: `:main`
+  output_category
 
   # Method to call by all available workers at the same time.
   # Reverse the characters for each line: 

data/bin/rocketjob_batch_perf

@@ -2,7 +2,7 @@
 require "rocketjob_batch"
 
 # Log to console
-SemanticLogger.add_appender(io: STDOUT, formatter: :color)
+SemanticLogger.add_appender(io: $stdout, formatter: :color)
 
 perf = RocketJob::Batch::Performance.new
 perf.parse(ARGV)

data/bin/rocketjob_perf

@@ -2,7 +2,7 @@
 require "rocketjob"
 
 # Log to console
-SemanticLogger.add_appender(io: STDOUT, formatter: :color)
+SemanticLogger.add_appender(io: $stdout, formatter: :color)
 
 perf = RocketJob::Performance.new
 perf.parse(ARGV)

data/lib/rocket_job/batch.rb

@@ -7,6 +7,8 @@ require "rocket_job/batch/state_machine"
 require "rocket_job/batch/throttle"
 require "rocket_job/batch/throttle_running_workers"
 require "rocket_job/batch/worker"
+# Ensure after_perform is run first and #upload override is after IO#upload is defined.
+require "rocket_job/batch/categories"
 
 module RocketJob
   module Batch
@@ -17,6 +19,7 @@ module RocketJob
     include Callbacks
     include Logger
     include Worker
+    include Categories
     include Throttle
     include ThrottleRunningWorkers
     include IO

data/lib/rocket_job/batch/categories.rb

@@ -0,0 +1,341 @@
+require "active_support/concern"
+
+module RocketJob
+  module Batch
+    module Categories
+      extend ActiveSupport::Concern
+
+      included do
+        after_initialize :rocketjob_categories_assign, if: :new_record?
+        after_initialize :rocketjob_categories_migrate, unless: :new_record?
+        before_perform :rocketjob_categories_input_render
+        after_perform :rocketjob_categories_output_render
+
+        # List of categories that this job can load input data into
+        embeds_many :input_categories, class_name: "RocketJob::Category::Input"
+
+        # List of categories that this job can save output data into
+        embeds_many :output_categories, class_name: "RocketJob::Category::Output"
+
+        # Internal attributes
+        class_attribute :defined_input_categories, instance_accessor: false, instance_predicate: false
+        class_attribute :defined_output_categories, instance_accessor: false, instance_predicate: false
+
+        # For RJMC to be able to edit jobs
+        accepts_nested_attributes_for :input_categories, :output_categories
+      end
+
+      module ClassMethods
+        # Define a new input category
+        # @see RocketJob::Category::Input
+        def input_category(**args)
+          category = RocketJob::Category::Input.new(**args)
+          if defined_input_categories.nil?
+            self.defined_input_categories = [category]
+          else
+            rocketjob_categories_set(category, defined_input_categories)
+          end
+        end
+
+        # Define a new output category
+        # @see RocketJob::Category::Output
+        def output_category(**args)
+          category = RocketJob::Category::Output.new(**args)
+          if defined_output_categories.nil?
+            self.defined_output_categories = [category]
+          else
+            rocketjob_categories_set(category, defined_output_categories)
+          end
+        end
+
+        # Builds this job instance from the supplied properties hash that may contain input and output categories.
+        # Keeps the defaults and merges in settings without replacing existing categories.
+        def from_properties(properties)
+          return super(properties) unless properties.key?("input_categories") || properties.key?("output_categories")
+
+          properties        = properties.dup
+          input_categories  = properties.delete("input_categories")
+          output_categories = properties.delete("output_categories")
+          job               = super(properties)
+          job.merge_input_categories(input_categories)
+          job.merge_output_categories(output_categories)
+          job
+        end
+
+        private
+
+        def rocketjob_categories_set(category, categories)
+          index = categories.find_index { |cat| cat.name == category.name }
+          index ? categories[index] = category : categories << category
+          category
+        end
+      end
+
+      def input_category(category_name = :main)
+        category_name = category_name.to_sym
+        category      = nil
+        # .find does not work against this association
+        input_categories.each { |catg| category = catg if catg.name == category_name }
+        unless category
+          # Auto-register main input category if missing
+          if category_name == :main
+            category              = Category::Input.new
+            self.input_categories = [category]
+          else
+            raise(ArgumentError,
+                  "Unknown Input Category: #{category_name.inspect}. Registered categories: #{input_categories.collect(&:name).join(',')}")
+          end
+        end
+        category
+      end
+
+      def output_category(category_name = :main)
+        category_name = category_name.to_sym
+        category      = nil
+        # .find does not work against this association
+        output_categories.each { |catg| category = catg if catg.name == category_name }
+        unless category
+          raise(ArgumentError,
+                "Unknown Output Category: #{category_name.inspect}. Registered categories: #{output_categories.collect(&:name).join(',')}")
+        end
+
+        category
+      end
+
+      # Returns [true|false] whether the named category has already been defined
+      def input_category?(category_name)
+        category_name = category_name.to_sym
+        # .find does not work against this association
+        input_categories.each { |catg| return true if catg.name == category_name }
+        false
+      end
+
+      def output_category?(category_name)
+        category_name = category_name.to_sym
+        # .find does not work against this association
+        output_categories.each { |catg| return true if catg.name == category_name }
+        false
+      end
+
+      def merge_input_categories(categories)
+        return if categories.blank?
+
+        categories.each do |properties|
+          category_name = (properties["name"] || properties[:name] || :main).to_sym
+          category      = input_category(category_name)
+          properties.each { |key, value| category.public_send("#{key}=".to_sym, value) }
+        end
+      end
+
+      def merge_output_categories(categories)
+        return if categories.blank?
+
+        categories.each do |properties|
+          category_name = (properties["name"] || properties[:name] || :main).to_sym
+          category      = output_category(category_name)
+          properties.each { |key, value| category.public_send("#{key}=".to_sym, value) }
+        end
+      end
+
+      private
+
+      def rocketjob_categories_assign
+        # Input categories defaults to :main if none was set in the class
+        if input_categories.empty?
+          self.input_categories =
+            if self.class.defined_input_categories
+              self.class.defined_input_categories.deep_dup
+            else
+              [RocketJob::Category::Input.new]
+            end
+        end
+
+        return if !self.class.defined_output_categories || !output_categories.empty?
+
+        # Input categories defaults to nil if none was set in the class
+        self.output_categories = self.class.defined_output_categories.deep_dup
+      end
+
+      # Render the output from the perform.
+      def rocketjob_categories_output_render
+        return if @rocket_job_output.nil?
+
+        # TODO: ..
+        return unless output_categories
+        return if output_categories.empty?
+
+        @rocket_job_output = rocketjob_categories_output_render_row(@rocket_job_output)
+      end
+
+      # Parse the input data before passing to the perform method
+      def rocketjob_categories_input_render
+        return if @rocket_job_input.nil?
+
+        @rocket_job_input = rocketjob_categories_input_render_row(@rocket_job_input)
+      end
+
+      def rocketjob_categories_input_render_row(row)
+        return if row.nil?
+
+        category = input_category
+        return row if category.nil? || !category.tabular?
+        return nil if row.blank?
+
+        tabular = category.tabular
+
+        # Return the row as-is if the required header has not yet been set.
+        if tabular.header?
+          raise(ArgumentError,
+                "The tabular header columns _must_ be set before attempting to parse data that requires it.")
+        end
+
+        tabular.record_parse(row)
+      end
+
+      def rocketjob_categories_output_render_row(row)
+        return if row.nil?
+
+        if row.is_a?(Batch::Result)
+          category  = output_category(row.category)
+          row.value = category.tabular.render(row.value) if category.tabular?
+          return row
+        end
+
+        if row.is_a?(Batch::Results)
+          results = Batch::Results.new
+          row.each { |result| results << rocketjob_categories_output_render_row(result) }
+          return results
+        end
+
+        category = output_category
+        return row unless category.tabular?
+        return nil if row.blank?
+
+        category.tabular.render(row)
+      end
+
+      # Migrate existing v4 batch jobs to v5.0
+      def rocketjob_categories_migrate
+        return unless attribute_present?(:input_categories) && self[:input_categories]&.first.is_a?(Symbol)
+
+        serializer = :none
+        if attribute_present?(:compress)
+          serializer = :compress if self[:compress]
+          remove_attribute(:compress)
+        end
+
+        if attribute_present?(:encrypt)
+          serializer = :encrypt if self[:encrypt]
+          remove_attribute(:encrypt)
+        end
+
+        slice_size = 100
+        if attribute_present?(:slice_size)
+          slice_size = self[:slice_size].to_i
+          remove_attribute(:slice_size)
+        end
+
+        main_input_format  = nil
+        main_input_mode    = :line
+        main_input_columns = nil
+        # Only migrate tabular attributes if the job also removed the tabular plugin.
+        unless respond_to?(:tabular_input_render)
+          if attribute_present?(:tabular_input_format)
+            main_input_format = self[:tabular_input_format]
+            remove_attribute(:tabular_input_format)
+          end
+
+          if attribute_present?(:tabular_input_mode)
+            main_input_mode = self[:tabular_input_mode]
+            remove_attribute(:tabular_input_mode)
+          end
+
+          if attribute_present?(:tabular_input_header)
+            main_input_columns = self[:tabular_input_header]
+            remove_attribute(:tabular_input_header)
+          end
+        end
+
+        file_name = nil
+        if attribute_present?(:upload_file_name)
+          file_name = self[:upload_file_name]
+          remove_attribute(:upload_file_name)
+        end
+
+        existing                = self[:input_categories]
+        self[:input_categories] = []
+        self[:input_categories] = existing.collect do |category_name|
+          RocketJob::Category::Input.new(
+            name:       category_name,
+            file_name:  file_name,
+            serializer: serializer,
+            slice_size: slice_size,
+            format:     [:main, "main"].include?(category_name) ? main_input_format : nil,
+            columns:    [:main, "main"].include?(category_name) ? main_input_columns : nil,
+            mode:       [:main, "main"].include?(category_name) ? main_input_mode : nil
+          ).as_document
+        end
+
+        collect_output = false
+        if attribute_present?(:collect_output)
+          collect_output = self[:collect_output]
+          remove_attribute(:collect_output)
+        end
+
+        collect_nil_output = true
+        if attribute_present?(:collect_nil_output)
+          collect_nil_output = self[:collect_nil_output]
+          remove_attribute(:collect_nil_output)
+        end
+
+        main_output_format  = nil
+        main_output_columns = nil
+        main_output_options = nil
+
+        # Only migrate tabular attributes if the job also removed the tabular plugin.
+        unless respond_to?(:tabular_output_render)
+          if attribute_present?(:tabular_output_format)
+            main_output_format = self[:tabular_output_format]
+            remove_attribute(:tabular_output_format)
+          end
+
+          if attribute_present?(:tabular_output_header)
+            main_output_columns = self[:tabular_output_header]
+            remove_attribute(:tabular_output_header)
+          end
+
+          if attribute_present?(:tabular_output_options)
+            main_output_options = self[:tabular_output_options]
+            remove_attribute(:tabular_output_options)
+          end
+        end
+
+        existing                 = self[:output_categories]
+        self[:output_categories] = []
+        if collect_output
+          if existing.blank?
+            self[:output_categories] = [
+              RocketJob::Category::Output.new(
+                nils:           collect_nil_output,
+                format:         main_output_format,
+                columns:        main_output_columns,
+                format_options: main_output_options
+              ).as_document
+            ]
+          elsif existing.first.is_a?(Symbol)
+            self[:output_categories] = existing.collect do |category_name|
+              RocketJob::Category::Output.new(
+                name:           category_name,
+                serializer:     serializer,
+                nils:           collect_nil_output,
+                format:         [:main, "main"].include?(category_name) ? main_output_format : nil,
+                columns:        [:main, "main"].include?(category_name) ? main_output_columns : nil,
+                format_options: [:main, "main"].include?(category_name) ? main_output_options : nil
+              ).as_document
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rocket_job/batch/io.rb

@@ -9,32 +9,68 @@ module RocketJob
       # Returns [RocketJob::Sliced::Input] input collection for holding input slices
       #
       # Parameters:
-      #   category [Symbol]
-      #     The name of the category to access or upload data into
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to access or upload data into
       #     Default: None ( Uses the single default input collection for this job )
       #     Validates: This value must be one of those listed in #input_categories
       def input(category = :main)
-        unless input_categories.include?(category) || (category == :main)
-          raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}"
-        end
+        raise(ArgumentError, "Cannot supply Output Category to input category") if category.is_a?(Category::Output)
+
+        category = input_category(category) unless category.is_a?(Category::Input)
 
-        (@inputs ||= {})[category] ||= RocketJob::Sliced.factory(:input, category, self)
+        (@inputs ||= {})[category.name] ||= RocketJob::Sliced.factory(:input, category, self)
       end
 
       # Returns [RocketJob::Sliced::Output] output collection for holding output slices
       # Returns nil if no output is being collected
       #
       # Parameters:
-      #   category [Symbol]
-      #     The name of the category to access or download data from
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to access or download data from
       #     Default: None ( Uses the single default output collection for this job )
       #     Validates: This value must be one of those listed in #output_categories
       def output(category = :main)
-        unless output_categories.include?(category) || (category == :main)
-          raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}"
-        end
+        raise(ArgumentError, "Cannot supply Input Category to output category") if category.is_a?(Category::Input)
+
+        category = output_category(category) unless category.is_a?(Category::Output)
+
+        (@outputs ||= {})[category.name] ||= RocketJob::Sliced.factory(:output, category, self)
+      end
+
+      # Rapidly upload individual records in batches.
+      #
+      # Operates directly on a Mongo Collection to avoid the overhead of creating Mongoid objects
+      # for each and every row.
+      #
+      #   input_category(:my_lookup).find(id: 123).first
+      #
+      # Lookup collection.
+      #
+      # Upload side / secondary lookup tables that can be accessed during job processing.
+      #
+      # Example:
+      #   lookup_collection(:my_lookup).upload do |io|
+      #     io << {id: 123, data: "first record"}
+      #     io << {id: 124, data: "second record"}
+      #   end
+      #
+      # Parameters:
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to access or download data from
+      #     Default: None ( Uses the single default output collection for this job )
+      #     Validates: This value must be one of those listed in #input_categories
+      def lookup_collection(category = :main)
+        category = input_category(category) unless category.is_a?(Category::Input)
 
-        (@outputs ||= {})[category] ||= RocketJob::Sliced.factory(:output, category, self)
+        collection = (@lookup_collections ||= {})[category.name]
+
+        unless collection
+          collection_name = "rocket_job.inputs.#{id}"
+          collection_name << ".#{category.name}" unless category.name == :main
+
+          @lookup_collections[category.name] ||=
+            LookupCollection.new(Sliced::Slice.collection.database, collection_name)
+        end
       end
 
       # Upload the supplied file, io, IOStreams::Path, or IOStreams::Stream.
@@ -65,6 +101,11 @@ module RocketJob
       #       Parses each line from the file into a Hash and uploads each hash for processing by workers.
       #     See IOStreams::Stream#each.
       #
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to access or download data from
+      #     Default: None ( Uses the single default output collection for this job )
+      #     Validates: This value must be one of those listed in #input_categories
+      #
       # Example:
       #   # Load plain text records from a file
       #   job.upload('hello.csv')
@@ -116,22 +157,46 @@ module RocketJob
       def upload(stream = nil, file_name: nil, category: :main, stream_mode: :line, on_first: nil, **args, &block)
         raise(ArgumentError, "Either stream, or a block must be supplied") unless stream || block
 
-        stream_mode = stream_mode.to_sym
-        # Backward compatibility with existing v4 jobs
-        stream_mode = :array if stream_mode == :row
-        stream_mode = :hash if stream_mode == :record
+        category = input_category(category) unless category.is_a?(Category::Input)
+        stream ||= category.file_name
+        path     = nil
 
-        count             =
+        if stream
+          path               = IOStreams.new(stream)
+          path.file_name     = file_name if file_name
+          category.file_name = path.file_name
+
+          # Auto detect the format based on the upload file name if present.
+          if category.format == :auto
+            format = path.format
+            if format
+              # Rebuild tabular with the above file name
+              category.reset_tabular
+              category.format = format
+            end
+          end
+        end
+
+        # Tabular transformations required for upload?
+        if category.tabular?
+          # Remove non-printable characters from tabular input formats
+          # Cannot change the length of fixed width lines
+          replace = category.format == :fixed ? " " : ""
+          path&.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
+
+          # Extract the header line during the file upload when needed.
+          on_first = rocket_job_upload_header_lambda(category, on_first) if category.tabular.header?
+        end
+
+        count =
           if block
             input(category).upload(on_first: on_first, &block)
           else
-            path                  = IOStreams.new(stream)
-            path.file_name        = file_name if file_name
-            self.upload_file_name = path.file_name
             input(category).upload(on_first: on_first) do |io|
               path.each(stream_mode, **args) { |line| io << line }
             end
           end
+
         self.record_count = (record_count || 0) + count
         count
       end
@@ -144,6 +209,9 @@ module RocketJob
       #     and uploaded into the job
       #     These columns are automatically added to the select list to reduce overhead
       #
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to upload to.
+      #
       # If a Block is supplied it is passed the model returned from the database and should
       # return the work item to be uploaded into the job.
       #
@@ -221,7 +289,7 @@ module RocketJob
       # for each slice processed
       #
       # Example
-      #   job.slice_size = 100
+      #   job.input_category.slice_size = 100
       #   job.upload_integer_range(200, 421)
       #
       #   # Equivalent to calling:
@@ -253,7 +321,7 @@ module RocketJob
       # in a database based on the id column
       #
       # Example
-      #   job.slice_size = 100
+      #   job.input_category.slice_size = 100
       #   job.upload_integer_range_in_reverse_order(200, 421)
       #
       #   # Equivalent to calling:
@@ -285,12 +353,12 @@ module RocketJob
       #     For example the following types are not supported: Date
       #
       # Note:
-      #   The caller should honor `:slice_size`, the entire slice is loaded as-is.
+      #   The caller should implement `:slice_size`, since the entire slice is saved as-is.
       #
       # Note:
       #   Not thread-safe. Only call from one thread at a time
-      def upload_slice(slice)
-        input.insert(slice)
+      def upload_slice(slice, category: :main)
+        input(category).insert(slice)
         count             = slice.size
         self.record_count = (record_count || 0) + count
         count
@@ -353,54 +421,54 @@ module RocketJob
       def download(stream = nil, category: :main, header_line: nil, **args, &block)
         raise "Cannot download incomplete job: #{id}. Currently in state: #{state}-#{sub_state}" if rocket_job_processing?
 
-        return output(category).download(header_line: header_line, &block) if block
+        category           = output_category(category) unless category.is_a?(Category::Output)
+        output_collection  = output(category)
 
-        output_collection = output(category)
+        # Store the output file name in the category
+        category.file_name = stream if !block && (stream.is_a?(String) || stream.is_a?(IOStreams::Path))
 
         if output_collection.binary?
-          IOStreams.new(stream).stream(:none).writer(**args) do |io|
-            raise(ArgumenError, "A `header_line` is not supported with binary output collections") if header_line
+          raise(ArgumentError, "A `header_line` is not supported with binary output collections") if header_line
+
+          return output_collection.download(&block) if block
 
+          IOStreams.new(stream || category.file_name).stream(:none).writer(**args) do |io|
             output_collection.download { |record| io << record[:binary] }
           end
         else
-          IOStreams.new(stream).writer(:line, **args) do |io|
+          header_line ||= category.render_header
+
+          return output_collection.download(header_line: header_line, &block) if block
+
+          raise(ArgumentError, "Missing mandatory `stream` or `category.file_name`") unless stream || category.file_name
+
+          IOStreams.new(stream || category.file_name).writer(:line, **args) do |io|
             output_collection.download(header_line: header_line) { |record| io << record }
           end
         end
       end
 
-      # Writes the supplied result, Batch::Result or Batch::Results to the relevant collections.
-      #
-      # If a block is supplied, the block is supplied with a writer that should be used to
-      # accumulate the results.
-      #
-      # Examples
-      #
-      # job.write_output('hello world')
-      #
-      # job.write_output do |writer|
-      #   writer << 'hello world'
-      # end
-      #
-      # job.write_output do |writer|
-      #   result = RocketJob::Batch::Results
-      #   result << RocketJob::Batch::Result.new(:main, 'hello world')
-      #   result << RocketJob::Batch::Result.new(:errors, 'errors')
-      #   writer << result
-      # end
-      #
-      # result = RocketJob::Batch::Results
-      # result << RocketJob::Batch::Result.new(:main, 'hello world')
-      # result << RocketJob::Batch::Result.new(:errors, 'errors')
-      # job.write_output(result)
-      def write_output(result = nil, input_slice = nil, &block)
-        if block
-          RocketJob::Sliced::Writer::Output.collect(self, input_slice, &block)
-        else
-          raise(ArgumentError, "result parameter is required when no block is supplied") unless result
+      private
 
-          RocketJob::Sliced::Writer::Output.collect(self, input_slice) { |writer| writer << result }
+      # Return a lambda to extract the header row from the uploaded file.
+      def rocket_job_upload_header_lambda(category, on_first)
+        case category.mode
+        when :line
+          lambda do |line|
+            category.tabular.parse_header(line)
+            category.cleanse_header!
+            category.columns = category.tabular.header.columns
+            # Call chained on_first if present
+            on_first&.call(line)
+          end
+        when :array
+          lambda do |row|
+            category.tabular.header.columns = row
+            category.cleanse_header!
+            category.columns = category.tabular.header.columns
+            # Call chained on_first if present
+            on_first&.call(line)
+          end
         end
       end
     end