rocketjob 5.4.0.beta2 → 6.0.0.rc3

data/lib/rocket_job/batch/io.rb

@@ -9,32 +9,68 @@ module RocketJob
       # Returns [RocketJob::Sliced::Input] input collection for holding input slices
       #
       # Parameters:
-      #   category [Symbol]
-      #     The name of the category to access or upload data into
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to access or upload data into
       #     Default: None ( Uses the single default input collection for this job )
       #     Validates: This value must be one of those listed in #input_categories
       def input(category = :main)
-        unless input_categories.include?(category) || (category == :main)
-          raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}"
-        end
+        raise(ArgumentError, "Cannot supply Output Category to input category") if category.is_a?(Category::Output)
+
+        category = input_category(category) unless category.is_a?(Category::Input)
 
-        (@inputs ||= {})[category] ||= RocketJob::Sliced.factory(:input, category, self)
+        (@inputs ||= {})[category.name] ||= RocketJob::Sliced.factory(:input, category, self)
       end
 
       # Returns [RocketJob::Sliced::Output] output collection for holding output slices
       # Returns nil if no output is being collected
       #
       # Parameters:
-      #   category [Symbol]
-      #     The name of the category to access or download data from
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to access or download data from
       #     Default: None ( Uses the single default output collection for this job )
       #     Validates: This value must be one of those listed in #output_categories
       def output(category = :main)
-        unless output_categories.include?(category) || (category == :main)
-          raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}"
-        end
+        raise(ArgumentError, "Cannot supply Input Category to output category") if category.is_a?(Category::Input)
+
+        category = output_category(category) unless category.is_a?(Category::Output)
+
+        (@outputs ||= {})[category.name] ||= RocketJob::Sliced.factory(:output, category, self)
+      end
+
+      # Rapidly upload individual records in batches.
+      #
+      # Operates directly on a Mongo Collection to avoid the overhead of creating Mongoid objects
+      # for each and every row.
+      #
+      #   input_category(:my_lookup).find(id: 123).first
+      #
+      # Lookup collection.
+      #
+      # Upload side / secondary lookup tables that can be accessed during job processing.
+      #
+      # Example:
+      #   lookup_collection(:my_lookup).upload do |io|
+      #     io << {id: 123, data: "first record"}
+      #     io << {id: 124, data: "second record"}
+      #   end
+      #
+      # Parameters:
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to access or download data from
+      #     Default: None ( Uses the single default output collection for this job )
+      #     Validates: This value must be one of those listed in #input_categories
+      def lookup_collection(category = :main)
+        category = input_category(category) unless category.is_a?(Category::Input)
 
-        (@outputs ||= {})[category] ||= RocketJob::Sliced.factory(:output, category, self)
+        collection = (@lookup_collections ||= {})[category.name]
+
+        unless collection
+          collection_name = "rocket_job.inputs.#{id}"
+          collection_name << ".#{category.name}" unless category.name == :main
+
+          @lookup_collections[category.name] ||=
+            LookupCollection.new(Sliced::Slice.collection.database, collection_name)
+        end
       end
 
       # Upload the supplied file, io, IOStreams::Path, or IOStreams::Stream.
@@ -65,6 +101,11 @@ module RocketJob
       #       Parses each line from the file into a Hash and uploads each hash for processing by workers.
       #     See IOStreams::Stream#each.
       #
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to access or download data from
+      #     Default: None ( Uses the single default output collection for this job )
+      #     Validates: This value must be one of those listed in #input_categories
+      #
       # Example:
       #   # Load plain text records from a file
       #   job.upload('hello.csv')
@@ -116,22 +157,46 @@ module RocketJob
       def upload(stream = nil, file_name: nil, category: :main, stream_mode: :line, on_first: nil, **args, &block)
         raise(ArgumentError, "Either stream, or a block must be supplied") unless stream || block
 
-        stream_mode = stream_mode.to_sym
-        # Backward compatibility with existing v4 jobs
-        stream_mode = :array if stream_mode == :row
-        stream_mode = :hash if stream_mode == :record
+        category = input_category(category) unless category.is_a?(Category::Input)
+        stream ||= category.file_name
+        path     = nil
 
-        count             =
+        if stream
+          path               = IOStreams.new(stream)
+          path.file_name     = file_name if file_name
+          category.file_name = path.file_name
+
+          # Auto detect the format based on the upload file name if present.
+          if category.format == :auto
+            format = path.format
+            if format
+              # Rebuild tabular with the above file name
+              category.reset_tabular
+              category.format = format
+            end
+          end
+        end
+
+        # Tabular transformations required for upload?
+        if category.tabular?
+          # Remove non-printable characters from tabular input formats
+          # Cannot change the length of fixed width lines
+          replace = category.format == :fixed ? " " : ""
+          path&.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
+
+          # Extract the header line during the file upload when needed.
+          on_first = rocket_job_upload_header_lambda(category, on_first) if category.tabular.header?
+        end
+
+        count =
           if block
             input(category).upload(on_first: on_first, &block)
           else
-            path                  = IOStreams.new(stream)
-            path.file_name        = file_name if file_name
-            self.upload_file_name = path.file_name
             input(category).upload(on_first: on_first) do |io|
               path.each(stream_mode, **args) { |line| io << line }
             end
           end
+
         self.record_count = (record_count || 0) + count
         count
       end
@@ -144,6 +209,9 @@ module RocketJob
       #     and uploaded into the job
       #     These columns are automatically added to the select list to reduce overhead
       #
+      #   category [Symbol|RocketJob::Category::Input]
+      #     The category or the name of the category to upload to.
+      #
       # If a Block is supplied it is passed the model returned from the database and should
       # return the work item to be uploaded into the job.
       #
@@ -221,7 +289,7 @@ module RocketJob
       # for each slice processed
       #
       # Example
-      #   job.slice_size = 100
+      #   job.input_category.slice_size = 100
       #   job.upload_integer_range(200, 421)
       #
       #   # Equivalent to calling:
@@ -253,7 +321,7 @@ module RocketJob
       # in a database based on the id column
       #
       # Example
-      #   job.slice_size = 100
+      #   job.input_category.slice_size = 100
       #   job.upload_integer_range_in_reverse_order(200, 421)
       #
       #   # Equivalent to calling:
@@ -285,12 +353,12 @@ module RocketJob
       #     For example the following types are not supported: Date
       #
       # Note:
-      #   The caller should honor `:slice_size`, the entire slice is loaded as-is.
+      #   The caller should implement `:slice_size`, since the entire slice is saved as-is.
       #
       # Note:
       #   Not thread-safe. Only call from one thread at a time
-      def upload_slice(slice)
-        input.insert(slice)
+      def upload_slice(slice, category: :main)
+        input(category).insert(slice)
         count             = slice.size
         self.record_count = (record_count || 0) + count
         count
@@ -353,54 +421,54 @@ module RocketJob
       def download(stream = nil, category: :main, header_line: nil, **args, &block)
         raise "Cannot download incomplete job: #{id}. Currently in state: #{state}-#{sub_state}" if rocket_job_processing?
 
-        return output(category).download(header_line: header_line, &block) if block
+        category           = output_category(category) unless category.is_a?(Category::Output)
+        output_collection  = output(category)
 
-        output_collection = output(category)
+        # Store the output file name in the category
+        category.file_name = stream if !block && (stream.is_a?(String) || stream.is_a?(IOStreams::Path))
 
         if output_collection.binary?
-          IOStreams.new(stream).stream(:none).writer(**args) do |io|
-            raise(ArgumenError, "A `header_line` is not supported with binary output collections") if header_line
+          raise(ArgumentError, "A `header_line` is not supported with binary output collections") if header_line
+
+          return output_collection.download(&block) if block
 
+          IOStreams.new(stream || category.file_name).stream(:none).writer(**args) do |io|
             output_collection.download { |record| io << record[:binary] }
           end
         else
-          IOStreams.new(stream).writer(:line, **args) do |io|
+          header_line ||= category.render_header
+
+          return output_collection.download(header_line: header_line, &block) if block
+
+          raise(ArgumentError, "Missing mandatory `stream` or `category.file_name`") unless stream || category.file_name
+
+          IOStreams.new(stream || category.file_name).writer(:line, **args) do |io|
             output_collection.download(header_line: header_line) { |record| io << record }
           end
         end
       end
 
-      # Writes the supplied result, Batch::Result or Batch::Results to the relevant collections.
-      #
-      # If a block is supplied, the block is supplied with a writer that should be used to
-      # accumulate the results.
-      #
-      # Examples
-      #
-      # job.write_output('hello world')
-      #
-      # job.write_output do |writer|
-      #   writer << 'hello world'
-      # end
-      #
-      # job.write_output do |writer|
-      #   result = RocketJob::Batch::Results
-      #   result << RocketJob::Batch::Result.new(:main, 'hello world')
-      #   result << RocketJob::Batch::Result.new(:errors, 'errors')
-      #   writer << result
-      # end
-      #
-      # result = RocketJob::Batch::Results
-      # result << RocketJob::Batch::Result.new(:main, 'hello world')
-      # result << RocketJob::Batch::Result.new(:errors, 'errors')
-      # job.write_output(result)
-      def write_output(result = nil, input_slice = nil, &block)
-        if block
-          RocketJob::Sliced::Writer::Output.collect(self, input_slice, &block)
-        else
-          raise(ArgumentError, "result parameter is required when no block is supplied") unless result
+      private
 
-          RocketJob::Sliced::Writer::Output.collect(self, input_slice) { |writer| writer << result }
+      # Return a lambda to extract the header row from the uploaded file.
+      def rocket_job_upload_header_lambda(category, on_first)
+        case category.mode
+        when :line
+          lambda do |line|
+            category.tabular.parse_header(line)
+            category.cleanse_header!
+            category.columns = category.tabular.header.columns
+            # Call chained on_first if present
+            on_first&.call(line)
+          end
+        when :array
+          lambda do |row|
+            category.tabular.header.columns = row
+            category.cleanse_header!
+            category.columns = category.tabular.header.columns
+            # Call chained on_first if present
+            on_first&.call(line)
+          end
         end
       end
     end

data/lib/rocket_job/batch/model.rb

@@ -11,46 +11,6 @@ module RocketJob
         #
         # The following attributes are set when the job is created
 
-        # Number of records to include in each slice that is processed
-        # Note:
-        #   slice_size is only used by SlicedJob#upload & Sliced::Input#upload
-        #   When slices are supplied directly, their size is not modified to match this number
-        field :slice_size, type: Integer, default: 100, class_attribute: true, user_editable: true, copy_on_restart: true
-
-        # Whether to retain nil results.
-        #
-        # Only applicable if `collect_output` is `true`
-        # Set to `false` to prevent collecting output from the perform
-        # method when it returns `nil`.
-        field :collect_nil_output, type: Boolean, default: true, class_attribute: true
-
-        # Optional Array<Symbol> list of categories that this job can output to
-        #
-        # By using categories the output from #perform can be placed in different
-        # output collections, and therefore different output files
-        #
-        # Categories must be declared in advance to avoid a #perform method
-        # accidentally writing its results to an unknown category
-        field :output_categories, type: Array, default: [:main], class_attribute: true
-
-        # Optional Array<Symbol> list of categories that this job can load input data into
-        field :input_categories, type: Array, default: [:main], class_attribute: true
-
-        # The file name of the uploaded file, if any.
-        # Set by #upload if a file name was supplied, but can also be set explicitly.
-        # May or may not include the fully qualified path name.
-        field :upload_file_name, type: String
-
-        # Compress uploaded records.
-        # The fields are not affected in any way, only the data stored in the
-        # records and results collections will compressed
-        field :compress, type: Object, default: false, class_attribute: true
-
-        # Encrypt uploaded records.
-        # The fields are not affected in any way, only the data stored in the
-        # records and results collections will be encrypted
-        field :encrypt, type: Object, default: false, class_attribute: true
-
         #
         # Values that jobs can also update during processing
         #
@@ -69,30 +29,7 @@ module RocketJob
 
         # Breaks the :running state up into multiple sub-states:
         #   :running -> :before -> :processing -> :after -> :complete
-        field :sub_state, type: Symbol
-
-        validates_presence_of :slice_size
-
-        validates_each :output_categories, :input_categories do |record, attr, value|
-          # Under some circumstances ActiveModel is passing in a nil value even though the
-          # attributes have default values
-          Array(value).each do |category|
-            record.errors.add(attr, "must only contain Symbol values") unless category.is_a?(Symbol)
-            unless category.to_s =~ /\A[a-z_0-9]+\Z/
-              record.errors.add(attr, "must only consist of lowercase characters, digits, and _")
-            end
-          end
-        end
-      end
-
-      # Returns [true|false] whether the slices for this job are encrypted
-      def encrypted?
-        encrypt == true
-      end
-
-      # Returns [true|false] whether the slices for this job are compressed
-      def compressed?
-        compress == true
+        field :sub_state, type: Mongoid::StringifiedSymbol
       end
 
       # Returns [Integer] percent of records completed so far
@@ -102,10 +39,10 @@ module RocketJob
         return 0 unless record_count.to_i.positive?
 
         # Approximate number of input records
-        input_records = input.count.to_f * slice_size
+        input_records = input.count.to_f * input_category.slice_size
         if input_records > record_count
           # Sanity check in case slice_size is not being adhered to
-          99
+          0
         else
           ((1.0 - (input_records.to_f / record_count)) * 100).to_i
         end
@@ -120,6 +57,10 @@ module RocketJob
           h["active_slices"] = worker_count
           h["failed_slices"] = input.failed.count
           h["queued_slices"] = input.queued.count
+          output_categories.each do |category|
+            name_str                      = category.name == :main ? "" : "_#{category.name}"
+            h["output_slices#{name_str}"] = output(category).count
+          end
           # Very high level estimated time left
           if record_count && running? && record_count.positive?
             percent = percent_complete
@@ -129,10 +70,9 @@ module RocketJob
             end
           end
         elsif completed?
-          secs = seconds.to_f
+          secs                  = seconds.to_f
           h["records_per_hour"] = ((record_count.to_f / secs) * 60 * 60).round if record_count&.positive? && (secs > 0.0)
         end
-        h["output_slices"] = output.count if collect_output? && !completed?
         h.merge!(super(time_zone))
         h.delete("result")
         # Worker name should be retrieved from the slices when processing
@@ -172,6 +112,18 @@ module RocketJob
         @worker_count_last = Time.now.to_i
         @worker_count
       end
+
+      # @deprecated
+      # For backward compatibility
+      def upload_file_name
+        input_category.file_name
+      end
+
+      # @deprecated
+      # For backward compatibility
+      def upload_file_name=(upload_file_name)
+        input_category.file_name = upload_file_name
+      end
     end
   end
 end

data/lib/rocket_job/batch/performance.rb

@@ -22,12 +22,15 @@ module RocketJob
         count_running_workers
 
         puts "Loading job with #{count} records/lines"
-        args = {log_level: :warn, slice_size: slice_size}
-        if defined?(::RocketJob)
-          args[:compress] = compress
-          args[:encrypt]  = encrypt
+        job                           = RocketJob::Jobs::PerformanceJob.new(log_level: :warn)
+        job.input_category.slice_size = slice_size
+        if encrypt
+          job.input_category.serializer  = :encrypt
+          job.output_category.serializer = :encrypt
+        elsif !compress
+          job.input_category.serializer  = :none
+          job.output_category.serializer = :none
         end
-        job = RocketJob::Jobs::PerformanceJob.new(args)
         job.upload do |writer|
           count.times { |i| writer << i }
         end
@@ -37,7 +40,15 @@ module RocketJob
         sleep 3 until job.reload.completed?
 
         duration = job.completed_at - job.started_at
-        {count: count, duration: duration, records_per_second: (count.to_f / duration).round(3), workers: workers, servers: servers, compress: compress, encrypt: encrypt}
+        {
+          count:              count,
+          duration:           duration,
+          records_per_second: (count.to_f / duration).round(3),
+          workers:            workers,
+          servers:            servers,
+          compress:           compress,
+          encrypt:            encrypt
+        }
       end
 
       # Export the Results hash to a CSV file
@@ -60,7 +71,8 @@ module RocketJob
           o.on("-m", "--mongo MONGO_CONFIG_FILE_NAME", "Location of mongoid.yml config file") do |arg|
             self.mongo_config = arg
           end
-          o.on("-e", "--environment ENVIRONMENT", "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
+          o.on("-e", "--environment ENVIRONMENT",
+               "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
             self.environment = arg
           end
           o.on("-z", "--compress", "Turn on compression") do