job-iteration 1.13.1 → 1.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8e5e3fa1240a67b88f2b83290f7b01fcadb28b4a7be257f646d199cff9c2e97
-  data.tar.gz: 6a2130de073c8c5eb29c13934f7e4bb73af31cf15fc33af74b5738b420a9b38b
+  metadata.gz: a6cd8bb708ce42d3333f9631f344bb33b506cd2e3d0fc8a5d73882f2f52a811a
+  data.tar.gz: 30b9ece97b5e981a61fbef2110e60fdffcbd95dfdeec14def3652f9384f7e4d5
 SHA512:
-  metadata.gz: b6316aa38f0e6f7df7874a478253b6e285db61c6077089c43ef1edfce408ec1835cdd7d50d4ccf257e096677cc4fcdb23e67cc609efc48998ae91a0667a8974a
-  data.tar.gz: ac9fdbf6afeb440010323c70aa1fcdf60288460788893b0a0dbc536684caeca3291a0e90c80142439c7aac4efbf4853039b482ae37078d2b2c69ab2d565680be
+  metadata.gz: cc9bd924dfbce3e582356a9b29edec62ca597c6b50cad39afb29970e9d1b1ce799678797b7441b6aa6732d25e8b5e8fd896f88b7dd118cae552f5acc50a31143
+  data.tar.gz: 532e91769910a4172d8f7773bfd59b3fa54372fed4b9fd22d6309afa0861a626a878dedb50972127658a76e90b651cf1837a7d87fc61deebc81655ee418698d9

data/CHANGELOG.md

@@ -16,6 +16,22 @@ nil
 
 nil
 
+## v1.14.0 (May 14, 2026)
+
+### Breaking Changes
+
+- [704](https://github.com/Shopify/job-iteration/pull/704) - Drop support for Rails 7.0. The minimum supported Rails version is now 7.1.
+
+### Features
+
+- [702](https://github.com/Shopify/job-iteration/pull/702) - Add support for parallel iteration with `enumerator_builder.parallel`. This enqueues multiple jobs, allowing you to split up the work across multiple instances.
+- [705](https://github.com/Shopify/job-iteration/pull/705) - Add support for parallel array iteration with `enumerator_builder.parallel_array`.
+- [706](https://github.com/Shopify/job-iteration/pull/706) - Add support for parallel Active Record relation iteration with `enumerator_builder.parallel_active_record_on_records` and `enumerator_builder.parallel_active_record_on_batches`.
+
+### Bug fixes
+
+- [709](https://github.com/Shopify/job-iteration/pull/709) - Fix an issue with parallel iteration when `instances` changed after child jobs were enqueued.
+
 ## v1.13.1 (Apr 28, 2026)
 
 ### Bug fixes

data/README.md

@@ -167,7 +167,7 @@ Job-iteration currently supports the following queue adapters (in order of imple
 It supports the following platforms:
 
 - Ruby 3.1 and later
-- Rails 7.0 and later
+- Rails 7.1 and later
 
 Support for older platforms that have reached end of life may occasionally be dropped if maintaining backwards compatibility is cumbersome.
 

data/job-iteration.gemspec

@@ -26,5 +26,5 @@ Gem::Specification.new do |spec|
   spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md"
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
-  spec.add_dependency("activejob", ">= 7.0")
+  spec.add_dependency("activejob", ">= 7.1")
 end

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -66,10 +66,6 @@ module JobIteration
       pkey = @column_mgr.primary_key
       pkey_values = primary_key_values
 
-      # If the primary key is only composed of a single column, simplify the
-      # query. This keeps us compatible with Rails prior to 7.1 where composite
-      # primary keys were introduced along with the syntax that allows you to
-      # query for multi-column values.
       if pkey.size <= 1
         pkey = pkey.first
         pkey_values = pkey_values.map(&:first)

data/lib/job-iteration/active_record_cursor.rb

@@ -18,7 +18,7 @@ module JobIteration
       end
     end
 
-    def initialize(relation, columns = nil, position = nil)
+    def initialize(relation, columns, position, instance, instances)
       @columns = if columns
         Array(columns)
       else
@@ -35,6 +35,17 @@ module JobIteration
       end
 
       @base_relation = relation.reorder(@columns.join(","))
+
+      if instances.present?
+        pk = relation.primary_key
+        unless pk.is_a?(String) && relation.klass.column_for_attribute(pk).type == :integer
+          raise ArgumentError, "Parallel iteration requires a single integer primary key. " \
+            "For more complex cases, use the enumerator_builder.parallel primitive directly."
+        end
+
+        @base_relation = @base_relation.where("#{relation.table_name}.#{pk} % ? = ?", instances, instance)
+      end
+
       @reached_end = false
     end
 
@@ -56,7 +67,7 @@ module JobIteration
       self.position = @columns.map do |column|
         method = column.to_s.split(".").last
 
-        if ActiveRecord.version >= Gem::Version.new("7.1.0.alpha") && method == "id"
+        if method == "id"
           record.id_value
         else
           record.send(method.to_sym)

data/lib/job-iteration/active_record_enumerator.rb

@@ -7,7 +7,7 @@ module JobIteration
   class ActiveRecordEnumerator
     SQL_DATETIME_WITH_NSEC = "%Y-%m-%d %H:%M:%S.%N"
 
-    def initialize(relation, columns: nil, batch_size: 100, timezone: nil, cursor: nil)
+    def initialize(relation, columns: nil, batch_size: 100, timezone: nil, cursor: nil, instance: nil, instances: nil)
       @relation = relation
       @batch_size = batch_size
       @timezone = timezone
@@ -17,6 +17,8 @@ module JobIteration
         Array(relation.primary_key).map { |pk| "#{relation.table_name}.#{pk}" }
       end
       @cursor = cursor
+      @instance = instance
+      @instances = instances
     end
 
     def records
@@ -39,7 +41,8 @@ module JobIteration
     end
 
     def size
-      @relation.count(:all)
+      full_size = @relation.count(:all)
+      @instances.present? ? full_size / @instances : full_size
     end
 
     private
@@ -61,7 +64,7 @@ module JobIteration
     end
 
     def finder_cursor
-      JobIteration::ActiveRecordCursor.new(@relation, @columns, @cursor)
+      JobIteration::ActiveRecordCursor.new(@relation, @columns, @cursor, @instance, @instances)
     end
 
     def column_value(record, attribute)

data/lib/job-iteration/enumerator_builder.rb

@@ -5,6 +5,7 @@ require_relative "active_record_enumerator"
 require_relative "csv_enumerator"
 require_relative "throttle_enumerator"
 require_relative "nested_enumerator"
+require_relative "parallel_enumerator"
 require "forwardable"
 
 module JobIteration
@@ -65,6 +66,23 @@ module JobIteration
       wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size - drop })
     end
 
+    # Builds an Enumerator that iterates over a given array, across +instances+ parallel jobs.
+    #
+    # Child job i iterates over the slice of the array starting at
+    # index (array.size / instances * i).floor and ending at index (array.size / instances * (i + 1)).floor - 1.
+    def build_parallel_array_enumerator(array, instances:, cursor:)
+      unless array.is_a?(Array)
+        raise ArgumentError, "array must be an Array"
+      end
+
+      build_parallel_enumerator(instances: instances, cursor: cursor) do |instance, instances, inner_cursor|
+        slice_start = (array.size.to_f / instances * instance).floor
+        next_slice_start = (array.size.to_f / instances * (instance + 1)).floor
+        slice = array[slice_start...next_slice_start]
+        build_array_enumerator(slice, cursor: inner_cursor)
+      end
+    end
+
     # Builds Enumerator from Active Record Relation. Each Enumerator tick moves the cursor one row forward.
     #
     # +columns:+ argument is used to build the actual query for iteration. +columns+: defaults to primary key:
@@ -103,6 +121,21 @@ module JobIteration
       wrap(self, enum)
     end
 
+    # Builds an Enumerator that iterates over a given Active Record Relation, across +instances+ parallel jobs.
+    #
+    # Child job i iterates over the records where the id is equal to (instance % instances).
+    def build_parallel_active_record_enumerator_on_records(scope, instances:, cursor:, **args)
+      build_parallel_enumerator(instances: instances, cursor: cursor) do |instance, instances, inner_cursor|
+        build_active_record_enumerator(
+          scope,
+          cursor: inner_cursor,
+          instances: instances,
+          instance: instance,
+          **args,
+        ).records
+      end
+    end
+
     # Builds Enumerator from Active Record Relation and enumerates on batches of records.
     # Each Enumerator tick moves the cursor +batch_size+ rows forward.
     #
@@ -118,6 +151,21 @@ module JobIteration
       wrap(self, enum)
     end
 
+    # Builds an Enumerator that iterates over a given Active Record Relation, across +instances+ parallel jobs, and enumerates on batches.
+    #
+    # Child job i iterates over the batches of records where the id is equal to (instance % instances).
+    def build_parallel_active_record_enumerator_on_batches(scope, instances:, cursor:, **args)
+      build_parallel_enumerator(instances: instances, cursor: cursor) do |instance, instances, inner_cursor|
+        build_active_record_enumerator(
+          scope,
+          cursor: inner_cursor,
+          instances: instances,
+          instance: instance,
+          **args,
+        ).batches
+      end
+    end
+
     # Builds Enumerator from Active Record Relation and enumerates on batches, yielding Active Record Relations.
     # See documentation for #build_active_record_enumerator_on_batches.
     def build_active_record_enumerator_on_batch_relations(scope, wrap: true, cursor:, **args)
@@ -185,27 +233,48 @@ module JobIteration
       wrap(self, enum)
     end
 
+    def build_parallel_enumerator(instances:, cursor:, &block)
+      unless instances.is_a?(Integer) && instances.positive?
+        raise ArgumentError, "instances must be a positive Integer"
+      end
+
+      return ParallelEnumerator::EnqueueJobs.new(instances) if cursor.nil?
+
+      enum = ParallelEnumerator.new(block, cursor: cursor).to_enum
+      wrap(self, enum)
+    end
+
     alias_method :once, :build_once_enumerator
     alias_method :times, :build_times_enumerator
     alias_method :array, :build_array_enumerator
+    alias_method :parallel_array, :build_parallel_array_enumerator
     alias_method :active_record_on_records, :build_active_record_enumerator_on_records
+    alias_method :parallel_active_record_on_records, :build_parallel_active_record_enumerator_on_records
     alias_method :active_record_on_batches, :build_active_record_enumerator_on_batches
+    alias_method :parallel_active_record_on_batches, :build_parallel_active_record_enumerator_on_batches
     alias_method :active_record_on_batch_relations, :build_active_record_enumerator_on_batch_relations
     alias_method :throttle, :build_throttle_enumerator
     alias_method :csv, :build_csv_enumerator
     alias_method :csv_on_batches, :build_csv_enumerator_on_batches
     alias_method :nested, :build_nested_enumerator
+    alias_method :parallel, :build_parallel_enumerator
 
     private
 
-    def build_active_record_enumerator(scope, cursor:, **args)
+    def build_active_record_enumerator(scope, cursor:, instance: nil, instances: nil, **args)
       unless scope.is_a?(ActiveRecord::Relation)
         raise ArgumentError, "scope must be an ActiveRecord::Relation"
       end
 
+      if (instance.nil? && instances.present?) || (instance.present? && instances.nil?)
+        raise ArgumentError, "instance and instances must both be provided or both be nil"
+      end
+
       JobIteration::ActiveRecordEnumerator.new(
         scope,
         cursor: cursor,
+        instance: instance,
+        instances: instances,
         **args,
       )
     end

data/lib/job-iteration/iteration.rb

@@ -141,6 +141,14 @@ module JobIteration
         return
       end
 
+      if enumerator.is_a?(ParallelEnumerator::EnqueueJobs)
+        tags = instrumentation_tags.merge(instances: enumerator.instances)
+        ActiveSupport::Notifications.instrument("enqueue_parallel_jobs.iteration", tags) do
+          enumerator.enqueue_jobs(self)
+        end
+        return
+      end
+
       assert_enumerator!(enumerator)
 
       if executions == 1 && times_interrupted == 0

data/lib/job-iteration/log_subscriber.rb

@@ -19,6 +19,12 @@ module JobIteration
       end
     end
 
+    def enqueue_parallel_jobs(event)
+      info do
+        "[JobIteration::Iteration] Enqueued #{event.payload[:instances]} parallel jobs."
+      end
+    end
+
     def interrupted(event)
       info do
         "[JobIteration::Iteration] Interrupting and re-enqueueing the job " \

data/lib/job-iteration/parallel_enumerator.rb

@@ -0,0 +1,52 @@
+# typed: true
+# frozen_string_literal: true
+
+module JobIteration
+  # ParallelEnumerator allows you to parallelize iterations.
+  class ParallelEnumerator
+    class EnqueueError < StandardError; end
+
+    class EnqueueJobs
+      def initialize(instances)
+        @instances = instances
+      end
+
+      attr_reader :instances
+
+      def enqueue_jobs(job)
+        child_jobs = instances.times.map do |index|
+          job.class.new(*job.arguments).tap do |child_job|
+            child_job.cursor_position = { "instance" => index, "instances" => instances, "inner_cursor" => nil }
+
+            # Carry forward potential overrides from the parent job
+            child_job.queue_name = job.queue_name
+            child_job.priority = job.priority if job.priority
+          end
+        end
+
+        ActiveJob.perform_all_later(child_jobs)
+
+        unless child_jobs.all?(&:successfully_enqueued?)
+          failed_count = instances - child_jobs.count(&:successfully_enqueued?)
+          raise EnqueueError, "Failed to enqueue #{failed_count} out of #{instances} child jobs"
+        end
+      end
+    end
+
+    def initialize(block, cursor:)
+      @instance = cursor["instance"]
+      @instances = cursor["instances"]
+      inner_cursor = cursor["inner_cursor"]
+      @inner_enum = block.call(@instance, @instances, inner_cursor)
+    end
+
+    def to_enum
+      Enumerator.new(-> { @inner_enum.size }) do |yielder|
+        @inner_enum.each do |object_from_enumerator, cursor_from_enumerator|
+          parallel_cursor = { "instance" => @instance, "instances" => @instances, "inner_cursor" => cursor_from_enumerator }
+          yielder.yield(object_from_enumerator, parallel_cursor)
+        end
+      end
+    end
+  end
+end

data/lib/job-iteration/railtie.rb

@@ -5,8 +5,7 @@ return unless defined?(Rails::Railtie)
 module JobIteration
   class Railtie < Rails::Railtie
     initializer "job_iteration.register_deprecator" do |app|
-      # app.deprecators was added in Rails 7.1
-      app.deprecators[:job_iteration] = JobIteration::Deprecation if app.respond_to?(:deprecators)
+      app.deprecators[:job_iteration] = JobIteration::Deprecation
     end
   end
 end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.13.1"
+  VERSION = "1.14.0"
 end

data/lib/tapioca/dsl/compilers/job_iteration.rb

@@ -105,15 +105,11 @@ module Tapioca
           ).returns(T::Array[RBI::TypedParam])
         end
         def perform_later_parameters(parameters, returned_job_class)
-          if ::Gem::Requirement.new(">= 7.0").satisfied_by?(::ActiveJob.gem_version)
-            parameters.reject! { |typed_param| RBI::BlockParam === typed_param.param }
-            parameters + [create_block_param(
-              "block",
-              type: "T.nilable(T.proc.params(job: #{returned_job_class}).void)",
-            )]
-          else
-            parameters
-          end
+          parameters.reject! { |typed_param| RBI::BlockParam === typed_param.param }
+          parameters + [create_block_param(
+            "block",
+            type: "T.nilable(T.proc.params(job: #{returned_job_class}).void)",
+          )]
         end
 
         class << self

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.13.1
+  version: 1.14.0
 platform: ruby
 authors:
 - Shopify
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '7.1'
 description: Makes your background jobs interruptible and resumable.
 email:
 - ops-accounts+shipit@shopify.com
@@ -52,6 +52,7 @@ files:
 - lib/job-iteration/iteration.rb
 - lib/job-iteration/log_subscriber.rb
 - lib/job-iteration/nested_enumerator.rb
+- lib/job-iteration/parallel_enumerator.rb
 - lib/job-iteration/railtie.rb
 - lib/job-iteration/test_helper.rb
 - lib/job-iteration/throttle_enumerator.rb
@@ -77,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.11
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.
 test_files: []