pluck_in_batches 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d86f3ce009db02836e820ec434ae17fa43685651a74e85e2260675d7f2aeb945
-  data.tar.gz: e27703d5c07b89db1d75adc5331ce597c8b95669575a65596e0e141a44287412
+  metadata.gz: 7c30844758feb52e696cbde2155d47204b537201975ec2a7857db5400ca982c9
+  data.tar.gz: 69c73ac8249b95a2b9d8e4542396ee7969be7d28aa3e2277b06f5ed652eedc87
 SHA512:
-  metadata.gz: f96ef381074b16ab8cd5fa48ca6f38245cae06ae0213fcb3c1ebb159b7c6712a441bbdf08c5ae36bb63a12bc46d04daa70c38ef2f5bd04462397bf254ab610f0
-  data.tar.gz: bd5e5382cbd444f4cc0d9b81bad56792c6a6f436cad3376064d1b1cfe46699c2b129e36b4311cedda26aec827b5a140d7f344ebeb959bb1e22594414e1577d6b
+  metadata.gz: 7e49e5929fefd29e092c0dc5da7f051fd735dd2175a347dfa0a34af274c105b3dd5c4897f90b350ad2ac982b260f4cfbe64d2504878f5f554036bb7466210579
+  data.tar.gz: a1ba0c9f0a426c0fd6cd73985af955cad50dbca4c7777049b94cbf85f655721edffeb13dc4bc71d8c8bc78e357ff22b620b96b7718a64c59a8fb091e3029164d

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 ## master (unreleased)
 
+## 0.3.0 (2024-11-03)
+
+- Support plucking custom Arel columns
+
+  ```ruby
+  User.pluck_in_batches(:id, Arel.sql("json_extract(users.metadata, '$.rank')"))
+  ```
+
+## 0.2.0 (2023-07-24)
+
+- Support specifying per cursor column ordering when batching
+
+  ```ruby
+  Book.pluck_in_batches(:title, cursor_columns: [:author_id, :version], order: [:asc, :desc])
+  ```
+
+- Add `:of` as an alias for `:batch_size` option
+
 ## 0.1.0 (2023-05-16)
 
 - First release

data/README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://github.com/fatkodima/pluck_in_batches/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/fatkodima/pluck_in_batches/actions/workflows/ci.yml)
 
-ActiveRecord comes with `find_each` and `find_in_batches` methods to batch process records from a database.
+ActiveRecord comes with `find_each` / `find_in_batches` / `in_batches` methods to batch process records from a database.
 ActiveRecord also has the `pluck` method which allows the selection of a set of fields without pulling
 the entire record into memory.
 
@@ -14,7 +14,7 @@ It performs half of the number of SQL queries, allocates up to half of the memor
 
 ```ruby
 # Before
-User.in_batches do |batch|
+User.in_batches do |batch| # or .find_in_batches, or .select(:email).find_each etc
   emails = batch.pluck(:emails)
   # do something with emails
 end
@@ -25,6 +25,8 @@ User.pluck_in_batches(:email) do |emails|
 end
 ```
 
+**Note**: You may also find [`sidekiq-iteration`](https://github.com/fatkodima/sidekiq-iteration) useful when iterating over large collections in Sidekiq jobs.
+
 ## Requirements
 
 - Ruby 2.7+
@@ -89,18 +91,25 @@ User.pluck_in_batches(:name, :email).with_index do |group, index|
   jobs = group.map { |name, email| PartyReminderJob.new(name, email) }
   ActiveJob.perform_all_later(jobs)
 end
+
+# Custom arel column
+User.pluck_in_batches(:id, Arel.sql("json_extract(users.metadata, '$.rank')")).with_index do |group, index|
+  # ...
+end
 ```
 
 Both methods support the following configuration options:
 
 * `:batch_size` - Specifies the size of the batch. Defaults to 1000.
+  Also aliased as `:of`.
 * `:start` - Specifies the primary key value to start from, inclusive of the value.
 * `:finish` - Specifies the primary key value to end at, inclusive of the value.
 * `:error_on_ignore` - Overrides the application config to specify if an error should be raised when
   an order is present in the relation.
 * `:cursor_column` - Specifies the column(s) on which the iteration should be done.
   This column(s) should be orderable (e.g. an integer or string). Defaults to primary key.
-* `:order` - Specifies the primary key order (can be `:asc` or `:desc`). Defaults to `:asc`.
+* `:order` - Specifies the primary key order (can be `:asc` or `:desc` or
+  an array consisting of :asc or :desc). Defaults to `:asc`.
 
 ## Development
 

data/lib/pluck_in_batches/extensions.rb

@@ -13,7 +13,7 @@ module PluckInBatches
       #
       # See #pluck_in_batches for all the details.
       #
-      def pluck_each(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: :asc, cursor_column: primary_key, &block)
+      def pluck_each(*columns, start: nil, finish: nil, of: 1000, batch_size: of, error_on_ignore: nil, order: :asc, cursor_column: primary_key, &block)
         iterator = Iterator.new(self)
         iterator.each(*columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order, &block)
       end
@@ -37,13 +37,23 @@ module PluckInBatches
       #
       # ==== Options
       # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
+      # * <tt>:of</tt> - Same as +:batch_size+.
       # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
       # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
       # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
       #   an order is present in the relation.
       # * <tt>:cursor_column</tt> - Specifies the column(s) on which the iteration should be done.
       #   This column(s) should be orderable (e.g. an integer or string). Defaults to primary key.
-      # * <tt>:order</tt> - Specifies the cursor column(s) order (can be +:asc+ or +:desc+). Defaults to +:asc+.
+      # * <tt>:order</tt> - Specifies the cursor column(s) order (can be +:asc+ or +:desc+ or an array consisting
+      #   of :asc or :desc). Defaults to +:asc+.
+      #
+      #     class Book < ActiveRecord::Base
+      #       self.primary_key = [:author_id, :version]
+      #     end
+      #
+      #     Book.pluck_in_batches(:title, order: [:asc, :desc])
+      #
+      #   In the above code, +author_id+ is sorted in ascending order and +version+ in descending order.
       #
       # Limits are honored, and if present there is no requirement for the batch
       # size: it can be less than, equal to, or greater than the limit.
@@ -68,7 +78,7 @@ module PluckInBatches
       # NOTE: By its nature, batch processing is subject to race conditions if
       # other processes are modifying the database.
       #
-      def pluck_in_batches(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: primary_key, order: :asc, &block)
+      def pluck_in_batches(*columns, start: nil, finish: nil, of: 1000, batch_size: of, error_on_ignore: nil, cursor_column: primary_key, order: :asc, &block)
         iterator = Iterator.new(self)
         iterator.each_batch(*columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order, &block)
       end

data/lib/pluck_in_batches/iterator.rb

@@ -2,12 +2,15 @@
 
 module PluckInBatches
   class Iterator # :nodoc:
+    VALID_ORDERS = [:asc, :desc].freeze
+    DEFAULT_ORDER = :asc
+
     def initialize(relation)
       @relation = relation
       @klass = relation.klass
     end
 
-    def each(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: @relation.primary_key, order: :asc, &block)
+    def each(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: @relation.primary_key, order: DEFAULT_ORDER, &block)
       if columns.empty?
         raise ArgumentError, "Call `pluck_each' with at least one column."
       end
@@ -18,21 +21,28 @@ module PluckInBatches
         end
       else
         enum_for(__callee__, *columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order) do
-          apply_limits(@relation, start, finish, order).size
+          apply_limits(@relation, start, finish, build_batch_orders(order)).size
         end
       end
     end
 
-    def each_batch(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: @relation.primary_key, order: :asc)
+    def each_batch(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: @relation.primary_key, order: DEFAULT_ORDER)
       if columns.empty?
         raise ArgumentError, "Call `pluck_in_batches' with at least one column."
       end
 
-      unless order == :asc || order == :desc
-        raise ArgumentError, ":order must be :asc or :desc, got #{order.inspect}"
+      unless Array(order).all? { |ord| VALID_ORDERS.include?(ord) }
+        raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
+      end
+
+      pluck_columns = columns.map do |column|
+        if Arel.arel_node?(column)
+          column
+        else
+          column.to_s
+        end
       end
 
-      pluck_columns = columns.map(&:to_s)
       cursor_columns = Array(cursor_column).map(&:to_s)
       cursor_column_indexes = cursor_column_indexes(pluck_columns, cursor_columns)
       missing_cursor_columns = cursor_column_indexes.count(&:nil?)
@@ -44,10 +54,11 @@ module PluckInBatches
       end
 
       relation = @relation
+      batch_orders = build_batch_orders(cursor_columns, order)
 
       unless block_given?
         return to_enum(__callee__, *columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order) do
-          total = apply_limits(relation, cursor_columns, start, finish, order).size
+          total = apply_limits(relation, cursor_columns, start, finish, batch_orders).size
           (total - 1).div(batch_size) + 1
         end
       end
@@ -62,8 +73,8 @@ module PluckInBatches
         batch_limit = remaining if remaining < batch_limit
       end
 
-      relation = relation.reorder(*batch_order(cursor_columns, order)).limit(batch_limit)
-      relation = apply_limits(relation, cursor_columns, start, finish, order)
+      relation = relation.reorder(batch_orders.to_h).limit(batch_limit)
+      relation = apply_limits(relation, cursor_columns, start, finish, batch_orders)
       relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
       batch_relation = relation
 
@@ -99,9 +110,13 @@ module PluckInBatches
           end
         end
 
-        batch_relation = batch_condition(
-          relation, cursor_columns, cursor_column_offsets, order == :desc ? :lt : :gt
-        )
+        _last_column, last_order = batch_orders.last
+        operators = batch_orders.map do |_column, order| # rubocop:disable Lint/ShadowingOuterLocalVariable
+          order == :desc ? :lteq : :gteq
+        end
+        operators[-1] = (last_order == :desc ? :lt : :gt)
+
+        batch_relation = batch_condition(relation, cursor_columns, cursor_column_offsets, operators)
       end
     end
 
@@ -135,29 +150,33 @@ module PluckInBatches
         end
       end
 
-      def apply_limits(relation, columns, start, finish, order)
-        relation = apply_start_limit(relation, columns, start, order) if start
-        relation = apply_finish_limit(relation, columns, finish, order) if finish
+      def apply_limits(relation, columns, start, finish, batch_orders)
+        relation = apply_start_limit(relation, columns, start, batch_orders) if start
+        relation = apply_finish_limit(relation, columns, finish, batch_orders) if finish
         relation
       end
 
-      def apply_start_limit(relation, columns, start, order)
-        batch_condition(relation, columns, start, order == :desc ? :lteq : :gteq)
+      def apply_start_limit(relation, columns, start, batch_orders)
+        operators = batch_orders.map do |_column, order|
+          order == :desc ? :lteq : :gteq
+        end
+        batch_condition(relation, columns, start, operators)
       end
 
-      def apply_finish_limit(relation, columns, finish, order)
-        batch_condition(relation, columns, finish, order == :desc ? :gteq : :lteq)
+      def apply_finish_limit(relation, columns, finish, batch_orders)
+        operators = batch_orders.map do |_column, order|
+          order == :desc ? :gteq : :lteq
+        end
+        batch_condition(relation, columns, finish, operators)
       end
 
-      def batch_condition(relation, columns, values, operator)
-        columns = Array(columns)
-        values = Array(values)
-        cursor_positions = columns.zip(values)
+      def batch_condition(relation, columns, values, operators)
+        cursor_positions = Array(columns).zip(Array(values), operators)
 
-        first_clause_column, first_clause_value = cursor_positions.pop
+        first_clause_column, first_clause_value, operator = cursor_positions.pop
         where_clause = build_attribute_predicate(first_clause_column, first_clause_value, operator)
 
-        cursor_positions.reverse_each do |column_name, value|
+        cursor_positions.reverse_each do |column_name, value, operator| # rubocop:disable Lint/ShadowingOuterLocalVariable
           where_clause = build_attribute_predicate(column_name, value, operator == :lteq ? :lt : :gt).or(
             build_attribute_predicate(column_name, value, :eq).and(where_clause)
           )
@@ -170,9 +189,9 @@ module PluckInBatches
         @relation.bind_attribute(column, value) { |attr, bind| attr.public_send(operator, bind) }
       end
 
-      def batch_order(cursor_columns, order)
-        cursor_columns.map do |column|
-          @relation.arel_table[column].public_send(order)
+      def build_batch_orders(cursor_columns, order)
+        cursor_columns.zip(Array(order)).map do |column, ord|
+          [column, ord || DEFAULT_ORDER]
         end
       end
 

data/lib/pluck_in_batches/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PluckInBatches
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pluck_in_batches
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-16 00:00:00.000000000 Z
+date: 2024-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -60,8 +60,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.12
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
-summary: Change
+summary: A faster alternative to the custom use of `in_batches` with `pluck`.
 test_files: []