pluck_in_batches 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d86f3ce009db02836e820ec434ae17fa43685651a74e85e2260675d7f2aeb945
+  data.tar.gz: e27703d5c07b89db1d75adc5331ce597c8b95669575a65596e0e141a44287412
+SHA512:
+  metadata.gz: f96ef381074b16ab8cd5fa48ca6f38245cae06ae0213fcb3c1ebb159b7c6712a441bbdf08c5ae36bb63a12bc46d04daa70c38ef2f5bd04462397bf254ab610f0
+  data.tar.gz: bd5e5382cbd444f4cc0d9b81bad56792c6a6f436cad3376064d1b1cfe46699c2b129e36b4311cedda26aec827b5a140d7f344ebeb959bb1e22594414e1577d6b

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## master (unreleased)
+
+## 0.1.0 (2023-05-16)
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 fatkodima
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,115 @@
+# PluckInBatches
+
+[![Build Status](https://github.com/fatkodima/pluck_in_batches/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/fatkodima/pluck_in_batches/actions/workflows/ci.yml)
+
+ActiveRecord comes with `find_each` and `find_in_batches` methods to batch process records from a database.
+ActiveRecord also has the `pluck` method which allows the selection of a set of fields without pulling
+the entire record into memory.
+
+This gem combines these ideas and provides `pluck_each` and `pluck_in_batches` methods to allow
+batch processing of plucked fields from the database.
+
+It performs half of the number of SQL queries, allocates up to half of the memory and is up to 2x faster
+(or more, depending on how far is your database from the application) than the available alternative:
+
+```ruby
+# Before
+User.in_batches do |batch|
+  emails = batch.pluck(:emails)
+  # do something with emails
+end
+
+# Now, using this gem (up to 2x faster)
+User.pluck_in_batches(:email) do |emails|
+  # do something with emails
+end
+```
+
+## Requirements
+
+- Ruby 2.7+
+- ActiveRecord 6+
+
+If you need support for older versions, [open an issue](https://github.com/fatkodima/pluck_in_batches/issues/new).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pluck_in_batches'
+```
+
+And then execute:
+
+```sh
+$ bundle
+```
+
+Or install it yourself as:
+
+```sh
+$ gem install pluck_in_batches
+```
+
+## Usage
+
+### `pluck_each`
+
+Behaves similarly to `find_each` ActiveRecord's method, but yields each set of values corresponding
+to the specified columns.
+
+```ruby
+# Single column
+User.where(active: true).pluck_each(:email) do |email|
+  # do something with email
+end
+
+# Multiple columns
+User.where(active: true).pluck_each(:id, :email) do |id, email|
+  # do something with id and email
+end
+```
+
+### `pluck_in_batches`
+
+Behaves similarly to `in_batches` ActiveRecord's method, but yields each batch
+of values corresponding to the specified columns.
+
+```ruby
+# Single column
+User.where("age > 21").pluck_in_batches(:email) do |emails|
+  jobs = emails.map { |email| PartyReminderJob.new(email) }
+  ActiveJob.perform_all_later(jobs)
+end
+
+# Multiple columns
+User.pluck_in_batches(:name, :email).with_index do |group, index|
+  puts "Processing group ##{index}"
+  jobs = group.map { |name, email| PartyReminderJob.new(name, email) }
+  ActiveJob.perform_all_later(jobs)
+end
+```
+
+Both methods support the following configuration options:
+
+* `:batch_size` - Specifies the size of the batch. Defaults to 1000.
+* `:start` - Specifies the primary key value to start from, inclusive of the value.
+* `:finish` - Specifies the primary key value to end at, inclusive of the value.
+* `:error_on_ignore` - Overrides the application config to specify if an error should be raised when
+  an order is present in the relation.
+* `:cursor_column` - Specifies the column(s) on which the iteration should be done.
+  This column(s) should be orderable (e.g. an integer or string). Defaults to primary key.
+* `:order` - Specifies the primary key order (can be `:asc` or `:desc`). Defaults to `:asc`.
+
+## Development
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/fatkodima/pluck_in_batches.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/pluck_in_batches/extensions.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module PluckInBatches
+  module Extensions
+    module ModelExtension
+      delegate :pluck_each, :pluck_in_batches, to: :all
+    end
+
+    module RelationExtension
+      # Yields each set of values corresponding to the specified columns that was found
+      # by the passed options. If one column specified - returns its value, if an array of columns -
+      # returns an array of values.
+      #
+      # See #pluck_in_batches for all the details.
+      #
+      def pluck_each(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: :asc, cursor_column: primary_key, &block)
+        iterator = Iterator.new(self)
+        iterator.each(*columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order, &block)
+      end
+
+      # Yields each batch of values corresponding to the specified columns that was found
+      # by the passed options as an array.
+      #
+      #   User.where("age > 21").pluck_in_batches(:email) do |emails|
+      #     jobs = emails.map { |email| PartyReminderJob.new(email) }
+      #     ActiveJob.perform_all_later(jobs)
+      #   end
+      #
+      # If you do not provide a block to #pluck_in_batches, it will return an Enumerator
+      # for chaining with other methods:
+      #
+      #   User.pluck_in_batches(:name, :email).with_index do |group, index|
+      #     puts "Processing group ##{index}"
+      #     jobs = group.map { |name, email| PartyReminderJob.new(name, email) }
+      #     ActiveJob.perform_all_later(jobs)
+      #   end
+      #
+      # ==== Options
+      # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
+      # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
+      # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
+      # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
+      #   an order is present in the relation.
+      # * <tt>:cursor_column</tt> - Specifies the column(s) on which the iteration should be done.
+      #   This column(s) should be orderable (e.g. an integer or string). Defaults to primary key.
+      # * <tt>:order</tt> - Specifies the cursor column(s) order (can be +:asc+ or +:desc+). Defaults to +:asc+.
+      #
+      # Limits are honored, and if present there is no requirement for the batch
+      # size: it can be less than, equal to, or greater than the limit.
+      #
+      # The options +start+ and +finish+ are especially useful if you want
+      # multiple workers dealing with the same processing queue. You can make
+      # worker 1 handle all the records between id 1 and 9999 and worker 2
+      # handle from 10000 and beyond by setting the +:start+ and +:finish+
+      # option on each worker.
+      #
+      #   # Let's process from record 10_000 on.
+      #   User.pluck_in_batches(:email, start: 10_000) do |emails|
+      #     jobs = emails.map { |email| PartyReminderJob.new(email) }
+      #     ActiveJob.perform_all_later(jobs)
+      #   end
+      #
+      # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
+      # ascending on the primary key ("id ASC").
+      # This also means that this method only works when the primary key is
+      # orderable (e.g. an integer or string).
+      #
+      # NOTE: By its nature, batch processing is subject to race conditions if
+      # other processes are modifying the database.
+      #
+      def pluck_in_batches(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: primary_key, order: :asc, &block)
+        iterator = Iterator.new(self)
+        iterator.each_batch(*columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order, &block)
+      end
+    end
+  end
+end

data/lib/pluck_in_batches/iterator.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module PluckInBatches
+  class Iterator # :nodoc:
+    def initialize(relation)
+      @relation = relation
+      @klass = relation.klass
+    end
+
+    def each(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: @relation.primary_key, order: :asc, &block)
+      if columns.empty?
+        raise ArgumentError, "Call `pluck_each' with at least one column."
+      end
+
+      if block_given?
+        each_batch(*columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order) do |batch|
+          batch.each(&block)
+        end
+      else
+        enum_for(__callee__, *columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order) do
+          apply_limits(@relation, start, finish, order).size
+        end
+      end
+    end
+
+    def each_batch(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: @relation.primary_key, order: :asc)
+      if columns.empty?
+        raise ArgumentError, "Call `pluck_in_batches' with at least one column."
+      end
+
+      unless order == :asc || order == :desc
+        raise ArgumentError, ":order must be :asc or :desc, got #{order.inspect}"
+      end
+
+      pluck_columns = columns.map(&:to_s)
+      cursor_columns = Array(cursor_column).map(&:to_s)
+      cursor_column_indexes = cursor_column_indexes(pluck_columns, cursor_columns)
+      missing_cursor_columns = cursor_column_indexes.count(&:nil?)
+      cursor_column_indexes.each_with_index do |column_index, index|
+        unless column_index
+          cursor_column_indexes[index] = pluck_columns.size
+          pluck_columns << cursor_columns[index]
+        end
+      end
+
+      relation = @relation
+
+      unless block_given?
+        return to_enum(__callee__, *columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order) do
+          total = apply_limits(relation, cursor_columns, start, finish, order).size
+          (total - 1).div(batch_size) + 1
+        end
+      end
+
+      if relation.arel.orders.present?
+        act_on_ignored_order(error_on_ignore)
+      end
+
+      batch_limit = batch_size
+      if relation.limit_value
+        remaining   = relation.limit_value
+        batch_limit = remaining if remaining < batch_limit
+      end
+
+      relation = relation.reorder(*batch_order(cursor_columns, order)).limit(batch_limit)
+      relation = apply_limits(relation, cursor_columns, start, finish, order)
+      relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
+      batch_relation = relation
+
+      loop do
+        batch = batch_relation.pluck(*pluck_columns)
+        break if batch.empty?
+
+        cursor_column_offsets =
+          if pluck_columns.size == 1
+            Array(batch.last)
+          else
+            cursor_column_indexes.map.with_index do |column_index, index|
+              batch.last[column_index || (batch.last.size - cursor_column_indexes.size + index)]
+            end
+          end
+
+        missing_cursor_columns.times { batch.each(&:pop) }
+        batch.flatten!(1) if columns.size == 1
+
+        yield batch
+
+        break if batch.length < batch_limit
+
+        if @relation.limit_value
+          remaining -= batch.length
+
+          if remaining == 0
+            # Saves a useless iteration when the limit is a multiple of the
+            # batch size.
+            break
+          elsif remaining < batch_limit
+            relation = relation.limit(remaining)
+          end
+        end
+
+        batch_relation = batch_condition(
+          relation, cursor_columns, cursor_column_offsets, order == :desc ? :lt : :gt
+        )
+      end
+    end
+
+    private
+      def cursor_column_indexes(columns, cursor_column)
+        cursor_column.map do |column|
+          columns.index(column) ||
+            columns.index("#{@klass.table_name}.#{column}") ||
+            columns.index("#{@klass.quoted_table_name}.#{@klass.connection.quote_column_name(column)}")
+        end
+      end
+
+      def act_on_ignored_order(error_on_ignore)
+        raise_error =
+          if error_on_ignore.nil?
+            if ar_version >= 7.0
+              ActiveRecord.error_on_ignored_order
+            else
+              @klass.error_on_ignored_order
+            end
+          else
+            error_on_ignore
+          end
+
+        message = "Scoped order is ignored, it's forced to be batch order."
+
+        if raise_error
+          raise ArgumentError, message
+        elsif (logger = ActiveRecord::Base.logger)
+          logger.warn(message)
+        end
+      end
+
+      def apply_limits(relation, columns, start, finish, order)
+        relation = apply_start_limit(relation, columns, start, order) if start
+        relation = apply_finish_limit(relation, columns, finish, order) if finish
+        relation
+      end
+
+      def apply_start_limit(relation, columns, start, order)
+        batch_condition(relation, columns, start, order == :desc ? :lteq : :gteq)
+      end
+
+      def apply_finish_limit(relation, columns, finish, order)
+        batch_condition(relation, columns, finish, order == :desc ? :gteq : :lteq)
+      end
+
+      def batch_condition(relation, columns, values, operator)
+        columns = Array(columns)
+        values = Array(values)
+        cursor_positions = columns.zip(values)
+
+        first_clause_column, first_clause_value = cursor_positions.pop
+        where_clause = build_attribute_predicate(first_clause_column, first_clause_value, operator)
+
+        cursor_positions.reverse_each do |column_name, value|
+          where_clause = build_attribute_predicate(column_name, value, operator == :lteq ? :lt : :gt).or(
+            build_attribute_predicate(column_name, value, :eq).and(where_clause)
+          )
+        end
+
+        relation.where(where_clause)
+      end
+
+      def build_attribute_predicate(column, value, operator)
+        @relation.bind_attribute(column, value) { |attr, bind| attr.public_send(operator, bind) }
+      end
+
+      def batch_order(cursor_columns, order)
+        cursor_columns.map do |column|
+          @relation.arel_table[column].public_send(order)
+        end
+      end
+
+      def ar_version
+        ActiveRecord.version.to_s.to_f
+      end
+  end
+end

data/lib/pluck_in_batches/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module PluckInBatches
+  VERSION = "0.1.0"
+end

data/lib/pluck_in_batches.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+require_relative "pluck_in_batches/iterator"
+require_relative "pluck_in_batches/extensions"
+require_relative "pluck_in_batches/version"
+
+module PluckInBatches
+end
+
+ActiveSupport.on_load(:active_record) do
+  extend(PluckInBatches::Extensions::ModelExtension)
+  ActiveRecord::Relation.include(PluckInBatches::Extensions::RelationExtension)
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: pluck_in_batches
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- fatkodima
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-05-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+description:
+email:
+- fatkodima123@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/pluck_in_batches.rb
+- lib/pluck_in_batches/extensions.rb
+- lib/pluck_in_batches/iterator.rb
+- lib/pluck_in_batches/version.rb
+homepage: https://github.com/fatkodima/pluck_in_batches
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/fatkodima/pluck_in_batches
+  source_code_uri: https://github.com/fatkodima/pluck_in_batches
+  changelog_uri: https://github.com/fatkodima/pluck_in_batches/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.12
+signing_key:
+specification_version: 4
+summary: Change
+test_files: []