chewy 7.2.0 → 7.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a958ecfd3273f1e8086b3c07aa823f3db8e5d2300a0ca2a86a563e1d6bd2df42
-  data.tar.gz: 8195d2162307dbf368865cc5e5975372e169796aa85a5a6bd8ac404484b99d75
+  metadata.gz: 70c6d061a7070669eafe4f28bd09cd8e7a9138d0a66dd6fdbdf1f387f3f3f0af
+  data.tar.gz: '03838f9c5e5b323e68482066b057bccff969e2826e581b994b92a962bb45e272'
 SHA512:
-  metadata.gz: 7b3286e50679ce8b43c9a595d90502cbd028ac8e94c93e240a3f820de15fbc19052887f1bf973670421bcc8a2e1cfe9dd4d9b2183cb0ccd828726220df005c7c
-  data.tar.gz: 00a24e51c43cd0e191af699949ca3ac1ca738777fd3a22dfe1d97d7d55feb6ebfea29e2577d488ff1f97189e1430266110395afea3efcb949917d07a6c26477f
+  metadata.gz: 8fcafb7b237abf426ed941e6b8d53b537db65186ce98f40b6c1b50257bc570b500b9bbff71aedb6828dfb0be11e13f031f3cf92413fd0f3efb5b860deca53a28
+  data.tar.gz: c74fe5e6db07346b9e77f4a0d294317a6ade800fa506f231d7368058a6f058785bb0c9756110378b810398a7bc24ca5c695bbb46c330a97976fd3b1bf6a8d3da

data/CHANGELOG.md

@@ -8,6 +8,25 @@
 
 ### Bugs Fixed
 
+## 7.2.1 (2021-05-11)
+
+### New Features
+
+  * [#469](https://github.com/toptal/chewy/issues/469): Add ability to output the progressbar with `ENV['PROGRESS']` during `reset` rake tasks ([@Vitalina-Vakulchyk][]):
+    * for `rake chewy:reset` and `rake chewy:parallel:reset`
+    * progressbar is hidden by default, set `ENV['PROGRESS']` to `true` to display it
+
+### Bugs Fixed
+
+  * [#796](https://github.com/toptal/chewy/pull/796): Fix clear scroll: pass `scroll_id` in body, as passing in path parameters is deprecated and can overflow `http.max_initial_line_length` ([@rabotyaga][])
+
+## 7.0.1 (2021-05-03)
+
+### Changes
+
+  * [#792](https://github.com/toptal/chewy/pull/792): Skip ES version memoization for search requests ([@rabotyaga][])
+    * See the Migration Guide for details
+
 ## 7.2.0 (2021-04-19)
 
 ### New Features

data/README.md

@@ -57,7 +57,12 @@ Chewy is compatible with MRI 2.6-3.0¹.
 | 6.0.0         | 5.x, 6.x                           |
 | 5.x           | 5.x, limited support for 1.x & 2.x |
 
-See [Migration guide](migration_guide.md).
+**Important:** Chewy doesn't follow SemVer, so you should always
+check the release notes before upgrading. The major version is linked to the
+newest supported Elasticsearch and the minor version bumps may include breaking changes.
+
+See our [migration guide](migration_guide.md) for detailed upgrade instructions between
+various Chewy versions.
 
 ### Active Record
 
@@ -97,7 +102,7 @@ $ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.t
 
 ### Index
 
-Create `app/chewy/user_index.rb` with User Index:
+Create `app/chewy/users_index.rb` with User Index:
 
 ```ruby
 class UsersIndex < Chewy::Index
@@ -1027,6 +1032,12 @@ rake chewy:reset[users,cities] # resets UsersIndex and CitiesIndex
 rake chewy:reset[-users,cities] # resets every index in the application except specified ones
 ```
 
+#### Progressbar for `chewy:reset` tasks
+
+You can optionally output the `progressbar` for `chewy:reset` and `chewy:parallel:reset` during import.
+
+Progressbar is hidden by default. Set `ENV['PROGRESS']` to `true` to display it.
+
 #### `chewy:upgrade`
 
 Performs reset exactly the same way as `chewy:reset` does, but only when the index specification (setting or mapping) was changed.

data/chewy.gemspec

@@ -33,4 +33,5 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
   spec.add_dependency 'activesupport', '>= 5.2'
   spec.add_dependency 'elasticsearch', '>= 6.3.0'
   spec.add_dependency 'elasticsearch-dsl'
+  spec.add_dependency 'ruby-progressbar'
 end

data/lib/chewy/index/actions.rb

@@ -146,7 +146,7 @@ module Chewy
         # @param journal [true, false] journaling is switched off for import during reset by default
         # @param import_options [Hash] options, passed to the import call
         # @return [true, false] false in case of errors
-        def reset!(suffix = nil, apply_journal: true, journal: false, **import_options)
+        def reset!(suffix = nil, apply_journal: true, journal: false, progressbar: false, **import_options)
           result = if suffix.present?
             start_time = Time.now
             indexes = self.indexes - [index_name]
@@ -159,17 +159,13 @@ module Chewy
             result = import import_options.merge(
               suffix: suffix,
               journal: journal,
-              refresh: !Chewy.reset_disable_refresh_interval
+              refresh: !Chewy.reset_disable_refresh_interval,
+              progressbar: progressbar
             )
             original_index_settings suffixed_name
 
             delete if indexes.blank?
-            client.indices.update_aliases body: {actions: [
-              *indexes.map do |index|
-                {remove: {index: index, alias: general_name}}
-              end,
-              {add: {index: suffixed_name, alias: general_name}}
-            ]}
+            update_aliases(indexes, general_name, suffixed_name)
             client.indices.delete index: indexes if indexes.present?
 
             self.journal.apply(start_time, **import_options) if apply_journal
@@ -237,6 +233,15 @@ module Chewy
 
       private
 
+        def update_aliases(indexes, general_name, suffixed_name)
+          client.indices.update_aliases body: {actions: [
+            *indexes.map do |index|
+              {remove: {index: index, alias: general_name}}
+            end,
+            {add: {index: suffixed_name, alias: general_name}}
+          ]}
+        end
+
         def optimize_index_settings(index_name)
           settings = {}
           settings[:refresh_interval] = -1 if Chewy.reset_disable_refresh_interval

data/lib/chewy/index/adapter/object.rb

@@ -192,6 +192,16 @@ module Chewy
           end
         end
 
+        def import_count(*args)
+          collection = if args.first.empty? && @target.respond_to?(import_all_method)
+            @target.send(import_all_method)
+          else
+            args.flatten(1).compact
+          end
+
+          collection.count
+        end
+
       private
 
         def import_objects(objects, options)

data/lib/chewy/index/adapter/orm.rb

@@ -108,6 +108,18 @@ module Chewy
           ids.map { |id| loaded_objects[id.to_s] }
         end
 
+        def import_count(*args)
+          collection = if args.first.empty?
+            default_scope
+          elsif args.first.is_a?(relation_class)
+            args.first
+          else
+            args.flatten.compact
+          end
+
+          collection.count
+        end
+
       private
 
         def import_objects(collection, options)

data/lib/chewy/index/import.rb

@@ -2,17 +2,19 @@ require 'chewy/index/import/journal_builder'
 require 'chewy/index/import/bulk_builder'
 require 'chewy/index/import/bulk_request'
 require 'chewy/index/import/routine'
+require 'chewy/index/import/thread_safe_progress_bar'
 
 module Chewy
   class Index
     module Import
       extend ActiveSupport::Concern
 
-      IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
+      IMPORT_WORKER = lambda do |index, options, total, progress_bar, ids, iteration|
         ::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
         routine = Routine.new(index, **options)
         index.adapter.import(*ids, routine.options) do |action_objects|
           routine.process(**action_objects)
+          progress_bar.increment(action_objects.map { |_, v| v.size }.sum) if routine.options[:progressbar]
         end
         {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
       end
@@ -150,8 +152,10 @@ module Chewy
 
         def import_linear(objects, routine)
           ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
+            progress_bar = ThreadSafeProgressBar.new(routine.options[:progressbar]) { adapter.import_count(objects) }
             adapter.import(*objects, routine.options) do |action_objects|
               routine.process(**action_objects)
+              progress_bar.increment(action_objects.map { |_, v| v.size }.sum) if routine.options[:progressbar]
             end
             routine.perform_bulk(routine.leftovers)
             payload[:import] = routine.stats
@@ -166,24 +170,20 @@ module Chewy
           ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
             batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
 
+            progress_bar = ThreadSafeProgressBar.new(routine.options[:progressbar]) { adapter.import_count(objects) }
             ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
-            results = ::Parallel.map_with_index(
-              batches,
-              routine.parallel_options,
-              &IMPORT_WORKER.curry[self, routine.options, batches.size]
-            )
+
+            results = ::Parallel.map_with_index(batches, routine.parallel_options) do |ids, index|
+              progress_bar.wait_until_ready
+              ActiveRecord::Base.connection_pool.with_connection do
+                IMPORT_WORKER.call(self, routine.options, total, progress_bar, ids, index)
+              end
+            end
+
             ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
             errors, import, leftovers = process_parallel_import_results(results)
 
-            if leftovers.present?
-              batches = leftovers.each_slice(routine.options[:batch_size])
-              results = ::Parallel.map_with_index(
-                batches,
-                routine.parallel_options,
-                &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
-              )
-              errors.concat(results.flatten(1))
-            end
+            execute_leftovers(leftovers, routine, self, errors)
 
             payload[:import] = import
             payload[:errors] = payload_errors(errors) if errors.present?
@@ -191,6 +191,18 @@ module Chewy
           end
         end
 
+        def execute_leftovers(leftovers, routine, self_object, errors)
+          return unless leftovers.present?
+
+          batches = leftovers.each_slice(routine.options[:batch_size])
+          results = ::Parallel.map_with_index(
+            batches,
+            routine.parallel_options,
+            &LEFTOVERS_WORKER.curry[self_object, routine.options, batches.size]
+          )
+          errors.concat(results.flatten(1))
+        end
+
         def process_parallel_import_results(results)
           results.each_with_object([[], {}, []]) do |r, (e, i, l)|
             e.concat(r[:errors])

data/lib/chewy/index/import/routine.rb

@@ -51,9 +51,9 @@ module Chewy
           @parallel_options = @options.delete(:parallel)
           if @parallel_options && !@parallel_options.is_a?(Hash)
             @parallel_options = if @parallel_options.is_a?(Integer)
-              {in_processes: @parallel_options}
+              {in_threads: @parallel_options}
             else
-              {}
+              {in_threads: [::Parallel.processor_count, ActiveRecord::Base.connection_pool.size].min}
             end
           end
           @errors = []

data/lib/chewy/index/import/thread_safe_progress_bar.rb

@@ -0,0 +1,40 @@
+module Chewy
+  class Index
+    module Import
+      # This class performs the threading for parallel import to avoid concurrency during progressbar output.
+      #
+      # @see Chewy::Type::Import::ClassMethods#import with `parallel: true` option
+      class ThreadSafeProgressBar
+        def initialize(enabled)
+          @enabled = enabled
+
+          return unless @enabled
+
+          @mutex = Mutex.new
+          @released = false
+          @progressbar = ProgressBar.create total: nil
+          Thread.new do
+            ActiveRecord::Base.connection_pool.with_connection do
+              @mutex.synchronize { @released = true }
+              @progressbar.total = yield
+            end
+          end
+        end
+
+        def increment(value)
+          return unless @enabled
+
+          @mutex.synchronize do
+            @progressbar.progress += value
+          end
+        end
+
+        def wait_until_ready
+          return true unless @enabled
+
+          @mutex.synchronize { @released } until @released
+        end
+      end
+    end
+  end
+end

data/lib/chewy/rake_helper.rb

@@ -271,7 +271,7 @@ module Chewy
 
       def reset_one(index, output, parallel: false)
         output.puts "Resetting #{index}"
-        index.reset!((Time.now.to_f * 1000).round, parallel: parallel)
+        index.reset!((Time.now.to_f * 1000).round, parallel: parallel, progressbar: ENV['PROGRESS'] == 'true')
       end
     end
   end

data/lib/chewy/search/scrolling.rb

@@ -44,7 +44,7 @@ module Chewy
           result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
         end
       ensure
-        Chewy.client.clear_scroll(scroll_id: scroll_id) if scroll_id
+        Chewy.client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
       end
 
       # @!method scroll_hits(batch_size: 1000, scroll: '1m')

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.2.0'.freeze
+  VERSION = '7.2.1'.freeze
 end

data/migration_guide.md

@@ -13,6 +13,11 @@ In order to upgrade Chewy 6/Elasticsearch 6 to Chewy 7/Elasticsearch 7 in the mo
 * Run your test suite on Chewy 7.0 / Elasticsearch 7
 * Run manual tests on Chewy 7.0 / Elasticsearch 7
 * Upgrade to Chewy 7.0
+  * The “total hits” counter is an integer for ES versions < 7 and an object (hash) for the versions starting from 7.0.0. Elasticsearch added a special option, `rest_total_hits_as_int`, to ease the upgrade, that could be appended to any request and results in the old “total hits” format. Unfortunately, this option is not recognized by ES versions prior to 7.0.0, which means that we have to check the version to decide if we need this option.
+    Normally Chewy does memoization of the current ES version, but this might be inappropriate for the upgrade, as the version changes live.
+    To handle that we have 2 versions of Chewy for this stage of the upgrade: 7.0.0 and 7.0.1. Version 7.0.0 does the memoization and version 7.0.1 requests the current version on every search request. 
+  * You can use the 7.0.0 version if it's fine for you to have an application restart immediately after ES cluster upgrade.
+  * If you're using the 7.0.1 version you might be interested in keeping the timeframe between this step and updating to Chewy 7.1 as small as possible, as version 7.0.1 skips ES version memoization for search requests to help dynamically detect ES version. This leads to an extra version request on each search request, i.e. could affect the overall performance/latency of the search and a load of ES cluster.
 * Perform a [rolling upgrade](https://www.elastic.co/guide/en/elasticsearch/reference//rolling-upgrades.html) of Elasticsearch
 * Run your test suite on Chewy 7.1 / Elasticsearch 7
 * Run manual tests on Chewy 7.1 / Elasticsearch 7

data/spec/chewy/index/actions_spec.rb

@@ -502,7 +502,7 @@ describe Chewy::Index::Actions do
         specify do
           expect(CitiesIndex.client.indices).to receive(:put_settings).with(index: name, body: before_import_body).once
           expect(CitiesIndex.client.indices).to receive(:put_settings).with(index: name, body: after_import_body).once
-          expect(CitiesIndex).to receive(:import).with(suffix: suffix, journal: false, refresh: false).and_call_original
+          expect(CitiesIndex).to receive(:import).with(suffix: suffix, progressbar: false, journal: false, refresh: false).and_call_original
           expect(CitiesIndex.reset!(suffix)).to eq(true)
         end
 
@@ -525,7 +525,7 @@ describe Chewy::Index::Actions do
               .to receive(:put_settings).with(index: name, body: before_import_body).once
             expect(CitiesIndex.client.indices).to receive(:put_settings).with(index: name, body: after_import_body).once
             expect(CitiesIndex)
-              .to receive(:import).with(suffix: suffix, journal: false, refresh: false).and_call_original
+              .to receive(:import).with(suffix: suffix, progressbar: false, journal: false, refresh: false).and_call_original
             expect(CitiesIndex.reset!(suffix)).to eq(true)
           end
 
@@ -541,7 +541,7 @@ describe Chewy::Index::Actions do
         let(:reset_disable_refresh_interval) { false }
         specify do
           expect(CitiesIndex.client.indices).not_to receive(:put_settings)
-          expect(CitiesIndex).to receive(:import).with(suffix: suffix, journal: false, refresh: true).and_call_original
+          expect(CitiesIndex).to receive(:import).with(suffix: suffix, progressbar: false, journal: false, refresh: true).and_call_original
           expect(CitiesIndex.reset!(suffix)).to eq(true)
         end
       end
@@ -568,7 +568,7 @@ describe Chewy::Index::Actions do
         specify do
           expect(CitiesIndex.client.indices).to receive(:put_settings).with(index: name, body: before_import_body).once
           expect(CitiesIndex.client.indices).to receive(:put_settings).with(index: name, body: after_import_body).once
-          expect(CitiesIndex).to receive(:import).with(suffix: suffix, journal: false, refresh: true).and_call_original
+          expect(CitiesIndex).to receive(:import).with(suffix: suffix, progressbar: false, journal: false, refresh: true).and_call_original
           expect(CitiesIndex.reset!(suffix)).to eq(true)
         end
       end
@@ -577,7 +577,7 @@ describe Chewy::Index::Actions do
         let(:reset_no_replicas) { false }
         specify do
           expect(CitiesIndex.client.indices).not_to receive(:put_settings)
-          expect(CitiesIndex).to receive(:import).with(suffix: suffix, journal: false, refresh: true).and_call_original
+          expect(CitiesIndex).to receive(:import).with(suffix: suffix, progressbar: false, journal: false, refresh: true).and_call_original
           expect(CitiesIndex.reset!(suffix)).to eq(true)
         end
       end
@@ -667,7 +667,7 @@ describe Chewy::Index::Actions do
       specify do
         expect(CitiesIndex)
           .to receive(:import)
-          .with(suffix: 'suffix', parallel: true, journal: false, refresh: true)
+          .with(suffix: 'suffix', progressbar: false, parallel: true, journal: false, refresh: true)
           .once.and_return(true)
         expect(CitiesIndex.reset!('suffix', parallel: true)).to eq(true)
       end

data/spec/chewy/index/import/routine_spec.rb

@@ -64,8 +64,8 @@ describe Chewy::Index::Import::Routine do
 
   describe '#parallel_options' do
     specify { expect(described_class.new(CitiesIndex).parallel_options).to be_nil }
-    specify { expect(described_class.new(CitiesIndex, parallel: true).parallel_options).to eq({}) }
-    specify { expect(described_class.new(CitiesIndex, parallel: 3).parallel_options).to eq(in_processes: 3) }
+    specify { expect(described_class.new(CitiesIndex, parallel: true).parallel_options).to eq({in_threads: [::Parallel.processor_count, ActiveRecord::Base.connection_pool.size].min}) }
+    specify { expect(described_class.new(CitiesIndex, parallel: 3).parallel_options).to eq(in_threads: 3) }
     specify do
       expect(described_class.new(CitiesIndex, parallel: {in_threads: 2}).parallel_options).to eq(in_threads: 2)
     end

data/spec/chewy/index/import_spec.rb

@@ -493,6 +493,35 @@ describe Chewy::Index::Import do
 
       it_behaves_like 'importing'
     end
+
+    context 'with progressbar output' do
+      let(:mocked_progressbar) { Struct.new(:progress, :total).new(0, 100) }
+
+      it 'imports tracks progress in a single batch' do
+        expect(ProgressBar).to receive(:create).and_return(mocked_progressbar)
+        expect(mocked_progressbar).to receive(:progress).at_least(:once).and_call_original
+        expect(CitiesIndex).to receive(:import_parallel).and_call_original
+
+        CitiesIndex.import(parallel: 1, progressbar: true)
+
+        expect(mocked_progressbar.progress).to eq(3)
+        expect(mocked_progressbar.total).to eq(3)
+      end
+
+      it 'imports tracks progress in many batches' do
+        expect(ProgressBar).to receive(:create).and_return(mocked_progressbar)
+        expect(mocked_progressbar).to receive(:progress).at_least(:once).and_call_original
+        expect(CitiesIndex).to receive(:import_parallel).and_call_original
+
+        batches = City.pluck(:id).map { |id| [id] }
+        expect(CitiesIndex.adapter).to receive(:import_references).and_return(batches)
+
+        CitiesIndex.import(parallel: 3, progressbar: true)
+
+        expect(mocked_progressbar.progress).to eq(3)
+        expect(mocked_progressbar.total).to eq(3)
+      end
+    end
   end
 
   describe '.import!', :orm do
@@ -506,7 +535,9 @@ describe Chewy::Index::Import do
         end
       end
 
-      specify { expect { CitiesIndex.import!(dummy_cities) }.to raise_error Chewy::ImportFailed }
+      specify do
+        expect { CitiesIndex.import!(dummy_cities) }.to raise_error Chewy::ImportFailed
+      end
     end
   end
 

data/spec/chewy/search/scrolling_spec.rb

@@ -106,7 +106,7 @@ describe Chewy::Search::Scrolling, :orm do
       end
 
       it 'clears the scroll after completion' do
-        expect(Chewy.client).to receive(:clear_scroll).with(scroll_id: anything).once.and_call_original
+        expect(Chewy.client).to receive(:clear_scroll).with(body: {scroll_id: anything}).once.and_call_original
         request.scroll_batches(batch_size: 3) {}
       end
 

data/spec/spec_helper.rb

@@ -8,6 +8,7 @@ require 'rspec/its'
 require 'rspec/collection_matchers'
 
 require 'timecop'
+require 'ruby-progressbar'
 
 Kaminari::Hooks.init if defined?(::Kaminari::Hooks)
 

data/spec/support/active_record.rb

@@ -1,6 +1,6 @@
 require 'database_cleaner'
 
-ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: 'file::memory:?cache=shared', pool: 10)
+ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: 'file::memory:?cache=shared', pool: 1)
 ActiveRecord::Base.logger = Logger.new('/dev/null')
 if ActiveRecord::Base.respond_to?(:raise_in_transactional_callbacks)
   ActiveRecord::Base.raise_in_transactional_callbacks = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 7.2.0
+  version: 7.2.1
 platform: ruby
 authors:
 - Toptal, LLC
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-19 00:00:00.000000000 Z
+date: 2021-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: database_cleaner
@@ -207,6 +207,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Chewy provides functionality for Elasticsearch index handling, documents
   import mappings and chainable query DSL
 email:
@@ -258,6 +272,7 @@ files:
 - lib/chewy/index/import/bulk_request.rb
 - lib/chewy/index/import/journal_builder.rb
 - lib/chewy/index/import/routine.rb
+- lib/chewy/index/import/thread_safe_progress_bar.rb
 - lib/chewy/index/mapping.rb
 - lib/chewy/index/observe.rb
 - lib/chewy/index/settings.rb