chewy 7.2.6 → 7.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96329a4913f6308f4c7435cd50b962a36dc63b728b50567ff53ce2cb8011086f
-  data.tar.gz: a25b93955769a85fdf5de68a6d284060a981c2e3f6e10c929b0cd49816c19720
+  metadata.gz: f720ba38bdda37de9d1a9b54d0b9c28efbe444e3c9cdfb8e3a1fef0f01aeaf85
+  data.tar.gz: 01041cfe59a33c5b9b07bc46f4e5c0d9a68802a7ddf6ad99aebf7a34c85b32b2
 SHA512:
-  metadata.gz: 66a3dfa94536e8d7b262a6a0ccfe32ff76ca7431850722eef21b14a11b3d162ad0346c3a469293696ce79916c44db7fb17b56f462fd4083c619b2dde09fa6c29
-  data.tar.gz: 6e15e9fcb103c64a5565add1e5200caa4bf1018d3fc7e9ac79a8db6829a5bb1699ac77910e838d59df2f6f195d019a88c22987d51f1b02ebdbb8f37844cce8d3
+  metadata.gz: 01567be06ff1aa7deb0cd9e2459c9a094d80c113e088e1eac7dbdea0cf6b74bbb78579923bf7335bb2114884a3cb3a99ca946009fe3f762566b76a82c17ea7f2
+  data.tar.gz: 0cd235971d30435c1d5ad2b33f70643733fb5af02d7d49891668d216396fdaa0326b7aad4c01bcbf3ed207ef3ee9ac37c02ff269087b9bb6c8e498304dd902d9

data/.github/workflows/ruby.yml

@@ -1,6 +1,13 @@
 name: CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [master]
+  pull_request:
+    types: [
+      synchronize, # PR was updated
+      opened # PR was open
+    ]
 
 jobs:
   ruby-2:

data/CHANGELOG.md

@@ -8,6 +8,18 @@
 
 ### Bugs Fixed
 
+## 7.2.7 (2022-11-15)
+
+### New Features
+
+* [#857](https://github.com/toptal/chewy/pull/857): Allow passing `wait_for_completion`, `request_per_second` and `scroll_size` options to `chewy:journal:clean` rake task and `delete_all` query builder method. ([@konalegi][])([@barthez][])
+
+### Changes
+
+### Bugs Fixed
+
+* [#863](https://github.com/toptal/chewy/pull/863): Fix `crutches` call doesn't respect `update_fields` option. ([@skcc321][])
+
 ## 7.2.6 (2022-06-13)
 
 ### New Features

data/README.md

@@ -677,6 +677,8 @@ You may be wondering why do you need it? The answer is simple: not to lose the d
 
 Imagine that you reset your index in a zero-downtime manner (to separate index), and at the meantime somebody keeps updating the data frequently (to old index). So all these actions will be written to the journal index and you'll be able to apply them after index reset using the `Chewy::Journal` interface.
 
+When enabled, journal can grow to enormous size, consider setting up cron job that would clean it occasionally using [`chewy:journal:clean` rake task](#chewyjournal).
+
 ### Index manipulation
 
 ```ruby
@@ -1144,6 +1146,17 @@ rake chewy:journal:apply["$(date -v-1H -u +%FT%TZ)"] # apply journaled changes f
 rake chewy:journal:apply["$(date -v-1H -u +%FT%TZ)",users] # apply journaled changes for the past hour on UsersIndex only
 ```
 
+When the size of the journal becomes very large, the classical way of deletion would be obstructive and resource consuming. Fortunately, Chewy internally uses [delete-by-query](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/docs-delete-by-query.html#docs-delete-by-query-task-api) ES function which supports async execution with batching and [throttling](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html#docs-delete-by-query-throttle).
+
+The available options, which can be set by ENV variables, are listed below:
+* `WAIT_FOR_COMPLETION` - a boolean flag. It controls async execution. It waits by default. When set to `false` (`0`, `f`, `false` or `off` in any case spelling is accepted as `false`), Elasticsearch performs some preflight checks, launches the request, and returns a task reference you can use to cancel the task or get its status.
+* `REQUESTS_PER_SECOND` - float. The throttle for this request in sub-requests per second. No throttling is enforced by default.
+* `SCROLL_SIZE` - integer. The number of documents to be deleted in single sub-request. The default batch size is 1000.
+
+```bash
+rake chewy:journal:clean WAIT_FOR_COMPLETION=false REQUESTS_PER_SECOND=10 SCROLL_SIZE=5000
+```
+
 ### RSpec integration
 
 Just add `require 'chewy/rspec'` to your spec_helper.rb and you will get additional features:

data/lib/chewy/index/import/bulk_builder.rb

@@ -48,12 +48,11 @@ module Chewy
         def index_entry(object)
           entry = {}
           entry[:_id] = index_object_ids[object] if index_object_ids[object]
+          entry[:routing] = routing(object) if join_field?
 
-          data = data_for(object)
           parent = cache(entry[:_id])
-
-          entry[:routing] = routing(object) if join_field?
-          if parent_changed?(data, parent)
+          data = data_for(object) if parent.present?
+          if parent.present? && parent_changed?(data, parent)
             reindex_entries(object, data) + reindex_descendants(object)
           elsif @fields.present?
             return [] unless entry[:_id]
@@ -61,7 +60,7 @@ module Chewy
             entry[:data] = {doc: data_for(object, fields: @fields)}
             [{update: entry}]
           else
-            entry[:data] = data
+            entry[:data] = data || data_for(object)
             [{index: entry}]
           end
         end

data/lib/chewy/journal.rb

@@ -43,8 +43,12 @@ module Chewy
     #
     # @param until_time [Time, DateTime] time to clean up until it
     # @return [Hash] delete_by_query ES API call result
-    def clean(until_time = nil)
-      Chewy::Stash::Journal.clean(until_time, only: @only)
+    def clean(until_time = nil, delete_by_query_options: {})
+      Chewy::Stash::Journal.clean(
+        until_time,
+        only: @only,
+        delete_by_query_options: delete_by_query_options.merge(refresh: false)
+      )
     end
 
   private

data/lib/chewy/rake_helper.rb

@@ -19,6 +19,9 @@ module Chewy
       output.puts "  Applying journal to #{targets}, #{count} entries, stage #{payload[:stage]}"
     end
 
+    DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
+    FALSE_VALUES = %w[0 f false off].freeze
+
     class << self
       # Performs zero-downtime reindexing of all documents for the specified indexes
       #
@@ -162,7 +165,7 @@ module Chewy
 
         subscribed_task_stats(output) do
           output.puts "Applying journal entries created after #{time}"
-          count = Chewy::Journal.new(indexes_from(only: only, except: except)).apply(time)
+          count = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).apply(time)
           output.puts 'No journal entries were created after the specified time' if count.zero?
         end
       end
@@ -181,12 +184,16 @@ module Chewy
       # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param output [IO] output io for logging
       # @return [Array<Chewy::Index>] indexes that were actually updated
-      def journal_clean(time: nil, only: nil, except: nil, output: $stdout)
+      def journal_clean(time: nil, only: nil, except: nil, delete_by_query_options: {}, output: $stdout)
         subscribed_task_stats(output) do
           output.puts "Cleaning journal entries created before #{time}" if time
-          response = Chewy::Journal.new(indexes_from(only: only, except: except)).clean(time)
-          count = response['deleted'] || response['_indices']['_all']['deleted']
-          output.puts "Cleaned up #{count} journal entries"
+          response = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).clean(time, delete_by_query_options: delete_by_query_options)
+          if response.key?('task')
+            output.puts "Task to cleanup the journal has been created, #{response['task']}"
+          else
+            count = response['deleted'] || response['_indices']['_all']['deleted']
+            output.puts "Cleaned up #{count} journal entries"
+          end
         end
       end
 
@@ -228,6 +235,26 @@ module Chewy
         end
       end
 
+      # Reads options that are required to run journal cleanup asynchronously from ENV hash
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+      #
+      # @example
+      #   Chewy::RakeHelper.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => 'false','REQUESTS_PER_SECOND' => '10','SCROLL_SIZE' => '5000'})
+      #   # => { wait_for_completion: false, requests_per_second: 10.0, scroll_size: 5000 }
+      #
+      def delete_by_query_options_from_env(env)
+        env
+          .slice(*DELETE_BY_QUERY_OPTIONS)
+          .transform_keys { |k| k.downcase.to_sym }
+          .to_h do |key, value|
+            case key
+            when :wait_for_completion then [key, !FALSE_VALUES.include?(value.downcase)]
+            when :requests_per_second then [key, value.to_f]
+            when :scroll_size then [key, value.to_i]
+            end
+          end
+      end
+
       def normalize_indexes(*identifiers)
         identifiers.flatten(1).map { |identifier| normalize_index(identifier) }
       end
@@ -248,6 +275,12 @@ module Chewy
 
     private
 
+      def journal_indexes_from(only: nil, except: nil)
+        return if Array.wrap(only).empty? && Array.wrap(except).empty?
+
+        indexes_from(only: only, except: except)
+      end
+
       def indexes_from(only: nil, except: nil)
         indexes = if only.present?
           normalize_indexes(Array.wrap(only))

data/lib/chewy/search/request.rb

@@ -962,10 +962,22 @@ module Chewy
       #
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
       # @note The result hash is different for different API used.
-      # @param refresh [true, false] field names
+      # @param refresh [true, false] Refreshes all shards involved in the delete by query
+      # @param wait_for_completion [true, false] wait for request completion or run it asynchronously
+      #    and return task reference at `.tasks/task/${taskId}`.
+      # @param requests_per_second [Float] The throttle for this request in sub-requests per second
+      # @param scroll_size [Integer] Size of the scroll request that powers the operation
+
       # @return [Hash] the result of query execution
-      def delete_all(refresh: true)
-        request_body = only(WHERE_STORAGES).render.merge(refresh: refresh)
+      def delete_all(refresh: true, wait_for_completion: nil, requests_per_second: nil, scroll_size: nil)
+        request_body = only(WHERE_STORAGES).render.merge(
+          {
+            refresh: refresh,
+            wait_for_completion: wait_for_completion,
+            requests_per_second: requests_per_second,
+            scroll_size: scroll_size
+          }.compact
+        )
         ActiveSupport::Notifications.instrument 'delete_query.chewy', notification_payload(request: request_body) do
           request_body[:body] = {query: {match_all: {}}} if request_body[:body].empty?
           Chewy.client.delete_by_query(request_body)

data/lib/chewy/stash.rb

@@ -28,12 +28,12 @@ module Chewy
       # Cleans up all the journal entries until the specified time. If nothing is
       # specified - cleans up everything.
       #
-      # @param since_time [Time, DateTime] the time top boundary
+      # @param until_time [Time, DateTime] Clean everything before that date
       # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
-      def self.clean(until_time = nil, only: [])
+      def self.clean(until_time = nil, only: [], delete_by_query_options: {})
         scope = self.for(only)
         scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
-        scope.delete_all
+        scope.delete_all(**delete_by_query_options)
       end
 
       # Selects all the journal entries for the specified indices.

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.2.6'.freeze
+  VERSION = '7.2.7'.freeze
 end

data/lib/tasks/chewy.rake

@@ -94,7 +94,13 @@ namespace :chewy do
 
     desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them'
     task clean: :environment do |_task, args|
-      Chewy::RakeHelper.journal_clean(**parse_journal_args(args.extras))
+      delete_options = Chewy::RakeHelper.delete_by_query_options_from_env(ENV)
+      Chewy::RakeHelper.journal_clean(
+        [
+          parse_journal_args(args.extras),
+          {delete_by_query_options: delete_options}
+        ].reduce({}, :merge)
+      )
     end
   end
 end

data/spec/chewy/index/import/bulk_builder_spec.rb

@@ -62,6 +62,8 @@ describe Chewy::Index::Import::BulkBuilder do
         let(:to_index) { cities.first(2) }
         let(:delete) { [cities.last] }
         specify do
+          expect(subject).to receive(:data_for).with(cities.first).and_call_original
+          expect(subject).to receive(:data_for).with(cities.second).and_call_original
           expect(subject.bulk_body).to eq([
             {index: {_id: 1, data: {'name' => 'City17', 'rating' => 42}}},
             {index: {_id: 2, data: {'name' => 'City18', 'rating' => 42}}},
@@ -72,6 +74,8 @@ describe Chewy::Index::Import::BulkBuilder do
         context ':fields' do
           let(:fields) { %w[name] }
           specify do
+            expect(subject).to receive(:data_for).with(cities.first, fields: [:name]).and_call_original
+            expect(subject).to receive(:data_for).with(cities.second, fields: [:name]).and_call_original
             expect(subject.bulk_body).to eq([
               {update: {_id: 1, data: {doc: {'name' => 'City17'}}}},
               {update: {_id: 2, data: {doc: {'name' => 'City18'}}}},

data/spec/chewy/rake_helper_spec.rb

@@ -426,6 +426,33 @@ Total: \\d+s\\Z
       described_class.journal_clean(except: CitiesIndex, output: output)
       expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE))
 \\ACleaned up 1 journal entries
+Total: \\d+s\\Z
+      OUTPUT
+    end
+
+    it 'executes asynchronously' do
+      output = StringIO.new
+      expect(Chewy.client).to receive(:delete_by_query).with(
+        {
+          body: {query: {match_all: {}}},
+          index: ['chewy_journal'],
+          refresh: false,
+          requests_per_second: 10.0,
+          scroll_size: 200,
+          wait_for_completion: false
+        }
+      ).and_call_original
+      described_class.journal_clean(
+        output: output,
+        delete_by_query_options: {
+          wait_for_completion: false,
+          requests_per_second: 10.0,
+          scroll_size: 200
+        }
+      )
+
+      expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE))
+\\ATask to cleanup the journal has been created, [^\\n]*
 Total: \\d+s\\Z
       OUTPUT
     end
@@ -502,4 +529,45 @@ Total: \\d+s\\Z
       end
     end
   end
+
+  describe '.delete_by_query_options_from_env' do
+    subject(:options) { described_class.delete_by_query_options_from_env(env) }
+    let(:env) do
+      {
+        'WAIT_FOR_COMPLETION' => 'false',
+        'REQUESTS_PER_SECOND' => '10',
+        'SCROLL_SIZE' => '5000'
+      }
+    end
+
+    it 'parses the options' do
+      expect(options).to eq(
+        wait_for_completion: false,
+        requests_per_second: 10.0,
+        scroll_size: 5000
+      )
+    end
+
+    context 'with different boolean values' do
+      it 'parses the option correctly' do
+        %w[1 t true TRUE on ON].each do |v|
+          expect(described_class.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => v}))
+            .to eq(wait_for_completion: true)
+        end
+
+        %w[0 f false FALSE off OFF].each do |v|
+          expect(described_class.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => v}))
+            .to eq(wait_for_completion: false)
+        end
+      end
+    end
+
+    context 'with other env' do
+      let(:env) { {'SOME_ENV' => '123', 'REQUESTS_PER_SECOND' => '15'} }
+
+      it 'parses only the options' do
+        expect(options).to eq(requests_per_second: 15.0)
+      end
+    end
+  end
 end

data/spec/chewy/search/request_spec.rb

@@ -817,6 +817,31 @@ describe Chewy::Search::Request do
           request: {index: ['products'], body: {query: {match: {name: 'name3'}}}, refresh: false}
         )
       end
+
+      it 'delete records asynchronously' do
+        outer_payload = nil
+        ActiveSupport::Notifications.subscribe('delete_query.chewy') do |_name, _start, _finish, _id, payload|
+          outer_payload = payload
+        end
+        subject.query(match: {name: 'name3'}).delete_all(
+          refresh: false,
+          wait_for_completion: false,
+          requests_per_second: 10.0,
+          scroll_size: 2000
+        )
+        expect(outer_payload).to eq(
+          index: ProductsIndex,
+          indexes: [ProductsIndex],
+          request: {
+            index: ['products'],
+            body: {query: {match: {name: 'name3'}}},
+            refresh: false,
+            wait_for_completion: false,
+            requests_per_second: 10.0,
+            scroll_size: 2000
+          }
+        )
+      end
     end
 
     describe '#response=' do

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 7.2.6
+  version: 7.2.7
 platform: ruby
 authors:
 - Toptal, LLC
 - pyromaniac
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-13 00:00:00.000000000 Z
+date: 2022-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: database_cleaner
@@ -449,7 +449,7 @@ homepage: https://github.com/toptal/chewy
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -464,8 +464,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
-signing_key: 
+rubygems_version: 3.2.33
+signing_key:
 specification_version: 4
 summary: Elasticsearch ODM client wrapper
 test_files: