chewy 7.2.2 → 7.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 229c412b1e3464060b367c9c378d3662bccf2329b4e4eaa7617541243dbd6e38
-  data.tar.gz: 9bc674550cd6152e8a18853fa46c91873c7a626c426cd50aa2cb8c67b1c9455e
+  metadata.gz: 37685481eff7312e08ba8d113f030ee5ab38e3414788c6cd2a0385829c99a712
+  data.tar.gz: d4a2a9d86d93e9a9c6ec5d11cabe51f67511f6b347faee3be9e276cf528c4dc2
 SHA512:
-  metadata.gz: e3e8409f81bc2504e9edbf5093fc9329bf443862cb96e113c9c00fc586c1e0e3a4273b3d6e82466f9b3d301e3d9eafcc72892cce718aff7dcfbbce85aba1698d
-  data.tar.gz: 33b5b0e21ef7c9896f1ed381130f225a1d51e073112213f631fb1c57c6d69c49af54817b1de3afc9be325c79ac24f9c173275b4448ffb7b43e82403bee03bd70
+  metadata.gz: c973f623b179e98284019a2b2dddc1cb7506719585064673cccab0e1c790f599dce43a1395e364d5ed8be814ee0af1fe4985e188f25dcaa32e75eefa57a4f6e3
+  data.tar.gz: bd1c55232fd02520382a20109899f3b4d59a174c1d1ff875592f08582b5f3072dd7d6f47760df3be58f38756718f31f802832476a1e0c64ebfb9909748e31eac

data/.github/workflows/ruby.yml

@@ -3,7 +3,7 @@ name: CI
 on: [push]
 
 jobs:
-  tests:
+  ruby-2:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -22,25 +22,33 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run Elasticsearch
-        uses: elastic/elastic-github-actions/elasticsearch@master
+        uses: elastic/elastic-github-actions/elasticsearch@9de0f78f306e4ebc0838f057e6b754364685e759
         with:
           stack-version: 7.10.1
           port: 9250
       - name: Tests
         run: bundle exec rspec
 
-  ruby-3-0-activerecord-6-1:
+  ruby-3:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.0', 3.1 ]
+        gemfile: [ rails.6.1.activerecord, rails.7.0.activerecord ]
+    name: ${{ matrix.ruby }}-${{ matrix.gemfile }}
+
     env:
-      BUNDLE_GEMFILE: gemfiles/rails.6.1.activerecord.gemfile
+      BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
+
     steps:
       - uses: actions/checkout@v2
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.0'
+          ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run Elasticsearch
-        uses: elastic/elastic-github-actions/elasticsearch@master
+        uses: elastic/elastic-github-actions/elasticsearch@9de0f78f306e4ebc0838f057e6b754364685e759
         with:
           stack-version: 7.10.1
           port: 9250

data/CHANGELOG.md

@@ -8,6 +8,45 @@
 
 ### Bugs Fixed
 
+## 7.2.5 (2022-03-04)
+
+### New Features
+
+ * [#827](https://github.com/toptal/chewy/pull/827): Add `:lazy_sidekiq` strategy, that defers not only importing but also `update_index` callback evaluation for created and updated objects. ([@sl4vr][])
+ * [#827](https://github.com/toptal/chewy/pull/827): Add `:atomic_no_refresh` strategy. Like `:atomic`, but `refresh=false` parameter is set. ([@barthez][])
+ * [#827](https://github.com/toptal/chewy/pull/827): Add `:no_refresh` chain call to `update_index` matcher to ensure import was called with `refresh=false`. ([@barthez][])
+ 
+### Bugs Fixed
+
+ * [#835](https://github.com/toptal/chewy/pull/835): Support keyword arguments in named scopes. ([@milk1000cc][])
+
+## 7.2.4 (2022-02-03)
+
+### New Features
+
+ * [#760](https://github.com/toptal/chewy/pull/760): Replace parent-child mapping with a [join field](https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html#parent-child-mapping-types) ([@mrzasa][])
+
+### Bugs Fixed
+
+ * [#825](https://github.com/toptal/chewy/issues/825): Fix mismatch argument names at update_mapping rake task ([@AgeevAndrew][])
+ * [#832](https://github.com/toptal/chewy/issues/832): Fix "cannot load such file -- `i18n/core_ext/hash`" ([@chrisandreae][])
+
+## 7.2.3 (2021-10-29)
+
+### New Features
+
+  * [#801](https://github.com/toptal/chewy/pull/801): Add the [`track_total_hits`](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-your-data.html#track-total-hits) option to the query. ([@milk1000cc][])
+  * [#810](https://github.com/toptal/chewy/pull/810): Use `unsafe_load` when loading config to support Psych 4.
+  * [#819](https://github.com/toptal/chewy/pull/819): Add the [`ignore_unavailable`](https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html#multi-index) option to the request. ([@kolauren][])
+
+### Changes
+
+  * [#817](https://github.com/toptal/chewy/pull/817): Show warning message during rake chewy:reset or chewy:upgrade if journaling is disabled. ([@konalegi][])
+
+### Bugs Fixed
+
+  * [#816](https://github.com/toptal/chewy/pull/816): Move query [`preference`](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/search-search.html#search-preference) from body to query parameters. ([@afg419][])
+
 ## 7.2.2 (2021-05-24)
 
 ### Changes
@@ -618,9 +657,11 @@
   * Initial version
 
 [@0x0badc0de]: https://github.com/0x0badc0de
+[@AgeevAndrew]: https://github.com/AgeevAndrew
 [@aglushkov]: https://github.com/aglushkov
 [@AlexVPopov]: https://github.com/AlexVPopov
 [@AndreySavelyev]: https://github.com/AndreySavelyev
+[@afg419]: https://github.com/afg419
 [@arion]: https://github.com/arion
 [@arturtr]: https://github.com/arturtr
 [@averell23]: https://github.com/averell23
@@ -631,6 +672,7 @@
 [@biow0lf]: https://github.com/biow0lf
 [@Borzik]: https://github.com/Borzik
 [@caldwecr]: https://github.com/caldwecr
+[@chrisandreae]: https://github.com/chrisandreae
 [@clupprich]: https://github.com/clupprich
 [@dalthon]: https://github.com/dalthon
 [@davekaro]: https://github.com/davekaro
@@ -661,6 +703,7 @@
 [@josephchoe]: https://github.com/josephchoe
 [@jshirley]: https://github.com/jshirley
 [@ka8725]: https://github.com/ka8725
+[@kolauren]: https://github.com/kolauren
 [@konalegi]: https://github.com/konalegi
 [@lardawge]: https://github.com/lardawge
 [@leemhenson]: https://github.com/leemhenson
@@ -674,6 +717,7 @@
 [@mattzollinhofer]: https://github.com/mattzollinhofer
 [@menglewis]: https://github.com/menglewis
 [@mikeyhogarth]: https://github.com/mikeyhogarth
+[@milk1000cc]: https://github.com/milk1000cc
 [@mkcode]: https://github.com/mkcode
 [@mpeychich]: https://github.com/mpeychich
 [@mrbrdo]: https://github.com/mrbrdo

data/Gemfile

@@ -19,3 +19,4 @@ gem 'redcarpet'
 gem 'yard'
 
 gem 'rexml' if RUBY_VERSION >= '3.0.0'
+gem 'ruby2_keywords' if RUBY_VERSION < '2.7'

data/README.md

@@ -340,7 +340,7 @@ Chewy.settings = {
   [See index settings here](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html).
   [See root object settings here](https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-field-mapping.html).
 
-  See [mapping.rb](lib/chewy/type/mapping.rb) for more details.
+  See [mapping.rb](lib/chewy/index/mapping.rb) for more details.
 
 5. Add model-observing code
 
@@ -446,6 +446,23 @@ end
 
 See the section on *Script fields* for details on calculating distance in a search.
 
+### Join fields
+
+You can use a [join field](https://www.elastic.co/guide/en/elasticsearch/reference/current/parent-join.html)
+to implement parent-child relationships between documents.
+It [replaces the old `parent_id` based parent-child mapping](https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html#parent-child-mapping-types)
+
+To use it, you need to pass `relations` and `join` (with `type` and `id`) options:
+```ruby
+field :hierarchy_link, type: :join, relations: {question: %i[answer comment], answer: :vote, vote: :subvote}, join: {type: :comment_type, id: :commented_id}
+```
+assuming you have `comment_type` and `commented_id` fields in your model.
+
+Note that when you reindex a parent, it's children and grandchildren will be reindexed as well.
+This may require additional queries to the primary database and to elastisearch.
+
+Also note that the join field doesn't support crutches (it should be a field directly defined on the model).
+
 ### Crutches™ technology
 
 Assume you are defining your index like this (product has_many categories through product_categories):
@@ -737,6 +754,23 @@ The default queue name is `chewy`, you can customize it in settings: `sidekiq.qu
 Chewy.settings[:sidekiq] = {queue: :low}
 ```
 
+#### `:lazy_sidekiq`
+
+This does the same thing as `:sidekiq`, but with lazy evaluation. Beware it does not allow you to use any non-persistent record state for indices and conditions because record will be re-fetched from database asynchronously using sidekiq. However for destroying records strategy will fallback to `:sidekiq` because it's not possible to re-fetch deleted records from database.
+
+The purpose of this strategy is to improve the response time of the code that should update indexes, as it does not only defer actual ES calls to a background job but `update_index` callbacks evaluation (for created and updated objects) too. Similar to `:sidekiq`, index update is asynchronous so this strategy cannot be used when data and index synchronization is required.
+
+```ruby
+Chewy.strategy(:lazy_sidekiq) do
+  City.popular.map(&:do_some_update_action!)
+end
+```
+
+The default queue name is `chewy`, you can customize it in settings: `sidekiq.queue_name`
+```
+Chewy.settings[:sidekiq] = {queue: :low}
+```
+
 #### `:active_job`
 
 This does the same thing as `:atomic`, but using ActiveJob. This will inherit the ActiveJob configuration settings including the `active_job.queue_adapter` setting for the environment. Patch `Chewy::Strategy::ActiveJob::Worker` for index updates improving.
@@ -959,7 +993,7 @@ Main methods of the request DSL are: `query`, `filter` and `post_filter`, it is
 ```ruby
 CitiesIndex
   .filter(term: {name: 'Bangkok'})
-  .query { match name: 'London' }
+  .query(match: {name: 'London'})
   .query.not(range: {population: {gt: 1_000_000}})
 ```
 
@@ -969,7 +1003,7 @@ You can query a set of indexes at once:
 CitiesIndex.indices(CountriesIndex).query(match: {name: 'Some'})
 ```
 
-See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html and https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-dsl for more details.
+See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html and https://github.com/elastic/elasticsearch-dsl-ruby for more details.
 
 An important part of requests manipulation is merging. There are 4 methods to perform it: `merge`, `and`, `or`, `not`. See [Chewy::Search::QueryProxy](lib/chewy/search/query_proxy.rb) for details. Also, `only` and `except` methods help to remove unneeded parts of the request.
 

data/chewy.gemspec

@@ -31,6 +31,6 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
   spec.add_development_dependency 'unparser'
 
   spec.add_dependency 'activesupport', '>= 5.2'
-  spec.add_dependency 'elasticsearch', '>= 7.12.0'
+  spec.add_dependency 'elasticsearch', '>= 7.12.0', '< 7.14.0'
   spec.add_dependency 'elasticsearch-dsl'
 end

data/gemfiles/rails.7.0.activerecord.gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 7.0.0'
+gem 'activerecord', '~> 7.0.0'
+gem 'activesupport', '~> 7.0.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml' if RUBY_VERSION >= '3.0.0'
+
+gemspec path: '../'

data/lib/chewy/config.rb

@@ -32,7 +32,7 @@ module Chewy
                   # Set number_of_replicas to 0 before reset and put the original value after
                   # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
                   :reset_no_replicas,
-                  # Refresh or not when import async (sidekiq, activejob)
+                  # Refresh or not when import async (sidekiq, lazy_sidekiq, activejob)
                   :disable_refresh_async,
                   # Default options for root of Chewy type. Allows to set default options
                   # for type mappings like `_all`.
@@ -133,7 +133,7 @@ module Chewy
 
           if File.exist?(file)
             yaml = ERB.new(File.read(file)).result
-            hash = YAML.load(yaml) # rubocop:disable Security/YAMLLoad
+            hash = YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(yaml) : YAML.load(yaml) # rubocop:disable Security/YAMLLoad
             hash[Rails.env].try(:deep_symbolize_keys) if hash
           end
         end || {}

data/lib/chewy/errors.rb

@@ -30,4 +30,10 @@ module Chewy
       super message
     end
   end
+
+  class InvalidJoinFieldType < Error
+    def initialize(join_field_type, join_field_name, relations)
+      super("`#{join_field_type}` set for the join field `#{join_field_name}` is not on the :relations list (#{relations})")
+    end
+  end
 end

data/lib/chewy/fields/base.rb

@@ -1,8 +1,8 @@
 module Chewy
   module Fields
     class Base
-      attr_reader :name, :options, :value, :children
-      attr_accessor :parent
+      attr_reader :name, :join_options, :options, :children
+      attr_accessor :parent # used by Chewy::Index::Mapping to expand nested fields
 
       def initialize(name, value: nil, **options)
         @name = name.to_sym
@@ -10,9 +10,11 @@ module Chewy
         update_options!(**options)
         @value = value
         @children = []
+        @allowed_relations = find_allowed_relations(options[:relations]) # for join fields
       end
 
       def update_options!(**options)
+        @join_options = options.delete(:join) || {}
         @options = options
       end
 
@@ -53,30 +55,70 @@ module Chewy
         {name => result}
       end
 
+      def value
+        if join_field?
+          join_type = join_options[:type]
+          join_id = join_options[:id]
+          # memoize
+          @value ||= proc do |object|
+            validate_join_type!(value_by_name_proc(join_type).call(object))
+            # If it's a join field and it has join_id, the value is compound and contains
+            # both name (type) and id of the parent object
+            if value_by_name_proc(join_id).call(object).present?
+              {
+                name: value_by_name_proc(join_type).call(object), # parent type
+                parent: value_by_name_proc(join_id).call(object)  # parent id
+              }
+            else
+              value_by_name_proc(join_type).call(object)
+            end
+          end
+        else
+          @value
+        end
+      end
+
     private
 
       def geo_point?
         @options[:type].to_s == 'geo_point'
       end
 
+      def join_field?
+        @options[:type].to_s == 'join'
+      end
+
       def ignore_blank?
         @options.fetch(:ignore_blank) { geo_point? }
       end
 
       def evaluate(objects)
-        object = objects.first
-
         if value.is_a?(Proc)
-          if value.arity.zero?
-            object.instance_exec(&value)
-          elsif value.arity.negative?
-            value.call(*object)
-          else
-            value.call(*objects.first(value.arity))
-          end
+          value_by_proc(objects, value)
         else
-          message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+          value_by_name(objects, value)
+        end
+      end
 
+      def value_by_proc(objects, value)
+        object = objects.first
+        if value.arity.zero?
+          object.instance_exec(&value)
+        elsif value.arity.negative?
+          value.call(*object)
+        else
+          value.call(*objects.first(value.arity))
+        end
+      end
+
+      def value_by_name(objects, value)
+        object = objects.first
+        message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+        value_by_name_proc(message).call(object)
+      end
+
+      def value_by_name_proc(message)
+        proc do |object|
           if object.is_a?(Hash)
             if object.key?(message)
               object[message]
@@ -89,6 +131,20 @@ module Chewy
         end
       end
 
+      def validate_join_type!(type)
+        return unless type
+        return if @allowed_relations.include?(type.to_sym)
+
+        raise Chewy::InvalidJoinFieldType.new(type, @name, options[:relations])
+      end
+
+      def find_allowed_relations(relations)
+        return [] unless relations
+        return relations unless relations.is_a?(Hash)
+
+        (relations.keys + relations.values).flatten.uniq
+      end
+
       def compose_children(value, *parent_objects)
         return unless value
 

data/lib/chewy/fields/root.rb

@@ -1,7 +1,7 @@
 module Chewy
   module Fields
     class Root < Chewy::Fields::Base
-      attr_reader :dynamic_templates, :id, :parent, :parent_id
+      attr_reader :dynamic_templates, :id
 
       def initialize(name, **options)
         super(name, **options)
@@ -12,9 +12,7 @@ module Chewy
 
       def update_options!(**options)
         @id = options.fetch(:id, options.fetch(:_id, @id))
-        @parent = options.fetch(:parent, options.fetch(:_parent, @parent))
-        @parent_id = options.fetch(:parent_id, @parent_id)
-        @options.merge!(options.except(:id, :_id, :parent, :_parent, :parent_id, :type))
+        @options.merge!(options.except(:id, :_id, :type))
       end
 
       def mappings_hash
@@ -29,7 +27,7 @@ module Chewy
         mappings[name]
       end
 
-      def dynamic_template(*args)
+      ruby2_keywords def dynamic_template(*args)
         options = args.extract_options!.deep_symbolize_keys
         if args.first
           template_name = :"template_#{dynamic_templates.count.next}"
@@ -50,12 +48,6 @@ module Chewy
         end
       end
 
-      def compose_parent(object)
-        return unless parent_id
-
-        parent_id.arity.zero? ? object.instance_exec(&parent_id) : parent_id.call(object)
-      end
-
       def compose_id(object)
         return unless id
 

data/lib/chewy/index/actions.rb

@@ -156,11 +156,11 @@ module Chewy
             suffixed_name = index_name(suffix: suffix)
 
             optimize_index_settings suffixed_name
-            result = import import_options.merge(
+            result = import(**import_options.merge(
               suffix: suffix,
               journal: journal,
               refresh: !Chewy.reset_disable_refresh_interval
-            )
+            ))
             original_index_settings suffixed_name
 
             delete if indexes.blank?
@@ -176,7 +176,7 @@ module Chewy
             result
           else
             purge!
-            import import_options.merge(journal: journal)
+            import(**import_options.merge(journal: journal))
           end
 
           specification.lock!

data/lib/chewy/index/adapter/active_record.rb

@@ -94,6 +94,11 @@ module Chewy
           object_class.connection.execute(sql).map(&converter)
         end
 
+        def raw(scope, converter)
+          sql = scope.to_sql
+          object_class.connection.execute(sql).map(&converter)
+        end
+
         def relation_class
           ::ActiveRecord::Relation
         end

data/lib/chewy/index/adapter/object.rb

@@ -85,7 +85,7 @@ module Chewy
         # @param args [Array<#to_json>]
         # @option options [Integer] :batch_size import processing batch size
         # @return [true, false]
-        def import(*args, &block)
+        ruby2_keywords def import(*args, &block)
           collection, options = import_args(*args)
           import_objects(collection, options, &block)
         end
@@ -113,7 +113,7 @@ module Chewy
         #   end
         #
         # @see Chewy::Index::Adapter::Base#import_fields
-        def import_fields(*args, &block)
+        ruby2_keywords def import_fields(*args, &block)
           return enum_for(:import_fields, *args) unless block_given?
 
           options = args.extract_options!
@@ -139,7 +139,7 @@ module Chewy
         # For the Object adapter returns the objects themselves in batches.
         #
         # @see Chewy::Index::Adapter::Base#import_references
-        def import_references(*args, &block)
+        ruby2_keywords def import_references(*args, &block)
           return enum_for(:import_references, *args) unless block_given?
 
           collection, options = import_args(*args)

data/lib/chewy/index/adapter/orm.rb

@@ -72,7 +72,7 @@ module Chewy
         #   # or
         #   UsersIndex.import users.map(&:id) # user ids will be deleted from index
         #
-        def import(*args, &block)
+        ruby2_keywords def import(*args, &block)
           collection, options = import_args(*args)
 
           if !collection.is_a?(relation_class) || options[:direct_import]
@@ -82,7 +82,7 @@ module Chewy
           end
         end
 
-        def import_fields(*args, &block)
+        ruby2_keywords def import_fields(*args, &block)
           return enum_for(:import_fields, *args) unless block_given?
 
           collection, options = import_args(*args)
@@ -101,11 +101,13 @@ module Chewy
           additional_scope = options[options[:_index].to_sym].try(:[], :scope) || options[:scope]
 
           loaded_objects = load_scope_objects(scope, additional_scope)
-            .index_by do |object|
-              object.public_send(primary_key).to_s
-            end
+          loaded_objects = raw(loaded_objects, options[:raw_import]) if options[:raw_import]
+
+          indexed_objects = loaded_objects.index_by do |object|
+            object.public_send(primary_key).to_s
+          end
 
-          ids.map { |id| loaded_objects[id.to_s] }
+          ids.map { |id| indexed_objects[id.to_s] }
         end
 
       private