chewy 0.8.4 → 7.3.4

data/lib/tasks/chewy.rake

@@ -1,49 +1,116 @@
 require 'chewy/rake_helper'
 
+def parse_classes(args)
+  if args.present? && args.first.tr!('-', '')
+    {except: args}
+  else
+    {only: args}
+  end
+end
+
+def parse_parallel_args(args)
+  options = {}
+  options[:parallel] = args.first =~ /\A\d+\z/ ? Integer(args.shift) : true
+  options.merge!(parse_classes(args))
+end
+
+def parse_journal_args(args)
+  options = {}
+  options[:time] = Time.parse(args.shift) if args.first =~ /\A\d+/
+  options.merge!(parse_classes(args))
+end
+
 namespace :chewy do
-  desc 'Destroy, recreate and import data to specified index'
-  task :reset, [:index] => :environment do |task, args|
-    Chewy::RakeHelper.subscribed_task_stats do
+  desc 'Destroys, recreates and imports data for the specified indexes or all of them'
+  task reset: :environment do |_task, args|
+    Chewy::RakeHelper.reset(**parse_classes(args.extras))
+  end
 
-      if args[:index].present?
-        Chewy::RakeHelper.reset_index(args[:index])
-      else
-        Chewy::RakeHelper.reset_all
-      end
-    end
+  desc 'Resets data for the specified indexes or all of them only if the index specification is changed'
+  task upgrade: :environment do |_task, args|
+    Chewy::RakeHelper.upgrade(**parse_classes(args.extras))
   end
 
-  namespace :reset do
-    desc 'Destroy, recreate and import data for all found indexes'
-    task all: :environment do
-      ActiveSupport::Deprecation.warn('`rake chewy:reset:all` is deprecated and will be removed soon. Use `rake chewy:reset` instead')
+  desc 'Updates data for the specified indexes/types or all of them'
+  task update: :environment do |_task, args|
+    Chewy::RakeHelper.update(**parse_classes(args.extras))
+  end
 
-      Chewy::RakeHelper.subscribed_task_stats do
-        Chewy::RakeHelper.reset_all
-      end
-    end
+  desc 'Synchronizes data for the specified indexes/types or all of them'
+  task sync: :environment do |_task, args|
+    Chewy::RakeHelper.sync(**parse_classes(args.extras))
   end
 
-  desc 'Updates data specified index'
-  task :update, [:index] => :environment do |task, args|
-    Chewy::RakeHelper.subscribed_task_stats do
+  desc 'Resets all the indexes with the specification changed and synchronizes the rest of them'
+  task deploy: :environment do
+    processed = Chewy::RakeHelper.upgrade
+    Chewy::RakeHelper.sync(except: processed)
+  end
 
-      if args[:index].present?
-        Chewy::RakeHelper.update_index(args[:index])
-      else
-        Chewy::RakeHelper.update_all
-      end
+  desc 'Reindex data from source index to destination index'
+  task :reindex, %i[source dest] => :environment do |_task, args|
+    Chewy::RakeHelper.reindex(source: args[:source], dest: args[:dest])
+  end
+
+  desc 'Update mapping of exising index with body hash'
+  task :update_mapping, %i[index_name] => :environment do |_task, args|
+    Chewy::RakeHelper.update_mapping(name: args[:index_name])
+  end
+
+  desc 'Creates missing indexes'
+  task create_missing_indexes: :environment do
+    Chewy::RakeHelper.create_missing_indexes!
+  end
+
+  namespace :parallel do
+    desc 'Parallel version of `rake chewy:reset`'
+    task reset: :environment do |_task, args|
+      Chewy::RakeHelper.reset(**parse_parallel_args(args.extras))
+    end
+
+    desc 'Parallel version of `rake chewy:upgrade`'
+    task upgrade: :environment do |_task, args|
+      Chewy::RakeHelper.upgrade(**parse_parallel_args(args.extras))
+    end
+
+    desc 'Parallel version of `rake chewy:update`'
+    task update: :environment do |_task, args|
+      Chewy::RakeHelper.update(**parse_parallel_args(args.extras))
+    end
+
+    desc 'Parallel version of `rake chewy:sync`'
+    task sync: :environment do |_task, args|
+      Chewy::RakeHelper.sync(**parse_parallel_args(args.extras))
+    end
+
+    desc 'Parallel version of `rake chewy:deploy`'
+    task deploy: :environment do |_task, args|
+      parallel = args.extras.first =~ /\A\d+\z/ ? Integer(args.extras.first) : true
+      processed = Chewy::RakeHelper.upgrade(parallel: parallel)
+      Chewy::RakeHelper.sync(except: processed, parallel: parallel)
     end
   end
 
-  namespace :update do
-    desc 'Updates data for all found indexes'
-    task all: :environment do
-      ActiveSupport::Deprecation.warn('`rake chewy:update:all` is deprecated and will be removed soon. Use `rake chewy:update` instead')
+  namespace :journal do
+    desc 'Create manually journal, useful when `skip_journal_creation_on_import` is used'
+    task create: :environment do |_task, _args|
+      Chewy::RakeHelper.journal_create
+    end
+
+    desc 'Applies changes that were done after the specified time for the specified indexes/types or all of them'
+    task apply: :environment do |_task, args|
+      Chewy::RakeHelper.journal_apply(**parse_journal_args(args.extras))
+    end
 
-      Chewy::RakeHelper.subscribed_task_stats do
-        Chewy::RakeHelper.update_all
-      end
+    desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them'
+    task clean: :environment do |_task, args|
+      delete_options = Chewy::RakeHelper.delete_by_query_options_from_env(ENV)
+      Chewy::RakeHelper.journal_clean(
+        **[
+          parse_journal_args(args.extras),
+          {delete_by_query_options: delete_options}
+        ].reduce({}, :merge)
+      )
     end
   end
 end

data/migration_guide.md

@@ -0,0 +1,56 @@
+# Migration guide
+
+This document outlines the steps you need to take when migrating between major versions of
+Chewy and Elasticsearch. For simplicity's sake the guide will assume that you're using
+Chewy alongside a matching Elasticsearch version.
+
+## Chewy 6/Elasticsearch 6 to Chewy 7/Elasticsearch 7
+
+In order to upgrade Chewy 6/Elasticsearch 6 to Chewy 7/Elasticsearch 7 in the most seamless manner you have to:
+
+* Upgrade to the latest 6.x stable releases, namely Chewy 6.0, Elasticsearch 6.8
+* Study carefully [Breaking changes in 7.0](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/breaking-changes-7.0.html), make sure your application conforms.
+* Run your test suite on Chewy 7.0 / Elasticsearch 7
+* Run manual tests on Chewy 7.0 / Elasticsearch 7
+* Upgrade to Chewy 7.0
+  * The “total hits” counter is an integer for ES versions < 7 and an object (hash) for the versions starting from 7.0.0. Elasticsearch added a special option, `rest_total_hits_as_int`, to ease the upgrade, that could be appended to any request and results in the old “total hits” format. Unfortunately, this option is not recognized by ES versions prior to 7.0.0, which means that we have to check the version to decide if we need this option.
+    Normally Chewy does memoization of the current ES version, but this might be inappropriate for the upgrade, as the version changes live.
+    To handle that we have 2 versions of Chewy for this stage of the upgrade: 7.0.0 and 7.0.1. Version 7.0.0 does the memoization and version 7.0.1 requests the current version on every search request. 
+  * You can use the 7.0.0 version if it's fine for you to have an application restart immediately after ES cluster upgrade.
+  * If you're using the 7.0.1 version you might be interested in keeping the timeframe between this step and updating to Chewy 7.1 as small as possible, as version 7.0.1 skips ES version memoization for search requests to help dynamically detect ES version. This leads to an extra version request on each search request, i.e. could affect the overall performance/latency of the search and a load of ES cluster.
+* Perform a [rolling upgrade](https://www.elastic.co/guide/en/elasticsearch/reference//rolling-upgrades.html) of Elasticsearch
+* Run your test suite on Chewy 7.1 / Elasticsearch 7
+* Run manual tests on Chewy 7.1 / Elasticsearch 7
+* Upgrade to Chewy 7.1
+* Upgrade to Chewy 7.2:
+  * Remove all the the `Chewy::Type` class usages, e.g. remove `CitiesIndex::City` / `CitiesIndex.city`
+    * `CitiesIndex::City.import! ...` becomes `CitiesIndex.import! ...`
+  * Update indexes with simplified DSL:
+    * `define_type` block -> `index_scope` clause
+      * it can be omitted completely, if you don't need to specify the scope or options, e.g. `name`
+  * Remove type names from string representations:
+    * in `update_index` ActiveRecord helper and RSpec matcher, e.g.
+      * `update_index('cities#city')` -> `update_index('cities')`
+      * `update_index(UsersIndex::User)` -> `update_index(UsersIndex)`
+    * in rake tasks (e.g. `rake chewy:update[cities#city]` -> `rake chewy:update[cities]`)
+    * rake tasks output is also changed (e.g. `Imported CitiesIndex::City in 1s, stats: index 3` -> `Imported CitiesIndex in 1s, stats: index 3`)
+  * Use index name instead of type name in loader additional scope
+    * e.g. `CitiesIndex.filter(...).load(city: {scope: City.where(...)})` -> `CitiesIndex.filter(...).load(cities: {scope: City.where(...)})`
+
+## Chewy 5/Elasticsearch 5 to Chewy 6/Elasticsearch 6
+
+In order to upgrade Chewy 5/Elasticsearch 5 to Chewy 6/Elasticsearch 6 in the most seamless manner you have to:
+
+* Upgrade to the latest 5.x stable releases, namely Chewy 5.2, Elasticsearch 5.6
+* [Migrate any multi-typed indexes into single-typed](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/removal-of-types.html)
+  * Using [multi-index queries](https://github.com/toptal/chewy/pull/657) could be helpful
+  * Parent/Child [relationship is deprecated](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/removal-of-types.html#parent-child-mapping-types) in favor of the [join field](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/parent-join.html)
+* Handle deprecation of `string` type & `not_analyzed` value for the `index` mapping parameter:
+  * replace fields with `{ type: 'string', index: 'not_analyzed'}` by `{type: 'keyword'}`
+  * replace fields with `{ type: 'string', index: 'analyzed'}` by `{type: 'text'}`
+* `PathHierarchy` tokenizer' param `delimiter` now accepts only one argument, [others should be replaced by character filter ](https://discuss.elastic.co/t/multichar-delimiter-in-path-hierarchy-tokenizer/16203)
+* Make sure you don't use any other of the [deprecated Elasticsearch 5 features](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/breaking-changes-6.0.html)
+* Run your test suite on Chewy 6 / Elasticsearch 6
+* Run manual tests on Chewy 6 / Elasticsearch 6
+* Upgrade to Chewy 6
+* Perform a [rolling upgrade](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/rolling-upgrades.html) of Elasticsearch

data/spec/chewy/config_spec.rb

@@ -6,34 +6,106 @@ describe Chewy::Config do
   its(:logger) { should be_nil }
   its(:transport_logger) { should be_nil }
   its(:transport_logger) { should be_nil }
-  its(:query_mode) { should == :must }
-  its(:filter_mode) { should == :and }
-  its(:post_filter_mode) { should be_nil }
   its(:root_strategy) { should == :base }
   its(:request_strategy) { should == :atomic }
+  its(:console_strategy) { should == :urgent }
   its(:use_after_commit_callbacks) { should == true }
+  its(:indices_path) { should == 'app/chewy' }
+  its(:reset_disable_refresh_interval) { should == false }
+  its(:reset_no_replicas) { should == false }
+  its(:disable_refresh_async) { should == false }
+  its(:search_class) { should be < Chewy::Search::Request }
 
   describe '#transport_logger=' do
     let(:logger) { Logger.new('/dev/null') }
     after { subject.transport_logger = nil }
 
-    specify { expect { subject.transport_logger = logger }
-      .to change { Chewy.client.transport.logger }.to(logger) }
-    specify { expect { subject.transport_logger = logger }
-      .to change { subject.transport_logger }.to(logger) }
-    specify { expect { subject.transport_logger = logger }
-      .to change { subject.configuration[:logger] }.from(nil).to(logger) }
+    specify do
+      expect { subject.transport_logger = logger }
+        .to change { Chewy.client.transport.logger }.to(logger)
+    end
+    specify do
+      expect { subject.transport_logger = logger }
+        .to change { subject.transport_logger }.to(logger)
+    end
+    specify do
+      expect { subject.transport_logger = logger }
+        .to change { subject.configuration[:logger] }.from(nil).to(logger)
+    end
   end
 
   describe '#transport_tracer=' do
     let(:tracer) { Logger.new('/dev/null') }
     after { subject.transport_tracer = nil }
 
-    specify { expect { subject.transport_tracer = tracer }
-      .to change { Chewy.client.transport.tracer }.to(tracer) }
-    specify { expect { subject.transport_tracer = tracer }
-      .to change { subject.transport_tracer }.to(tracer) }
-    specify { expect { subject.transport_tracer = tracer }
-      .to change { subject.configuration[:tracer] }.from(nil).to(tracer) }
+    specify do
+      expect { subject.transport_tracer = tracer }
+        .to change { Chewy.client.transport.tracer }.to(tracer)
+    end
+    specify do
+      expect { subject.transport_tracer = tracer }
+        .to change { subject.transport_tracer }.to(tracer)
+    end
+    specify do
+      expect { subject.transport_tracer = tracer }
+        .to change { subject.configuration[:tracer] }.from(nil).to(tracer)
+    end
+  end
+
+  describe '#search_class' do
+    context 'nothing is defined' do
+      before do
+        hide_const('Kaminari')
+      end
+
+      specify do
+        expect(subject.search_class.included_modules)
+          .not_to include(Chewy::Search::Pagination::Kaminari)
+      end
+    end
+
+    context 'kaminari' do
+      specify do
+        expect(subject.search_class.included_modules)
+          .to include(Chewy::Search::Pagination::Kaminari)
+      end
+    end
+  end
+
+  describe '#configuration' do
+    before { subject.settings = {indices_path: 'app/custom_indices_path'} }
+
+    specify do
+      expect(subject.configuration).to include(indices_path: 'app/custom_indices_path')
+    end
+
+    context 'when Rails::VERSION constant is defined' do
+      it 'looks for configuration in "config/chewy.yml"' do
+        module Rails
+          VERSION = '5.1.1'.freeze
+
+          def self.root
+            Pathname.new(__dir__)
+          end
+        end
+
+        expect(File).to receive(:exist?)
+          .with(Pathname.new(__dir__).join('config', 'chewy.yml'))
+        subject.configuration
+      end
+    end
+  end
+
+  describe '.console_strategy' do
+    context 'sets .console_strategy' do
+      let(:default_strategy) { subject.console_strategy }
+      let(:new_strategy) { :atomic }
+      after { subject.console_strategy = default_strategy }
+
+      specify do
+        expect { subject.console_strategy = new_strategy }
+          .to change { subject.console_strategy }.to(new_strategy)
+      end
+    end
   end
 end