chewy 7.5.1 → 8.0.0.pre.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64f061f379d740c56b7af2678ec03e3c80c1bd4c072b2aa3f68bf7803d4bc0fe
-  data.tar.gz: 6226557f9cec700d91473f36bb8623ff5bbf838211463188ac0f83eaf0ab227d
+  metadata.gz: fe6459bb964c9d44912601a2a5abc6ea93ea7ac7be94c384673ade2ce68fb9e6
+  data.tar.gz: 397d2104f889f9e9232ed673b3fe5dc50fe4c13f1330b4ce9b1b1e45f9a11e16
 SHA512:
-  metadata.gz: 8cb278cc5fa838a97e0c9fb94c90ea08d4a84effb041a809f850a267514c61ce78f1a919a8a0fdc5119ca968e217fe9b0aa38c307f9589f8413501f690efb314
-  data.tar.gz: 50ad4271e4948b22b626202843be7107fcc720fd3da490c14a372ca018ab0e35c0e151d5dc5b55f23b7c382a0d19cf72f7cc63e40a2dc1fc0ca286c1de717941
+  metadata.gz: ebe662c3e11aaebd6fb8de8058ed9199d94b7eb80e7f4b6347c0fe98481c37cdb6be51e72d13645e627ae7d653d43c4bcd6a9a1f81b4eafdbb7371c814f2e847
+  data.tar.gz: 0e3b1d9b5eb0724fb1b2453d75e7980d9d7c1a3b341dca3934c366c7ba6ec5242bd9e647538486c9be6203d1ca9a5c6f18bcefcfc7ea64cdf8a31b7893af9f72

data/.github/CODEOWNERS

@@ -1 +1 @@
-.github/workflows @toptal/platform-sre
+.github/workflows @toptal/sre

data/.github/workflows/ruby.yml

@@ -23,17 +23,29 @@ jobs:
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
 
+    services:
+      redis:
+        # Docker Hub image
+        image: redis
+        ports:
+          - '6379:6379'
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
-      - name: Run Elasticsearch
-        uses: elastic/elastic-github-actions/elasticsearch@9de0f78f306e4ebc0838f057e6b754364685e759
-        with:
-          stack-version: 7.10.1
-          port: 9250
+      - name: Start containers
+        run: |
+          docker compose up elasticsearch_test -d
+          sleep 15
+
       - name: Tests
         run: bundle exec rspec
 

data/CHANGELOG.md

@@ -8,6 +8,29 @@
 
 ### Bugs Fixed
 
+## 8.0.0-beta (2024-08-27)
+
+### New Features
+
+* [#962](https://github.com/toptal/chewy/pull/962): ElasticSearch v.8 support added
+
+* `delete_all_enabled` setting introduced to align Chewy.massacre with wildcard indices deletion disabled in ES 8 by default
+
+### Changes
+
+### Bugs Fixed
+
+## 7.6.0 (2024-05-03)
+
+### Changes
+
+* [#933](https://github.com/toptal/chewy/pull/933): Relax allowed `elasticsearch` dependency versions. ([@mjankowski][])
+
+### Bugs Fixed
+* [#937](https://github.com/toptal/chewy/pull/937): Fix for race condition while using the `delayed_sidekiq` strategy. Also, fix for Redis bloating in case of reindexing error ([@skcc321](https://github.com/skcc321))
+
+* [#947](https://github.com/toptal/chewy/pull/947): Fix intermittent time-based failure in delayed sidekiq spec. ([@mjankowski][])
+
 ## 7.5.1 (2024-01-30)
 
 ### New Features

data/README.md

@@ -51,6 +51,7 @@ Chewy is compatible with MRI 3.0-3.2¹.
 
 | Chewy version | Elasticsearch version              |
 | ------------- | ---------------------------------- |
+| 8.0.0         | 8.x                                |
 | 7.2.x         | 7.x                                |
 | 7.1.x         | 7.x                                |
 | 7.0.x         | 6.8, 7.x                           |
@@ -97,7 +98,36 @@ development:
 Make sure you have Elasticsearch up and running. You can [install](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html) it locally, but the easiest way is to use [Docker](https://www.docker.com/get-started):
 
 ```shell
-$ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.11.1
+$ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:8.15.0
+```
+
+### Security
+
+Please note that starting from version 8 ElasticSearch has security features enabled by default.
+Docker command above has it disabled for local testing convenience.  If you want to enable it, omit
+`"xpack.security.enabled=false"` part from Docker command, and run these command after starting container (container name `es8` assumed):
+
+Reset password for `elastic` user:
+```
+docker container exec es8 '/usr/share/elasticsearch/bin/elasticsearch-reset-password' -u elastic
+```
+
+Extract CA certificate generated by ElasticSearch on first run:
+```
+docker container cp es8:/usr/share/elasticsearch/config/certs/http_ca.crt tmp/
+```
+
+And then add them to settings:
+
+```yaml
+# config/chewy.yml
+development:
+  host: 'localhost:9200'
+  user: 'elastic'
+  password: 'SomeLongPassword'
+  transport_options:
+    ssl:
+      ca_file: './tmp/http_ca.crt'
 ```
 
 ### Index
@@ -776,9 +806,12 @@ Chewy.settings[:sidekiq] = {queue: :low}
 
 #### `:delayed_sidekiq`
 
-It accumulates ids of records to be reindexed during the latency window in redis and then does the reindexing of all accumulated records at once.
-The strategy is very useful in case of frequently mutated records.
-It supports `update_fields` option, so it will try to select just enough data from the DB
+It accumulates IDs of records to be reindexed during the latency window in Redis and then performs the reindexing of all accumulated records at once. 
+This strategy is very useful in the case of frequently mutated records. 
+It supports the `update_fields` option, so it will attempt to select just enough data from the database.
+
+Keep in mind, this strategy does not guarantee reindexing in the event of Sidekiq worker termination or an error during the reindexing phase. 
+This behavior is intentional to prevent continuous growth of Redis db.
 
 There are three options that can be defined in the index:
 ```ruby
@@ -938,6 +971,8 @@ Controller actions are wrapped with the configurable value of `Chewy.request_str
 
 It is also a good idea to set up the `:bypass` strategy inside your test suite and import objects manually only when needed, and use `Chewy.massacre` when needed to flush test ES indices before every example. This will allow you to minimize unnecessary ES requests and reduce overhead.
 
+Deprecation note: since version 8 wildcard removing of indices is disabled by default. You can enable it for a cluster with setting `action.destructive_requires_name` to false.
+
 ```ruby
 RSpec.configure do |config|
   config.before(:suite) do

data/chewy.gemspec

@@ -11,13 +11,14 @@ Gem::Specification.new do |spec|
   spec.description   = 'Chewy provides functionality for Elasticsearch index handling, documents import mappings and chainable query DSL'
   spec.homepage      = 'https://github.com/toptal/chewy'
   spec.license       = 'MIT'
+  spec.required_ruby_version = '~> 3.0'
 
   spec.files         = `git ls-files`.split($RS)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x
-  spec.add_dependency 'elasticsearch', '>= 7.12.0', '< 7.14.0'
+  spec.add_dependency 'activesupport', '>= 6.1'
+  spec.add_dependency 'elasticsearch', '>= 8.14', '< 9.0'
   spec.add_dependency 'elasticsearch-dsl'
   spec.metadata['rubygems_mfa_required'] = 'true'
 end

data/docker-compose.yml

@@ -0,0 +1,15 @@
+version: "3.4"
+services:
+  elasticsearch_test:
+    image: "elasticsearch:8.15.0"
+    environment:
+      - bootstrap.memory_lock=${ES_MEMORY_LOCK:-false}
+      - "ES_JAVA_OPTS=-Xms${TEST_ES_HEAP_SIZE:-500m} -Xmx${TEST_ES_HEAP_SIZE:-500m}"
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+    ports:
+      - "127.0.0.1:9250:9200"
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536

data/gemfiles/base.gemfile

@@ -1,12 +1,12 @@
 gem 'database_cleaner'
 gem 'elasticsearch-extensions'
 gem 'method_source'
-gem 'mock_redis'
 gem 'rake'
+gem 'redis', require: false
 gem 'rspec', '>= 3.7.0'
 gem 'rspec-collection_matchers'
 gem 'rspec-its'
-gem 'rubocop', '1.60.1'
-gem 'sqlite3'
+gem 'rubocop', '1.65.1'
+gem 'sqlite3', '~> 1.4'
 gem 'timecop'
 gem 'unparser'

data/lib/chewy/errors.rb

@@ -39,4 +39,7 @@ module Chewy
 
   class ImportScopeCleanupError < Error
   end
+
+  class FeatureDisabled < Error
+  end
 end

data/lib/chewy/fields/root.rb

@@ -4,7 +4,7 @@ module Chewy
       attr_reader :dynamic_templates, :id
 
       def initialize(name, **options)
-        super(name, **options)
+        super
 
         @value ||= -> { self }
         @dynamic_templates = []

data/lib/chewy/index/actions.rb

@@ -32,7 +32,7 @@ module Chewy
         #
         def create(*args, **kwargs)
           create!(*args, **kwargs)
-        rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+        rescue Elastic::Transport::Transport::Errors::BadRequest
           false
         end
 
@@ -83,9 +83,9 @@ module Chewy
           result = client.indices.delete index: index_names.join(',')
           Chewy.wait_for_status if result
           result
-          # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
+          # es-ruby >= 1.0.10 handles Elastic::Transport::Transport::Errors::NotFound
           # by itself, rescue is for previous versions
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue Elastic::Transport::Transport::Errors::NotFound
           false
         end
 
@@ -99,9 +99,9 @@ module Chewy
         #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
         #
         def delete!(suffix = nil)
-          # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
+          # es-ruby >= 1.0.10 handles Elastic::Transport::Transport::Errors::NotFound
           # by itself, so it is raised here
-          delete(suffix) or raise Elasticsearch::Transport::Transport::Errors::NotFound
+          delete(suffix) or raise Elastic::Transport::Transport::Errors::NotFound
         end
 
         # Deletes and recreates index. Supports suffixes.

data/lib/chewy/index/aliases.rb

@@ -22,7 +22,7 @@ module Chewy
 
         def empty_if_not_found
           yield
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue Elastic::Transport::Transport::Errors::NotFound
           []
         end
       end

data/lib/chewy/index/syncer.rb

@@ -213,7 +213,7 @@ module Chewy
         @outdated_sync_field_type = mappings
           .fetch('properties', {})
           .fetch(@index.outdated_sync_field.to_s, {})['type']
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      rescue Elastic::Transport::Transport::Errors::NotFound
         nil
       end
     end

data/lib/chewy/minitest/helpers.rb

@@ -142,7 +142,7 @@ module Chewy
         teardown do
           # always destroy indexes between tests
           # Prevent croll pollution of test cases due to indexing
-          Chewy.massacre
+          drop_indices
         end
       end
     end

data/lib/chewy/search/request.rb

@@ -46,7 +46,7 @@ module Chewy
 
       delegate :hits, :wrappers, :objects, :records, :documents,
                :object_hash, :record_hash, :document_hash,
-               :total, :max_score, :took, :timed_out?, to: :response
+               :total, :max_score, :took, :timed_out?, :terminated_early?, to: :response
       delegate :each, :size, :to_a, :[], to: :wrappers
       alias_method :to_ary, :to_a
       alias_method :total_count, :total
@@ -854,7 +854,7 @@ module Chewy
         else
           Chewy.client.count(only(WHERE_STORAGES).render)['count']
         end
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      rescue Elastic::Transport::Transport::Errors::NotFound
         0
       end
 
@@ -891,7 +891,7 @@ module Chewy
       def first(limit = UNDEFINED)
         request_limit = limit == UNDEFINED ? 1 : limit
 
-        if performed? && (request_limit <= size || size == total)
+        if performed? && (terminated_early? || request_limit <= size || size == total)
           limit == UNDEFINED ? wrappers.first : wrappers.first(limit)
         else
           result = except(EXTRA_STORAGES).limit(request_limit).to_a
@@ -1035,7 +1035,7 @@ module Chewy
         request_body = render.merge(additional)
         ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: request_body) do
           Chewy.client.search(request_body)
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue Elastic::Transport::Transport::Errors::NotFound
           {}
         end
       end

data/lib/chewy/search/response.rb

@@ -47,6 +47,13 @@ module Chewy
         @timed_out ||= @body['timed_out']
       end
 
+      # Has the request been terminated early?
+      #
+      # @return [true, false]
+      def terminated_early?
+        @terminated_early ||= @body['terminated_early']
+      end
+
       # The `suggest` response part. Returns empty hash if suggests
       # were not requested.
       #

data/lib/chewy/search/scrolling.rb

@@ -39,7 +39,8 @@ module Chewy
           hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
           yield(hits) if hits.present?
           scroll_id = result['_scroll_id']
-          break if fetched >= total
+
+          break if result['terminated_early'] || fetched >= total
 
           result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
         end

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -12,13 +12,43 @@ module Chewy
     class DelayedSidekiq
       require_relative 'worker'
 
+      LUA_SCRIPT = <<~LUA
+        local timechunk_key = KEYS[1]
+        local timechunks_key = KEYS[2]
+        local serialize_data = ARGV[1]
+        local at = ARGV[2]
+        local ttl = tonumber(ARGV[3])
+
+        local schedule_job = false
+
+        -- Check if the 'sadd?' method is available
+        if redis.call('exists', 'sadd?') == 1 then
+          redis.call('sadd?', timechunk_key, serialize_data)
+        else
+          redis.call('sadd', timechunk_key, serialize_data)
+        end
+
+        -- Set expiration for timechunk_key
+        redis.call('expire', timechunk_key, ttl)
+
+        -- Check if timechunk_key exists in the sorted set
+        if not redis.call('zrank', timechunks_key, timechunk_key) then
+            -- Add timechunk_key to the sorted set
+            redis.call('zadd', timechunks_key, at, timechunk_key)
+            -- Set expiration for timechunks_key
+            redis.call('expire', timechunks_key, ttl)
+            schedule_job = true
+        end
+
+        return schedule_job
+      LUA
+
       class Scheduler
         DEFAULT_TTL = 60 * 60 * 24 # in seconds
         DEFAULT_LATENCY = 10
         DEFAULT_MARGIN = 2
         DEFAULT_QUEUE = 'chewy'
         KEY_PREFIX = 'chewy:delayed_sidekiq'
-        ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze
         FALLBACK_FIELDS = 'all'
         FIELDS_IDS_SEPARATOR = ';'
         IDS_SEPARATOR = ','
@@ -67,21 +97,8 @@ module Chewy
         #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347868
         def postpone
           ::Sidekiq.redis do |redis|
-            # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
-            if redis.respond_to?(:sadd?)
-              redis.sadd?(ALL_SETS_KEY, timechunks_key)
-              redis.sadd?(timechunk_key, serialize_data)
-            else
-              redis.sadd(ALL_SETS_KEY, timechunks_key)
-              redis.sadd(timechunk_key, serialize_data)
-            end
-
-            redis.expire(timechunk_key, ttl)
-
-            unless redis.zrank(timechunks_key, timechunk_key)
-              redis.zadd(timechunks_key, at, timechunk_key)
-              redis.expire(timechunks_key, ttl)
-
+            # do the redis stuff in a single command to avoid concurrency issues
+            if redis.eval(LUA_SCRIPT, keys: [timechunk_key, timechunks_key], argv: [serialize_data, at, ttl])
               ::Sidekiq::Client.push(
                 'queue' => sidekiq_queue,
                 'at' => at + margin,

data/lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -6,13 +6,40 @@ module Chewy
       class Worker
         include ::Sidekiq::Worker
 
+        LUA_SCRIPT = <<~LUA
+          local type = ARGV[1]
+          local score = tonumber(ARGV[2])
+          local prefix = ARGV[3]
+          local timechunks_key = prefix .. ":" .. type .. ":timechunks"
+
+          -- Get timechunk_keys with scores less than or equal to the specified score
+          local timechunk_keys = redis.call('zrangebyscore', timechunks_key, '-inf', score)
+
+          -- Get all members from the sets associated with the timechunk_keys
+          local members = {}
+          for _, timechunk_key in ipairs(timechunk_keys) do
+              local set_members = redis.call('smembers', timechunk_key)
+              for _, member in ipairs(set_members) do
+                  table.insert(members, member)
+              end
+          end
+
+          -- Remove timechunk_keys and their associated sets
+          for _, timechunk_key in ipairs(timechunk_keys) do
+              redis.call('del', timechunk_key)
+          end
+
+          -- Remove timechunks with scores less than or equal to the specified score
+          redis.call('zremrangebyscore', timechunks_key, '-inf', score)
+
+          return members
+        LUA
+
         def perform(type, score, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
 
           ::Sidekiq.redis do |redis|
-            timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks"
-            timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score)
-            members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact
+            members = redis.eval(LUA_SCRIPT, keys: [], argv: [type, score, Scheduler::KEY_PREFIX])
 
             # extract ids and fields & do the reset of records
             ids, fields = extract_ids_and_fields(members)
@@ -22,9 +49,6 @@ module Chewy
             index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
               options.any? ? index.import!(ids, **options) : index.import!(ids)
             end
-
-            redis.del(timechunk_keys)
-            redis.zremrangebyscore(timechunks_key, -1, score)
           end
         end
 

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -9,11 +9,11 @@ module Chewy
       # leak and potential flaky tests.
       def self.clear_timechunks!
         ::Sidekiq.redis do |redis|
-          timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY)
-          break if timechunk_sets.empty?
+          keys_to_delete = redis.keys("#{Scheduler::KEY_PREFIX}*")
 
-          redis.pipelined do |pipeline|
-            timechunk_sets.each { |set| pipeline.del(set) }
+          # Delete keys one by one
+          keys_to_delete.each do |key|
+            redis.del(key)
           end
         end
       end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.5.1'.freeze
+  VERSION = '8.0.0-beta'.freeze
 end

data/lib/chewy.rb

@@ -116,6 +116,10 @@ module Chewy
     # Be careful, if current prefix is blank, this will destroy all the indexes.
     #
     def massacre
+      unless Chewy.settings[:delete_all_enabled]
+        raise FeatureDisabled, 'Feature disabled by default in ES 8. You can enable it in the cluster and set `delete_all_enabled` option in settings'
+      end
+
       Chewy.client.indices.delete(index: [Chewy.configuration[:prefix], '*'].reject(&:blank?).join('_'))
       Chewy.wait_for_status
     end

data/spec/chewy/elastic_client_spec.rb

@@ -6,7 +6,7 @@ describe Chewy::ElasticClient do
     let!(:filter_previous_value) { Chewy.before_es_request_filter }
 
     before do
-      Chewy.massacre
+      drop_indices
       stub_index(:products) do
         field :id, type: :integer
       end

data/spec/chewy/fields/base_spec.rb

@@ -5,7 +5,7 @@ describe Chewy::Fields::Base do
   specify { expect(described_class.new('name', type: 'integer').options[:type]).to eq('integer') }
 
   describe '#compose' do
-    let(:field) { described_class.new(:name, value: ->(o) { o.value }) }
+    let(:field) { described_class.new(:name, value: lambda(&:value)) }
 
     specify { expect(field.compose(double(value: 'hello'))).to eq(name: 'hello') }
     specify { expect(field.compose(double(value: %w[hello world]))).to eq(name: %w[hello world]) }
@@ -23,7 +23,7 @@ describe Chewy::Fields::Base do
 
     context 'nested fields' do
       before do
-        field.children.push(described_class.new(:subname1, value: ->(o) { o.subvalue1 }))
+        field.children.push(described_class.new(:subname1, value: lambda(&:subvalue1)))
         field.children.push(described_class.new(:subname2, value: -> { subvalue2 }))
         field.children.push(described_class.new(:subname3))
       end

data/spec/chewy/fields/time_fields_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe 'Time fields' do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_index(:posts) do

data/spec/chewy/index/actions_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Index::Actions do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_index :dummies
@@ -78,12 +78,12 @@ describe Chewy::Index::Actions do
       specify do
         expect do
           DummiesIndex.create!
-        end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies/)
+        end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies/)
       end
       specify do
         expect do
           DummiesIndex.create!('2013')
-        end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/Invalid alias name \[dummies\]/)
+        end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/Invalid alias name \[dummies\]/)
       end
     end
 
@@ -100,7 +100,7 @@ describe Chewy::Index::Actions do
       specify do
         expect do
           DummiesIndex.create!('2013')
-        end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies_2013/)
+        end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies_2013/)
       end
       specify { expect(DummiesIndex.create!('2014')['acknowledged']).to eq(true) }
 
@@ -190,11 +190,11 @@ describe Chewy::Index::Actions do
   end
 
   describe '.delete!' do
-    specify { expect { DummiesIndex.delete! }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) }
+    specify { expect { DummiesIndex.delete! }.to raise_error(Elastic::Transport::Transport::Errors::NotFound) }
     specify do
       expect do
         DummiesIndex.delete!('2013')
-      end.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end.to raise_error(Elastic::Transport::Transport::Errors::NotFound)
     end
 
     context do
@@ -768,7 +768,7 @@ describe Chewy::Index::Actions do
           .to receive(:clear_cache)
           .and_call_original
         expect { CitiesIndex.clear_cache({index: unexisted_index_name}) }
-          .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound
+          .to raise_error Elastic::Transport::Transport::Errors::NotFound
       end
     end
 
@@ -820,7 +820,7 @@ describe Chewy::Index::Actions do
             .to receive(:reindex)
             .and_call_original
           expect { CitiesIndex.reindex(source: unexisting_index, dest: dest_index_with_prefix) }
-            .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound
+            .to raise_error Elastic::Transport::Transport::Errors::NotFound
         end
       end
 
@@ -883,7 +883,7 @@ describe Chewy::Index::Actions do
       context 'index name' do
         specify do
           expect { CitiesIndex.update_mapping(unexisting_index, body_hash) }
-            .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound
+            .to raise_error Elastic::Transport::Transport::Errors::NotFound
         end
       end
 

data/spec/chewy/index/aliases_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Index::Aliases do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before { stub_index :dummies }
 

data/spec/chewy/index/import/bulk_builder_spec.rb

@@ -17,7 +17,7 @@ SimpleComment = Class.new do
 end
 
 describe Chewy::Index::Import::BulkBuilder do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   subject { described_class.new(index, to_index: to_index, delete: delete, fields: fields) }
   let(:index) { CitiesIndex }
@@ -216,7 +216,7 @@ describe Chewy::Index::Import::BulkBuilder do
       end
 
       def do_raw_index_comment(options:, data:)
-        CommentsIndex.client.index(options.merge(index: 'comments', type: '_doc', refresh: true, body: data))
+        CommentsIndex.client.index(options.merge(index: 'comments', refresh: true, body: data))
       end
 
       def raw_index_comment(comment)

data/spec/chewy/index/import/bulk_request_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Index::Import::BulkRequest do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   subject { described_class.new(index, suffix: suffix, bulk_size: bulk_size, **bulk_options) }
   let(:suffix) {}

data/spec/chewy/index/import/routine_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 # TODO: add more specs here later
 describe Chewy::Index::Import::Routine do
-  before { Chewy.massacre }
+  before { drop_indices }
   before do
     stub_index(:cities) do
       field :name

data/spec/chewy/index/import_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Index::Import do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_model(:city)
@@ -204,10 +204,10 @@ describe Chewy::Index::Import do
           end
         end
 
-        let(:mapper_parsing_exception) do
+        let(:document_parsing_exception) do
           {
-            'type' => 'mapper_parsing_exception',
-            'reason' => 'object mapping for [name] tried to parse field [name] as object, but found a concrete value'
+            'type' => 'document_parsing_exception',
+            'reason' => '[1:9] object mapping for [name] tried to parse field [name] as object, but found a concrete value'
           }
         end
 
@@ -215,7 +215,7 @@ describe Chewy::Index::Import do
           payload = subscribe_notification
           import dummy_cities, batch_size: 2
           expect(payload).to eq(index: CitiesIndex,
-                                errors: {index: {mapper_parsing_exception => %w[1 2 3]}},
+                                errors: {index: {document_parsing_exception => %w[1 2 3]}},
                                 import: {index: 3})
         end
       end
@@ -270,8 +270,8 @@ describe Chewy::Index::Import do
         expect(payload).to eq(
           errors: {
             index: {{
-              'type' => 'mapper_parsing_exception',
-              'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value'
+              'type' => 'document_parsing_exception',
+              'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value'
             } => %w[2 4]}
           },
           import: {index: 6},
@@ -293,8 +293,8 @@ describe Chewy::Index::Import do
         expect(payload).to eq(
           errors: {
             index: {{
-              'type' => 'mapper_parsing_exception',
-              'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value'
+              'type' => 'document_parsing_exception',
+              'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value'
             } => %w[2 4]}
           },
           import: {index: 6},
@@ -319,8 +319,8 @@ describe Chewy::Index::Import do
           expect(payload).to eq(
             errors: {
               index: {{
-                'type' => 'mapper_parsing_exception',
-                'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value'
+                'type' => 'document_parsing_exception',
+                'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value'
               } => %w[2 4]}
             },
             import: {index: 6},
@@ -383,8 +383,8 @@ describe Chewy::Index::Import do
 
           # Full match doesn't work here.
           expect(payload[:errors][:update].keys).to match([
-            hash_including('type' => 'document_missing_exception', 'reason' => '[_doc][1]: document missing'),
-            hash_including('type' => 'document_missing_exception', 'reason' => '[_doc][3]: document missing')
+            hash_including('type' => 'document_missing_exception', 'reason' => '[1]: document missing'),
+            hash_including('type' => 'document_missing_exception', 'reason' => '[3]: document missing')
           ])
           expect(payload[:errors][:update].values).to eq([['1'], ['3']])
           expect(imported_cities).to match_array([
@@ -431,8 +431,8 @@ describe Chewy::Index::Import do
           expect(payload).to eq(
             errors: {
               update: {{
-                'type' => 'mapper_parsing_exception',
-                'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value'
+                'type' => 'document_parsing_exception',
+                'reason' => '[1:26] object mapping for [object] tried to parse field [object] as object, but found a concrete value'
               } => %w[2 4]}
             },
             import: {index: 6},

data/spec/chewy/index/observe/callback_spec.rb

@@ -21,7 +21,7 @@ describe Chewy::Index::Observe::Callback do
     end
 
     context 'when executable is has arity 1' do
-      let(:executable) { ->(record) { record.population } }
+      let(:executable) { lambda(&:population) }
 
       it 'calls exectuable within context' do
         expect(callback.call(city)).to eq(city.population)

data/spec/chewy/index/specification_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Index::Specification do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   let(:index1) do
     stub_index(:places) do
@@ -46,7 +46,6 @@ describe Chewy::Index::Specification do
     specify do
       expect { specification1.lock! }.to change { Chewy::Stash::Specification.all.hits }.from([]).to([{
         '_index' => 'chewy_specifications',
-        '_type' => '_doc',
         '_id' => 'places',
         '_score' => 1.0,
         '_source' => {'specification' => Base64.encode64({
@@ -62,7 +61,6 @@ describe Chewy::Index::Specification do
       specify do
         expect { specification5.lock! }.to change { Chewy::Stash::Specification.all.hits }.to([{
           '_index' => 'chewy_specifications',
-          '_type' => '_doc',
           '_id' => 'places',
           '_score' => 1.0,
           '_source' => {'specification' => Base64.encode64({
@@ -71,7 +69,6 @@ describe Chewy::Index::Specification do
           }.to_json)}
         }, {
           '_index' => 'chewy_specifications',
-          '_type' => '_doc',
           '_id' => 'namespace/cities',
           '_score' => 1.0,
           '_source' => {'specification' => Base64.encode64({

data/spec/chewy/index/syncer_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Index::Syncer, :orm do
-  before { Chewy.massacre }
+  before { drop_indices }
   before do
     stub_model(:city)
     stub_index(:cities) do

data/spec/chewy/index_spec.rb

@@ -177,7 +177,7 @@ describe Chewy::Index do
 
     context do
       before do
-        Chewy.massacre
+        drop_indices
         PlacesIndex.import!(
           double(colors: ['red']),
           double(colors: %w[red green]),

data/spec/chewy/journal_spec.rb

@@ -21,7 +21,7 @@ describe Chewy::Journal do
             default_import_options journal: true
           end
 
-          Chewy.massacre
+          drop_indices
           Chewy.settings[:prefix] = 'some_prefix'
           Timecop.freeze(time)
         end
@@ -145,7 +145,7 @@ describe Chewy::Journal do
   end
 
   context do
-    before { Chewy.massacre }
+    before { drop_indices }
     before do
       stub_model(:city) do
         update_index 'cities', :self

data/spec/chewy/minitest/helpers_spec.rb

@@ -17,7 +17,7 @@ describe :minitest_helper do
   end
 
   before do
-    Chewy.massacre
+    drop_indices
   end
 
   before do

data/spec/chewy/multi_search_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 require 'chewy/multi_search'
 
 describe Chewy::MultiSearch do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_model(:city)

data/spec/chewy/rake_helper_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 require 'rake'
 
 describe Chewy::RakeHelper, :orm do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     described_class.instance_variable_set(:@journal_exists, journal_exists)

data/spec/chewy/rspec/update_index_spec.rb

@@ -9,7 +9,7 @@ describe :update_index do
   end
 
   before do
-    Chewy.massacre
+    drop_indices
     DummiesIndex.create!
   end
 

data/spec/chewy/runtime_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Chewy::Runtime do
   describe '.version' do
     specify { expect(described_class.version).to be_a(described_class::Version) }
-    specify { expect(described_class.version).to be >= '7.0' }
-    specify { expect(described_class.version).to be < '8.0' }
+    specify { expect(described_class.version).to be >= '8.0' }
+    specify { expect(described_class.version).to be < '9.0' }
   end
 end

data/spec/chewy/search/loader_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Search::Loader do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_model(:city)

data/spec/chewy/search/pagination/kaminari_examples.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 shared_examples :kaminari do |request_base_class|
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_index(:products) do

data/spec/chewy/search/request_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Search::Request do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_index(:products) do

data/spec/chewy/search/response_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Search::Response, :orm do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_model(:city)
@@ -39,7 +39,7 @@ describe Chewy::Search::Response, :orm do
     specify { expect(subject.hits).to all be_a(Hash) }
     specify do
       expect(subject.hits.flat_map(&:keys).uniq)
-        .to match_array(%w[_id _index _type _score _source sort])
+        .to match_array(%w[_id _index _score _source sort])
     end
 
     context do

data/spec/chewy/search/scrolling_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Search::Scrolling, :orm do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_model(:city)

data/spec/chewy/search_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Search do
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_index(:products)

data/spec/chewy/stash_spec.rb

@@ -5,7 +5,7 @@ describe Chewy::Stash::Journal, :orm do
     response['deleted'] || response['_indices']['_all']['deleted']
   end
 
-  before { Chewy.massacre }
+  before { drop_indices }
 
   before do
     stub_model(:city)

data/spec/chewy/strategy/delayed_sidekiq_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 if defined?(Sidekiq)
   require 'sidekiq/testing'
-  require 'mock_redis'
+  require 'redis'
 
   describe Chewy::Strategy::DelayedSidekiq do
     around do |example|
@@ -10,9 +10,10 @@ if defined?(Sidekiq)
     end
 
     before do
-      redis = MockRedis.new
+      redis = Redis.new
       allow(Sidekiq).to receive(:redis).and_yield(redis)
       Sidekiq::Worker.clear_all
+      described_class.clear_timechunks!
     end
 
     before do
@@ -35,7 +36,7 @@ if defined?(Sidekiq)
 
     it "respects 'refresh: false' options" do
       allow(Chewy).to receive(:disable_refresh_async).and_return(true)
-      expect(CitiesIndex).to receive(:import!).with([city.id, other_city.id], refresh: false)
+      expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), refresh: false)
       scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id, other_city.id])
       scheduler.postpone
       Chewy::Strategy::DelayedSidekiq::Worker.drain
@@ -47,7 +48,7 @@ if defined?(Sidekiq)
           expect(Sidekiq::Client).to receive(:push).with(
             hash_including(
               'queue' => 'chewy',
-              'at' => (Time.current.to_i.ceil(-1) + 2.seconds).to_i,
+              'at' => expected_at_time.to_i,
               'class' => Chewy::Strategy::DelayedSidekiq::Worker,
               'args' => ['CitiesIndex', an_instance_of(Integer)]
             )
@@ -62,6 +63,11 @@ if defined?(Sidekiq)
           end
         end
       end
+
+      def expected_at_time
+        target = described_class::Scheduler::DEFAULT_LATENCY.seconds.from_now.to_i
+        target - (target % described_class::Scheduler::DEFAULT_LATENCY) + described_class::Scheduler::DEFAULT_MARGIN.seconds
+      end
     end
 
     context 'with custom config' do
@@ -103,7 +109,7 @@ if defined?(Sidekiq)
     context 'two reindex call within the timewindow' do
       it 'accumulates all ids does the reindex one time' do
         Timecop.freeze do
-          expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once
+          expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once
           scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id])
           scheduler.postpone
           scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
@@ -115,7 +121,7 @@ if defined?(Sidekiq)
       context 'one call with update_fields another one without update_fields' do
         it 'does reindex of all fields' do
           Timecop.freeze do
-            expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once
+            expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
             scheduler.postpone
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
@@ -128,7 +134,7 @@ if defined?(Sidekiq)
       context 'both calls with different update fields' do
         it 'deos reindex with union of fields' do
           Timecop.freeze do
-            expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id], update_fields: %w[description name]).once
+            expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: match_array(%w[name description])).once
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
             scheduler.postpone
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description'])

data/spec/chewy/strategy_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Chewy::Strategy do
-  before { Chewy.massacre }
+  before { drop_indices }
   subject(:strategy) { Chewy::Strategy.new }
 
   describe '#current' do

data/spec/chewy_spec.rb

@@ -31,8 +31,8 @@ describe Chewy do
     end
   end
 
-  describe '.massacre' do
-    before { Chewy.massacre }
+  xdescribe '.massacre' do
+    before { drop_indices }
 
     before do
       allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix1'))
@@ -40,7 +40,7 @@ describe Chewy do
       allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix2'))
       stub_index(:developers).create!
 
-      Chewy.massacre
+      drop_indices
 
       allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix1'))
     end
@@ -84,7 +84,8 @@ describe Chewy do
       # To avoid flaky issues when previous specs were run
       allow(Chewy::Index).to receive(:descendants).and_return([CitiesIndex, PlacesIndex])
 
-      Chewy.massacre
+      CitiesIndex.delete
+      PlacesIndex.delete
     end
 
     specify do
@@ -111,7 +112,7 @@ describe Chewy do
       expect(CitiesIndex.exists?).to eq true
       expect(PlacesIndex.exists?).to eq true
 
-      expect { Chewy.create_indices! }.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest)
+      expect { Chewy.create_indices! }.to raise_error(Elastic::Transport::Transport::Errors::BadRequest)
     end
   end
 end

data/spec/spec_helper.rb

@@ -28,6 +28,32 @@ Chewy.settings = {
   }
 }
 
+# To work with security enabled:
+#
+# user = ENV['ES_USER'] || 'elastic'
+# password = ENV['ES_PASSWORD'] || ''
+# ca_cert = ENV['ES_CA_CERT'] || './tmp/http_ca.crt'
+#
+# Chewy.settings.merge!(
+#   user: user,
+#   password: password,
+#   transport_options: {
+#     ssl: {
+#       ca_file: ca_cert
+#     }
+#   }
+# )
+
+# Low-level substitute for now-obsolete drop_indices
+def drop_indices
+  response = Chewy.client.cat.indices
+  indices = response.body.lines.map { |line| line.split[2] }
+  return if indices.blank?
+
+  Chewy.client.indices.delete(index: indices)
+  Chewy.wait_for_status
+end
+
 # Chewy.transport_logger = Logger.new(STDERR)
 
 RSpec.configure do |config|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 7.5.1
+  version: 8.0.0.pre.beta
 platform: ruby
 authors:
 - Toptal, LLC
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-30 00:00:00.000000000 Z
+date: 2024-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -17,34 +17,34 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.12.0
+        version: '8.14'
     - - "<"
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: '9.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.12.0
+        version: '8.14'
     - - "<"
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: '9.0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch-dsl
   requirement: !ruby/object:Gem::Requirement
@@ -88,6 +88,7 @@ files:
 - README.md
 - Rakefile
 - chewy.gemspec
+- docker-compose.yml
 - filters
 - gemfiles/base.gemfile
 - gemfiles/rails.6.1.activerecord.gemfile
@@ -311,16 +312,16 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - "~>"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.2.33
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Elasticsearch ODM client wrapper