chewy 7.2.4 → 7.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -0
- data/.github/dependabot.yml +42 -0
- data/.github/workflows/ruby.yml +26 -32
- data/.rubocop.yml +4 -1
- data/CHANGELOG.md +144 -0
- data/Gemfile +4 -4
- data/README.md +165 -10
- data/chewy.gemspec +4 -17
- data/gemfiles/base.gemfile +12 -0
- data/gemfiles/rails.6.1.activerecord.gemfile +2 -1
- data/gemfiles/rails.7.0.activerecord.gemfile +2 -1
- data/gemfiles/{rails.5.2.activerecord.gemfile → rails.7.1.activerecord.gemfile} +6 -3
- data/lib/chewy/config.rb +22 -14
- data/lib/chewy/elastic_client.rb +31 -0
- data/lib/chewy/errors.rb +5 -2
- data/lib/chewy/fields/base.rb +1 -1
- data/lib/chewy/fields/root.rb +1 -1
- data/lib/chewy/index/adapter/active_record.rb +13 -3
- data/lib/chewy/index/adapter/object.rb +3 -3
- data/lib/chewy/index/adapter/orm.rb +2 -2
- data/lib/chewy/index/crutch.rb +15 -7
- data/lib/chewy/index/import/bulk_builder.rb +6 -7
- data/lib/chewy/index/import/routine.rb +1 -1
- data/lib/chewy/index/import.rb +31 -4
- data/lib/chewy/index/observe/active_record_methods.rb +87 -0
- data/lib/chewy/index/observe/callback.rb +34 -0
- data/lib/chewy/index/observe.rb +3 -58
- data/lib/chewy/index/syncer.rb +1 -1
- data/lib/chewy/index.rb +25 -0
- data/lib/chewy/journal.rb +17 -6
- data/lib/chewy/log_subscriber.rb +5 -1
- data/lib/chewy/minitest/helpers.rb +1 -1
- data/lib/chewy/minitest/search_index_receiver.rb +3 -1
- data/lib/chewy/rake_helper.rb +74 -13
- data/lib/chewy/rspec/update_index.rb +13 -6
- data/lib/chewy/runtime/version.rb +1 -1
- data/lib/chewy/search/parameters/collapse.rb +16 -0
- data/lib/chewy/search/parameters/indices.rb +1 -1
- data/lib/chewy/search/parameters/knn.rb +16 -0
- data/lib/chewy/search/parameters/storage.rb +1 -1
- data/lib/chewy/search/parameters.rb +3 -3
- data/lib/chewy/search/request.rb +45 -11
- data/lib/chewy/search.rb +6 -3
- data/lib/chewy/stash.rb +3 -3
- data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
- data/lib/chewy/strategy/base.rb +10 -0
- data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +168 -0
- data/lib/chewy/strategy/delayed_sidekiq/worker.rb +76 -0
- data/lib/chewy/strategy/delayed_sidekiq.rb +30 -0
- data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
- data/lib/chewy/strategy.rb +3 -0
- data/lib/chewy/version.rb +1 -1
- data/lib/chewy.rb +5 -8
- data/lib/tasks/chewy.rake +17 -1
- data/migration_guide.md +1 -1
- data/spec/chewy/config_spec.rb +2 -2
- data/spec/chewy/elastic_client_spec.rb +26 -0
- data/spec/chewy/fields/base_spec.rb +1 -0
- data/spec/chewy/index/actions_spec.rb +4 -4
- data/spec/chewy/index/adapter/active_record_spec.rb +62 -0
- data/spec/chewy/index/import/bulk_builder_spec.rb +7 -3
- data/spec/chewy/index/import_spec.rb +16 -3
- data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
- data/spec/chewy/index/observe/callback_spec.rb +139 -0
- data/spec/chewy/index/observe_spec.rb +27 -0
- data/spec/chewy/journal_spec.rb +13 -49
- data/spec/chewy/minitest/helpers_spec.rb +3 -3
- data/spec/chewy/minitest/search_index_receiver_spec.rb +6 -4
- data/spec/chewy/rake_helper_spec.rb +155 -4
- data/spec/chewy/rspec/helpers_spec.rb +1 -1
- data/spec/chewy/search/pagination/kaminari_examples.rb +1 -1
- data/spec/chewy/search/pagination/kaminari_spec.rb +1 -1
- data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
- data/spec/chewy/search/parameters/knn_spec.rb +5 -0
- data/spec/chewy/search/request_spec.rb +37 -0
- data/spec/chewy/search_spec.rb +9 -0
- data/spec/chewy/strategy/active_job_spec.rb +8 -8
- data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
- data/spec/chewy/strategy/delayed_sidekiq_spec.rb +208 -0
- data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
- data/spec/chewy/strategy/sidekiq_spec.rb +4 -4
- data/spec/chewy_spec.rb +7 -4
- data/spec/spec_helper.rb +1 -1
- metadata +32 -253
- data/gemfiles/rails.6.0.activerecord.gemfile +0 -11
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../../index'
|
|
4
|
+
|
|
5
|
+
# The class is responsible for accumulating in redis [type, ids]
|
|
6
|
+
# that were requested to be reindexed during `latency` seconds.
|
|
7
|
+
# The reindex job is going to be scheduled after a `latency` seconds.
|
|
8
|
+
# that job is going to read accumulated [type, ids] from the redis
|
|
9
|
+
# and reindex all them at once.
|
|
10
|
+
module Chewy
|
|
11
|
+
class Strategy
|
|
12
|
+
class DelayedSidekiq
|
|
13
|
+
require_relative 'worker'
|
|
14
|
+
|
|
15
|
+
LUA_SCRIPT = <<~LUA
|
|
16
|
+
local timechunk_key = KEYS[1]
|
|
17
|
+
local timechunks_key = KEYS[2]
|
|
18
|
+
local serialize_data = ARGV[1]
|
|
19
|
+
local at = ARGV[2]
|
|
20
|
+
local ttl = tonumber(ARGV[3])
|
|
21
|
+
|
|
22
|
+
local schedule_job = false
|
|
23
|
+
|
|
24
|
+
-- Check if the 'sadd?' method is available
|
|
25
|
+
if redis.call('exists', 'sadd?') == 1 then
|
|
26
|
+
redis.call('sadd?', timechunk_key, serialize_data)
|
|
27
|
+
else
|
|
28
|
+
redis.call('sadd', timechunk_key, serialize_data)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
-- Set expiration for timechunk_key
|
|
32
|
+
redis.call('expire', timechunk_key, ttl)
|
|
33
|
+
|
|
34
|
+
-- Check if timechunk_key exists in the sorted set
|
|
35
|
+
if not redis.call('zrank', timechunks_key, timechunk_key) then
|
|
36
|
+
-- Add timechunk_key to the sorted set
|
|
37
|
+
redis.call('zadd', timechunks_key, at, timechunk_key)
|
|
38
|
+
-- Set expiration for timechunks_key
|
|
39
|
+
redis.call('expire', timechunks_key, ttl)
|
|
40
|
+
schedule_job = true
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
return schedule_job
|
|
44
|
+
LUA
|
|
45
|
+
|
|
46
|
+
class Scheduler
|
|
47
|
+
DEFAULT_TTL = 60 * 60 * 24 # in seconds
|
|
48
|
+
DEFAULT_LATENCY = 10
|
|
49
|
+
DEFAULT_MARGIN = 2
|
|
50
|
+
DEFAULT_QUEUE = 'chewy'
|
|
51
|
+
KEY_PREFIX = 'chewy:delayed_sidekiq'
|
|
52
|
+
FALLBACK_FIELDS = 'all'
|
|
53
|
+
FIELDS_IDS_SEPARATOR = ';'
|
|
54
|
+
IDS_SEPARATOR = ','
|
|
55
|
+
|
|
56
|
+
def initialize(type, ids, options = {})
|
|
57
|
+
@type = type
|
|
58
|
+
@ids = ids
|
|
59
|
+
@options = options
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# the diagram:
|
|
63
|
+
#
|
|
64
|
+
# inputs:
|
|
65
|
+
# latency == 2
|
|
66
|
+
# reindex_time = Time.current
|
|
67
|
+
#
|
|
68
|
+
# Parallel OR Sequential triggers of reindex: | What is going on in reindex store (Redis):
|
|
69
|
+
# --------------------------------------------------------------------------------------------------
|
|
70
|
+
# |
|
|
71
|
+
# process 1 (reindex_time): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1]
|
|
72
|
+
# Schedule.new(CitiesIndex, [1]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
|
|
73
|
+
# | & schedule a DelayedSidekiq::Worker at 1679347869 (at + 3)
|
|
74
|
+
# | it will zpop chewy:delayed_sidekiq:timechunks up to 1679347866 score and reindex all ids with zpoped keys
|
|
75
|
+
# | chewy:delayed_sidekiq:CitiesIndex:1679347866
|
|
76
|
+
# |
|
|
77
|
+
# |
|
|
78
|
+
# process 2 (reindex_time): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2]
|
|
79
|
+
# Schedule.new(CitiesIndex, [2]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
|
|
80
|
+
# | & do not schedule a new worker
|
|
81
|
+
# |
|
|
82
|
+
# |
|
|
83
|
+
# process 1 (reindex_time + (latency - 1).seconds): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
|
|
84
|
+
# Schedule.new(CitiesIndex, [3]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
|
|
85
|
+
# | & do not schedule a new worker
|
|
86
|
+
# |
|
|
87
|
+
# |
|
|
88
|
+
# process 2 (reindex_time + (latency + 1).seconds): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
|
|
89
|
+
# Schedule.new(CitiesIndex, [4]).postpone | chewy:delayed_sidekiq:CitiesIndex:1679347868 = [4]
|
|
90
|
+
# | chewy:delayed_sidekiq:timechunks = [
|
|
91
|
+
# | { score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}
|
|
92
|
+
# | { score: 1679347868, "chewy:delayed_sidekiq:CitiesIndex:1679347868"}
|
|
93
|
+
# | ]
|
|
94
|
+
# | & schedule a DelayedSidekiq::Worker at 1679347871 (at + 3)
|
|
95
|
+
# | it will zpop chewy:delayed_sidekiq:timechunks up to 1679347868 score and reindex all ids with zpoped keys
|
|
96
|
+
# | chewy:delayed_sidekiq:CitiesIndex:1679347866 (in case of failed previous reindex),
|
|
97
|
+
# | chewy:delayed_sidekiq:CitiesIndex:1679347868
|
|
98
|
+
def postpone
|
|
99
|
+
::Sidekiq.redis do |redis|
|
|
100
|
+
# do the redis stuff in a single command to avoid concurrency issues
|
|
101
|
+
if redis.eval(LUA_SCRIPT, keys: [timechunk_key, timechunks_key], argv: [serialize_data, at, ttl])
|
|
102
|
+
::Sidekiq::Client.push(
|
|
103
|
+
'queue' => sidekiq_queue,
|
|
104
|
+
'at' => at + margin,
|
|
105
|
+
'class' => Chewy::Strategy::DelayedSidekiq::Worker,
|
|
106
|
+
'args' => [type_name, at]
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
private
|
|
113
|
+
|
|
114
|
+
attr_reader :type, :ids, :options
|
|
115
|
+
|
|
116
|
+
# this method returns predictable value that jumps by latency value
|
|
117
|
+
# another words each latency seconds it return the same value
|
|
118
|
+
def at
|
|
119
|
+
@at ||= begin
|
|
120
|
+
schedule_at = latency.seconds.from_now.to_f
|
|
121
|
+
|
|
122
|
+
(schedule_at - (schedule_at % latency)).to_i
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def fields
|
|
127
|
+
options[:update_fields].presence || [FALLBACK_FIELDS]
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def timechunks_key
|
|
131
|
+
"#{KEY_PREFIX}:#{type_name}:timechunks"
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def timechunk_key
|
|
135
|
+
"#{KEY_PREFIX}:#{type_name}:#{at}"
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def serialize_data
|
|
139
|
+
[ids.join(IDS_SEPARATOR), fields.join(IDS_SEPARATOR)].join(FIELDS_IDS_SEPARATOR)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def type_name
|
|
143
|
+
type.name
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def latency
|
|
147
|
+
strategy_config.latency || DEFAULT_LATENCY
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def margin
|
|
151
|
+
strategy_config.margin || DEFAULT_MARGIN
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def ttl
|
|
155
|
+
strategy_config.ttl || DEFAULT_TTL
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def sidekiq_queue
|
|
159
|
+
Chewy.settings.dig(:sidekiq, :queue) || DEFAULT_QUEUE
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def strategy_config
|
|
163
|
+
type.strategy_config.delayed_sidekiq
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Chewy
|
|
4
|
+
class Strategy
|
|
5
|
+
class DelayedSidekiq
|
|
6
|
+
class Worker
|
|
7
|
+
include ::Sidekiq::Worker
|
|
8
|
+
|
|
9
|
+
LUA_SCRIPT = <<~LUA
|
|
10
|
+
local type = ARGV[1]
|
|
11
|
+
local score = tonumber(ARGV[2])
|
|
12
|
+
local prefix = ARGV[3]
|
|
13
|
+
local timechunks_key = prefix .. ":" .. type .. ":timechunks"
|
|
14
|
+
|
|
15
|
+
-- Get timechunk_keys with scores less than or equal to the specified score
|
|
16
|
+
local timechunk_keys = redis.call('zrangebyscore', timechunks_key, '-inf', score)
|
|
17
|
+
|
|
18
|
+
-- Get all members from the sets associated with the timechunk_keys
|
|
19
|
+
local members = {}
|
|
20
|
+
for _, timechunk_key in ipairs(timechunk_keys) do
|
|
21
|
+
local set_members = redis.call('smembers', timechunk_key)
|
|
22
|
+
for _, member in ipairs(set_members) do
|
|
23
|
+
table.insert(members, member)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
-- Remove timechunk_keys and their associated sets
|
|
28
|
+
for _, timechunk_key in ipairs(timechunk_keys) do
|
|
29
|
+
redis.call('del', timechunk_key)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
-- Remove timechunks with scores less than or equal to the specified score
|
|
33
|
+
redis.call('zremrangebyscore', timechunks_key, '-inf', score)
|
|
34
|
+
|
|
35
|
+
return members
|
|
36
|
+
LUA
|
|
37
|
+
|
|
38
|
+
def perform(type, score, options = {})
|
|
39
|
+
options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
|
|
40
|
+
|
|
41
|
+
::Sidekiq.redis do |redis|
|
|
42
|
+
members = redis.eval(LUA_SCRIPT, keys: [], argv: [type, score, Scheduler::KEY_PREFIX])
|
|
43
|
+
|
|
44
|
+
# extract ids and fields & do the reset of records
|
|
45
|
+
ids, fields = extract_ids_and_fields(members)
|
|
46
|
+
options[:update_fields] = fields if fields
|
|
47
|
+
|
|
48
|
+
index = type.constantize
|
|
49
|
+
index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
|
|
50
|
+
options.any? ? index.import!(ids, **options) : index.import!(ids)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def extract_ids_and_fields(members)
|
|
58
|
+
ids = []
|
|
59
|
+
fields = []
|
|
60
|
+
|
|
61
|
+
members.each do |member|
|
|
62
|
+
member_ids, member_fields = member.split(Scheduler::FIELDS_IDS_SEPARATOR).map do |v|
|
|
63
|
+
v.split(Scheduler::IDS_SEPARATOR)
|
|
64
|
+
end
|
|
65
|
+
ids |= member_ids
|
|
66
|
+
fields |= member_fields
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS)
|
|
70
|
+
|
|
71
|
+
[ids.map(&:to_i), fields]
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Chewy
|
|
4
|
+
class Strategy
|
|
5
|
+
class DelayedSidekiq < Sidekiq
|
|
6
|
+
require_relative 'delayed_sidekiq/scheduler'
|
|
7
|
+
|
|
8
|
+
# cleanup the redis sets used internally. Useful mainly in tests to avoid
|
|
9
|
+
# leak and potential flaky tests.
|
|
10
|
+
def self.clear_timechunks!
|
|
11
|
+
::Sidekiq.redis do |redis|
|
|
12
|
+
keys_to_delete = redis.keys("#{Scheduler::KEY_PREFIX}*")
|
|
13
|
+
|
|
14
|
+
# Delete keys one by one
|
|
15
|
+
keys_to_delete.each do |key|
|
|
16
|
+
redis.del(key)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def leave
|
|
22
|
+
@stash.each do |type, ids|
|
|
23
|
+
next if ids.empty?
|
|
24
|
+
|
|
25
|
+
DelayedSidekiq::Scheduler.new(type, ids).postpone
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
module Chewy
|
|
2
|
+
class Strategy
|
|
3
|
+
# The strategy works the same way as sidekiq, but performs
|
|
4
|
+
# async evaluation of all index callbacks on model create and update
|
|
5
|
+
# driven by sidekiq
|
|
6
|
+
#
|
|
7
|
+
# Chewy.strategy(:lazy_sidekiq) do
|
|
8
|
+
# User.all.map(&:save) # Does nothing here
|
|
9
|
+
# Post.all.map(&:save) # And here
|
|
10
|
+
# # It schedules import of all the changed users and posts right here
|
|
11
|
+
# end
|
|
12
|
+
#
|
|
13
|
+
class LazySidekiq < Sidekiq
|
|
14
|
+
class IndicesUpdateWorker
|
|
15
|
+
include ::Sidekiq::Worker
|
|
16
|
+
|
|
17
|
+
def perform(models)
|
|
18
|
+
Chewy.strategy(strategy) do
|
|
19
|
+
models.each do |model_type, model_ids|
|
|
20
|
+
model_type.constantize.where(id: model_ids).each(&:run_chewy_callbacks)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
def strategy
|
|
28
|
+
Chewy.disable_refresh_async ? :atomic_no_refresh : :atomic
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def initialize
|
|
33
|
+
# Use parent's @stash to store destroyed records, since callbacks for them have to
|
|
34
|
+
# be run immediately on the strategy block end because we won't be able to fetch
|
|
35
|
+
# records further in IndicesUpdateWorker. This will be done by avoiding of
|
|
36
|
+
# LazySidekiq#update_chewy_indices call and calling LazySidekiq#update instead.
|
|
37
|
+
super
|
|
38
|
+
|
|
39
|
+
# @lazy_stash is used to store all the lazy evaluated callbacks with call of
|
|
40
|
+
# strategy's #update_chewy_indices.
|
|
41
|
+
@lazy_stash = {}
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def leave
|
|
45
|
+
# Fallback to Sidekiq#leave implementation for destroyed records stored in @stash.
|
|
46
|
+
super
|
|
47
|
+
|
|
48
|
+
# Proceed with other records stored in @lazy_stash
|
|
49
|
+
return if @lazy_stash.empty?
|
|
50
|
+
|
|
51
|
+
::Sidekiq::Client.push(
|
|
52
|
+
'queue' => sidekiq_queue,
|
|
53
|
+
'class' => Chewy::Strategy::LazySidekiq::IndicesUpdateWorker,
|
|
54
|
+
'args' => [@lazy_stash]
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def update_chewy_indices(object)
|
|
59
|
+
@lazy_stash[object.class.name] ||= []
|
|
60
|
+
@lazy_stash[object.class.name] |= Array.wrap(object.id)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
data/lib/chewy/strategy.rb
CHANGED
|
@@ -2,10 +2,13 @@ require 'chewy/strategy/base'
|
|
|
2
2
|
require 'chewy/strategy/bypass'
|
|
3
3
|
require 'chewy/strategy/urgent'
|
|
4
4
|
require 'chewy/strategy/atomic'
|
|
5
|
+
require 'chewy/strategy/atomic_no_refresh'
|
|
5
6
|
|
|
6
7
|
begin
|
|
7
8
|
require 'sidekiq'
|
|
8
9
|
require 'chewy/strategy/sidekiq'
|
|
10
|
+
require 'chewy/strategy/lazy_sidekiq'
|
|
11
|
+
require 'chewy/strategy/delayed_sidekiq'
|
|
9
12
|
rescue LoadError
|
|
10
13
|
nil
|
|
11
14
|
end
|
data/lib/chewy/version.rb
CHANGED
data/lib/chewy.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
require 'active_support'
|
|
1
2
|
require 'active_support/version'
|
|
2
3
|
require 'active_support/concern'
|
|
3
4
|
require 'active_support/deprecation'
|
|
@@ -47,10 +48,11 @@ require 'chewy/index'
|
|
|
47
48
|
require 'chewy/fields/base'
|
|
48
49
|
require 'chewy/fields/root'
|
|
49
50
|
require 'chewy/journal'
|
|
50
|
-
require 'chewy/railtie' if defined?(
|
|
51
|
+
require 'chewy/railtie' if defined?(Rails::Railtie)
|
|
52
|
+
require 'chewy/elastic_client'
|
|
51
53
|
|
|
52
54
|
ActiveSupport.on_load(:active_record) do
|
|
53
|
-
|
|
55
|
+
include Chewy::Index::Observe::ActiveRecordMethods
|
|
54
56
|
end
|
|
55
57
|
|
|
56
58
|
module Chewy
|
|
@@ -96,12 +98,7 @@ module Chewy
|
|
|
96
98
|
# Main elasticsearch-ruby client instance
|
|
97
99
|
#
|
|
98
100
|
def client
|
|
99
|
-
Chewy.current[:chewy_client] ||=
|
|
100
|
-
client_configuration = configuration.deep_dup
|
|
101
|
-
client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
|
|
102
|
-
block = client_configuration[:transport_options].try(:delete, :proc)
|
|
103
|
-
::Elasticsearch::Client.new(client_configuration, &block)
|
|
104
|
-
end
|
|
101
|
+
Chewy.current[:chewy_client] ||= Chewy::ElasticClient.new
|
|
105
102
|
end
|
|
106
103
|
|
|
107
104
|
# Sends wait_for_status request to ElasticSearch with status
|
data/lib/tasks/chewy.rake
CHANGED
|
@@ -57,6 +57,11 @@ namespace :chewy do
|
|
|
57
57
|
Chewy::RakeHelper.update_mapping(name: args[:index_name])
|
|
58
58
|
end
|
|
59
59
|
|
|
60
|
+
desc 'Creates missing indexes'
|
|
61
|
+
task create_missing_indexes: :environment do
|
|
62
|
+
Chewy::RakeHelper.create_missing_indexes!
|
|
63
|
+
end
|
|
64
|
+
|
|
60
65
|
namespace :parallel do
|
|
61
66
|
desc 'Parallel version of `rake chewy:reset`'
|
|
62
67
|
task reset: :environment do |_task, args|
|
|
@@ -87,6 +92,11 @@ namespace :chewy do
|
|
|
87
92
|
end
|
|
88
93
|
|
|
89
94
|
namespace :journal do
|
|
95
|
+
desc 'Create manually journal, useful when `skip_journal_creation_on_import` is used'
|
|
96
|
+
task create: :environment do |_task, _args|
|
|
97
|
+
Chewy::RakeHelper.journal_create
|
|
98
|
+
end
|
|
99
|
+
|
|
90
100
|
desc 'Applies changes that were done after the specified time for the specified indexes/types or all of them'
|
|
91
101
|
task apply: :environment do |_task, args|
|
|
92
102
|
Chewy::RakeHelper.journal_apply(**parse_journal_args(args.extras))
|
|
@@ -94,7 +104,13 @@ namespace :chewy do
|
|
|
94
104
|
|
|
95
105
|
desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them'
|
|
96
106
|
task clean: :environment do |_task, args|
|
|
97
|
-
Chewy::RakeHelper.
|
|
107
|
+
delete_options = Chewy::RakeHelper.delete_by_query_options_from_env(ENV)
|
|
108
|
+
Chewy::RakeHelper.journal_clean(
|
|
109
|
+
**[
|
|
110
|
+
parse_journal_args(args.extras),
|
|
111
|
+
{delete_by_query_options: delete_options}
|
|
112
|
+
].reduce({}, :merge)
|
|
113
|
+
)
|
|
98
114
|
end
|
|
99
115
|
end
|
|
100
116
|
end
|
data/migration_guide.md
CHANGED
|
@@ -9,7 +9,7 @@ Chewy alongside a matching Elasticsearch version.
|
|
|
9
9
|
In order to upgrade Chewy 6/Elasticsearch 6 to Chewy 7/Elasticsearch 7 in the most seamless manner you have to:
|
|
10
10
|
|
|
11
11
|
* Upgrade to the latest 6.x stable releases, namely Chewy 6.0, Elasticsearch 6.8
|
|
12
|
-
* Study carefully [Breaking changes in 7.0](https://www.elastic.co/guide/en/elasticsearch/reference/
|
|
12
|
+
* Study carefully [Breaking changes in 7.0](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/breaking-changes-7.0.html), make sure your application conforms.
|
|
13
13
|
* Run your test suite on Chewy 7.0 / Elasticsearch 7
|
|
14
14
|
* Run manual tests on Chewy 7.0 / Elasticsearch 7
|
|
15
15
|
* Upgrade to Chewy 7.0
|
data/spec/chewy/config_spec.rb
CHANGED
|
@@ -22,7 +22,7 @@ describe Chewy::Config do
|
|
|
22
22
|
|
|
23
23
|
specify do
|
|
24
24
|
expect { subject.transport_logger = logger }
|
|
25
|
-
.to change { Chewy.client.transport.logger }.to(logger)
|
|
25
|
+
.to change { Chewy.client.transport.transport.logger }.to(logger)
|
|
26
26
|
end
|
|
27
27
|
specify do
|
|
28
28
|
expect { subject.transport_logger = logger }
|
|
@@ -40,7 +40,7 @@ describe Chewy::Config do
|
|
|
40
40
|
|
|
41
41
|
specify do
|
|
42
42
|
expect { subject.transport_tracer = tracer }
|
|
43
|
-
.to change { Chewy.client.transport.tracer }.to(tracer)
|
|
43
|
+
.to change { Chewy.client.transport.transport.tracer }.to(tracer)
|
|
44
44
|
end
|
|
45
45
|
specify do
|
|
46
46
|
expect { subject.transport_tracer = tracer }
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
describe Chewy::ElasticClient do
|
|
4
|
+
describe 'payload inspection' do
|
|
5
|
+
let(:filter) { instance_double('Proc') }
|
|
6
|
+
let!(:filter_previous_value) { Chewy.before_es_request_filter }
|
|
7
|
+
|
|
8
|
+
before do
|
|
9
|
+
Chewy.massacre
|
|
10
|
+
stub_index(:products) do
|
|
11
|
+
field :id, type: :integer
|
|
12
|
+
end
|
|
13
|
+
ProductsIndex.create
|
|
14
|
+
Chewy.before_es_request_filter = filter
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
after do
|
|
18
|
+
Chewy.before_es_request_filter = filter_previous_value
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
it 'call filter with the request body' do
|
|
22
|
+
expect(filter).to receive(:call).with(:search, [{body: {size: 0}, index: ['products']}], {})
|
|
23
|
+
Chewy.client.search({index: ['products'], body: {size: 0}}).to_a
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -610,7 +610,7 @@ describe Chewy::Index::Actions do
|
|
|
610
610
|
specify 'with journal application' do
|
|
611
611
|
cities
|
|
612
612
|
p 'cities created1'
|
|
613
|
-
|
|
613
|
+
ActiveRecord::Base.connection.close if defined?(ActiveRecord::Base)
|
|
614
614
|
[
|
|
615
615
|
parallel_update,
|
|
616
616
|
Thread.new do
|
|
@@ -619,7 +619,7 @@ describe Chewy::Index::Actions do
|
|
|
619
619
|
p 'end reset1'
|
|
620
620
|
end
|
|
621
621
|
].map(&:join)
|
|
622
|
-
|
|
622
|
+
ActiveRecord::Base.connection.reconnect! if defined?(ActiveRecord::Base)
|
|
623
623
|
p 'expect1'
|
|
624
624
|
expect(CitiesIndex::City.pluck(:_id, :name)).to contain_exactly(%w[1 NewName1], %w[2 Name2], %w[3 NewName3])
|
|
625
625
|
p 'end expect1'
|
|
@@ -628,7 +628,7 @@ describe Chewy::Index::Actions do
|
|
|
628
628
|
specify 'without journal application' do
|
|
629
629
|
cities
|
|
630
630
|
p 'cities created2'
|
|
631
|
-
|
|
631
|
+
ActiveRecord::Base.connection.close if defined?(ActiveRecord::Base)
|
|
632
632
|
[
|
|
633
633
|
parallel_update,
|
|
634
634
|
Thread.new do
|
|
@@ -637,7 +637,7 @@ describe Chewy::Index::Actions do
|
|
|
637
637
|
p 'end reset2'
|
|
638
638
|
end
|
|
639
639
|
].map(&:join)
|
|
640
|
-
|
|
640
|
+
ActiveRecord::Base.connection.reconnect! if defined?(ActiveRecord::Base)
|
|
641
641
|
p 'expect2'
|
|
642
642
|
expect(CitiesIndex::City.pluck(:_id, :name)).to contain_exactly(%w[1 Name1], %w[2 Name2], %w[3 Name3])
|
|
643
643
|
p 'end expect2'
|
|
@@ -35,6 +35,68 @@ describe Chewy::Index::Adapter::ActiveRecord, :active_record do
|
|
|
35
35
|
specify { expect(described_class.new(City.where(rating: 10)).default_scope).to eq(City.where(rating: 10)) }
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
+
describe '.new' do
|
|
39
|
+
context 'with logger' do
|
|
40
|
+
let(:test_logger) { Logger.new('/dev/null') }
|
|
41
|
+
let(:default_scope_behavior) { :warn }
|
|
42
|
+
|
|
43
|
+
around do |example|
|
|
44
|
+
previous_logger = Chewy.logger
|
|
45
|
+
Chewy.logger = test_logger
|
|
46
|
+
|
|
47
|
+
previous_default_scope_behavior = Chewy.config.import_scope_cleanup_behavior
|
|
48
|
+
Chewy.config.import_scope_cleanup_behavior = default_scope_behavior
|
|
49
|
+
|
|
50
|
+
example.run
|
|
51
|
+
ensure
|
|
52
|
+
Chewy.logger = previous_logger
|
|
53
|
+
Chewy.config.import_scope_cleanup_behavior = previous_default_scope_behavior
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
specify do
|
|
57
|
+
expect(test_logger).to receive(:warn)
|
|
58
|
+
described_class.new(City.order(:id))
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
specify do
|
|
62
|
+
expect(test_logger).to receive(:warn)
|
|
63
|
+
described_class.new(City.offset(10))
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
specify do
|
|
67
|
+
expect(test_logger).to receive(:warn)
|
|
68
|
+
described_class.new(City.limit(10))
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
context 'ignore import scope warning' do
|
|
72
|
+
let(:default_scope_behavior) { :ignore }
|
|
73
|
+
|
|
74
|
+
specify do
|
|
75
|
+
expect(test_logger).not_to receive(:warn)
|
|
76
|
+
described_class.new(City.order(:id))
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
specify do
|
|
80
|
+
expect(test_logger).not_to receive(:warn)
|
|
81
|
+
described_class.new(City.offset(10))
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
specify do
|
|
85
|
+
expect(test_logger).not_to receive(:warn)
|
|
86
|
+
described_class.new(City.limit(10))
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
context 'raise exception on import scope with order/limit/offset' do
|
|
91
|
+
let(:default_scope_behavior) { :raise }
|
|
92
|
+
|
|
93
|
+
specify { expect { described_class.new(City.order(:id)) }.to raise_error(Chewy::ImportScopeCleanupError) }
|
|
94
|
+
specify { expect { described_class.new(City.limit(10)) }.to raise_error(Chewy::ImportScopeCleanupError) }
|
|
95
|
+
specify { expect { described_class.new(City.offset(10)) }.to raise_error(Chewy::ImportScopeCleanupError) }
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
38
100
|
describe '#type_name' do
|
|
39
101
|
specify { expect(described_class.new(City).type_name).to eq('city') }
|
|
40
102
|
specify { expect(described_class.new(City.order(:id)).type_name).to eq('city') }
|
|
@@ -62,6 +62,8 @@ describe Chewy::Index::Import::BulkBuilder do
|
|
|
62
62
|
let(:to_index) { cities.first(2) }
|
|
63
63
|
let(:delete) { [cities.last] }
|
|
64
64
|
specify do
|
|
65
|
+
expect(subject).to receive(:data_for).with(cities.first).and_call_original
|
|
66
|
+
expect(subject).to receive(:data_for).with(cities.second).and_call_original
|
|
65
67
|
expect(subject.bulk_body).to eq([
|
|
66
68
|
{index: {_id: 1, data: {'name' => 'City17', 'rating' => 42}}},
|
|
67
69
|
{index: {_id: 2, data: {'name' => 'City18', 'rating' => 42}}},
|
|
@@ -72,6 +74,8 @@ describe Chewy::Index::Import::BulkBuilder do
|
|
|
72
74
|
context ':fields' do
|
|
73
75
|
let(:fields) { %w[name] }
|
|
74
76
|
specify do
|
|
77
|
+
expect(subject).to receive(:data_for).with(cities.first, fields: [:name]).and_call_original
|
|
78
|
+
expect(subject).to receive(:data_for).with(cities.second, fields: [:name]).and_call_original
|
|
75
79
|
expect(subject.bulk_body).to eq([
|
|
76
80
|
{update: {_id: 1, data: {doc: {'name' => 'City17'}}}},
|
|
77
81
|
{update: {_id: 2, data: {doc: {'name' => 'City18'}}}},
|
|
@@ -128,7 +132,7 @@ describe Chewy::Index::Import::BulkBuilder do
|
|
|
128
132
|
before do
|
|
129
133
|
stub_index(:cities) do
|
|
130
134
|
crutch :names do |collection|
|
|
131
|
-
collection.
|
|
135
|
+
collection.to_h { |item| [item.id, "Name#{item.id}"] }
|
|
132
136
|
end
|
|
133
137
|
|
|
134
138
|
field :name, value: ->(o, c) { c.names[o.id] }
|
|
@@ -194,7 +198,7 @@ describe Chewy::Index::Import::BulkBuilder do
|
|
|
194
198
|
index_scope Comment
|
|
195
199
|
|
|
196
200
|
crutch :content_with_crutches do |collection| # collection here is a current batch of products
|
|
197
|
-
collection.
|
|
201
|
+
collection.to_h { |comment| [comment.id, "[crutches] #{comment.content}"] }
|
|
198
202
|
end
|
|
199
203
|
|
|
200
204
|
field :content
|
|
@@ -268,7 +272,7 @@ describe Chewy::Index::Import::BulkBuilder do
|
|
|
268
272
|
default_import_options raw_import: ->(hash) { SimpleComment.new(hash) }
|
|
269
273
|
|
|
270
274
|
crutch :content_with_crutches do |collection| # collection here is a current batch of products
|
|
271
|
-
collection.
|
|
275
|
+
collection.to_h { |comment| [comment.id, "[crutches] #{comment.content}"] }
|
|
272
276
|
end
|
|
273
277
|
|
|
274
278
|
field :content
|