caoutsearch 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +22 -0
- data/README.md +43 -0
- data/lib/caoutsearch/config/client.rb +13 -0
- data/lib/caoutsearch/config/mappings.rb +40 -0
- data/lib/caoutsearch/config/settings.rb +29 -0
- data/lib/caoutsearch/filter/base.rb +101 -0
- data/lib/caoutsearch/filter/boolean.rb +19 -0
- data/lib/caoutsearch/filter/date.rb +49 -0
- data/lib/caoutsearch/filter/default.rb +51 -0
- data/lib/caoutsearch/filter/geo_point.rb +11 -0
- data/lib/caoutsearch/filter/match.rb +57 -0
- data/lib/caoutsearch/filter/none.rb +7 -0
- data/lib/caoutsearch/filter/range.rb +28 -0
- data/lib/caoutsearch/filter.rb +29 -0
- data/lib/caoutsearch/index/base.rb +35 -0
- data/lib/caoutsearch/index/document.rb +107 -0
- data/lib/caoutsearch/index/indice.rb +55 -0
- data/lib/caoutsearch/index/indice_versions.rb +123 -0
- data/lib/caoutsearch/index/instrumentation.rb +19 -0
- data/lib/caoutsearch/index/internal_dsl.rb +77 -0
- data/lib/caoutsearch/index/naming.rb +29 -0
- data/lib/caoutsearch/index/reindex.rb +77 -0
- data/lib/caoutsearch/index/scoping.rb +54 -0
- data/lib/caoutsearch/index/serialization.rb +136 -0
- data/lib/caoutsearch/index.rb +7 -0
- data/lib/caoutsearch/instrumentation/base.rb +69 -0
- data/lib/caoutsearch/instrumentation/index.rb +57 -0
- data/lib/caoutsearch/instrumentation/search.rb +41 -0
- data/lib/caoutsearch/mappings.rb +79 -0
- data/lib/caoutsearch/search/base.rb +27 -0
- data/lib/caoutsearch/search/dsl/item.rb +42 -0
- data/lib/caoutsearch/search/query/base.rb +16 -0
- data/lib/caoutsearch/search/query/boolean.rb +63 -0
- data/lib/caoutsearch/search/query/cleaning.rb +29 -0
- data/lib/caoutsearch/search/query/getters.rb +35 -0
- data/lib/caoutsearch/search/query/merge.rb +27 -0
- data/lib/caoutsearch/search/query/nested.rb +23 -0
- data/lib/caoutsearch/search/query/setters.rb +68 -0
- data/lib/caoutsearch/search/sanitizer.rb +28 -0
- data/lib/caoutsearch/search/search/delete_methods.rb +21 -0
- data/lib/caoutsearch/search/search/inspect.rb +36 -0
- data/lib/caoutsearch/search/search/instrumentation.rb +21 -0
- data/lib/caoutsearch/search/search/internal_dsl.rb +77 -0
- data/lib/caoutsearch/search/search/naming.rb +47 -0
- data/lib/caoutsearch/search/search/query_builder.rb +94 -0
- data/lib/caoutsearch/search/search/query_methods.rb +180 -0
- data/lib/caoutsearch/search/search/resettable.rb +35 -0
- data/lib/caoutsearch/search/search/response.rb +88 -0
- data/lib/caoutsearch/search/search/scroll_methods.rb +113 -0
- data/lib/caoutsearch/search/search/search_methods.rb +230 -0
- data/lib/caoutsearch/search/type_cast.rb +76 -0
- data/lib/caoutsearch/search/value.rb +111 -0
- data/lib/caoutsearch/search/value_overflow.rb +17 -0
- data/lib/caoutsearch/search.rb +6 -0
- data/lib/caoutsearch/settings.rb +22 -0
- data/lib/caoutsearch/version.rb +5 -0
- data/lib/caoutsearch.rb +38 -0
- metadata +268 -0
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Caoutsearch
|
4
|
+
module Search
|
5
|
+
module Search
|
6
|
+
module InternalDSL
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
# Do to not make:
|
11
|
+
# self.config[:filter][key] = ...
|
12
|
+
#
|
13
|
+
# Changing only config members will lead to inheritance issue.
|
14
|
+
# Instead, make:
|
15
|
+
# self.config = config.deep_dup
|
16
|
+
# self.config[:filter][key] = ...
|
17
|
+
#
|
18
|
+
class_attribute :config, default: {
|
19
|
+
contexts: ActiveSupport::HashWithIndifferentAccess.new,
|
20
|
+
filters: ActiveSupport::HashWithIndifferentAccess.new,
|
21
|
+
defaults: ActiveSupport::HashWithIndifferentAccess.new,
|
22
|
+
aggregations: ActiveSupport::HashWithIndifferentAccess.new,
|
23
|
+
suggestions: ActiveSupport::HashWithIndifferentAccess.new,
|
24
|
+
sorts: ActiveSupport::HashWithIndifferentAccess.new
|
25
|
+
}
|
26
|
+
end
|
27
|
+
|
28
|
+
class_methods do
|
29
|
+
def match_all(&block)
|
30
|
+
self.config = config.deep_dup
|
31
|
+
config[:match_all] = block
|
32
|
+
end
|
33
|
+
|
34
|
+
%w[context default].each do |method|
|
35
|
+
config_attribute = method.pluralize.to_sym
|
36
|
+
|
37
|
+
define_method method do |name = nil, &block|
|
38
|
+
self.config = config.deep_dup
|
39
|
+
|
40
|
+
if name
|
41
|
+
config[config_attribute][name] = Caoutsearch::Search::DSL::Item.new(name, &block)
|
42
|
+
else
|
43
|
+
config[config_attribute][:__undef__] = block
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
%w[filter aggregation sort suggestion].each do |method|
|
49
|
+
config_attribute = method.pluralize.to_sym
|
50
|
+
|
51
|
+
define_method method do |name = nil, **options, &block|
|
52
|
+
self.config = config.deep_dup
|
53
|
+
|
54
|
+
if name
|
55
|
+
config[config_attribute][name.to_s] = Caoutsearch::Search::DSL::Item.new(name, options, &block)
|
56
|
+
else
|
57
|
+
config[config_attribute][:__undef__] = block
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def alias_filter(new_name, old_name)
|
63
|
+
filter(new_name) { |value| search_by(old_name => value) }
|
64
|
+
end
|
65
|
+
|
66
|
+
def alias_sort(new_name, old_name)
|
67
|
+
sort(new_name) { |direction| sort_by(old_name, direction) }
|
68
|
+
end
|
69
|
+
|
70
|
+
def alias_aggregation(new_name, old_name)
|
71
|
+
aggregation(new_name) { aggregate_with(old_name) }
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Caoutsearch
|
4
|
+
module Search
|
5
|
+
module Search
|
6
|
+
module Naming
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
delegate :model, :model_name, :index_name, to: :class
|
11
|
+
end
|
12
|
+
|
13
|
+
class_methods do
|
14
|
+
def model
|
15
|
+
@model ||= model_name.constantize
|
16
|
+
end
|
17
|
+
|
18
|
+
def model_name
|
19
|
+
@model_name ||= defaut_model_name
|
20
|
+
end
|
21
|
+
|
22
|
+
def model_name=(name)
|
23
|
+
@model_name = name
|
24
|
+
end
|
25
|
+
|
26
|
+
def index_name
|
27
|
+
@index_name ||= default_index_name
|
28
|
+
end
|
29
|
+
|
30
|
+
def index_name=(name)
|
31
|
+
@index_name = name
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def default_index_name
|
37
|
+
name.gsub(/Search$/, "").tableize.tr("/", "_")
|
38
|
+
end
|
39
|
+
|
40
|
+
def defaut_model_name
|
41
|
+
name.gsub(/Search$/, "")
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Caoutsearch
|
4
|
+
module Search
|
5
|
+
module Search
|
6
|
+
module QueryBuilder
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
include ActiveSupport::Callbacks
|
11
|
+
define_callbacks :build
|
12
|
+
end
|
13
|
+
|
14
|
+
def build
|
15
|
+
run_callbacks :build do
|
16
|
+
apply_prepend_hash
|
17
|
+
apply_search_criteria
|
18
|
+
apply_context
|
19
|
+
apply_defaults
|
20
|
+
apply_limits
|
21
|
+
apply_orders
|
22
|
+
apply_aggregations
|
23
|
+
apply_suggestions
|
24
|
+
apply_fields
|
25
|
+
apply_source
|
26
|
+
apply_total_hits_tracking
|
27
|
+
apply_append_hash
|
28
|
+
end
|
29
|
+
|
30
|
+
elasticsearch_query.clean
|
31
|
+
elasticsearch_query
|
32
|
+
end
|
33
|
+
|
34
|
+
def apply_search_criteria
|
35
|
+
search_by(search_criteria)
|
36
|
+
end
|
37
|
+
|
38
|
+
def apply_context
|
39
|
+
return unless current_context
|
40
|
+
|
41
|
+
item = config[:contexts][current_context.to_s]
|
42
|
+
instance_exec(&item.block) if item
|
43
|
+
end
|
44
|
+
|
45
|
+
def apply_defaults
|
46
|
+
keys = search_criteria_keys.map(&:to_s)
|
47
|
+
|
48
|
+
config[:defaults].each do |key, item|
|
49
|
+
instance_exec(&item.block) unless keys.include?(key.to_s)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def apply_limits
|
54
|
+
elasticsearch_query[:size] = current_limit.to_i if @current_page || @current_limit
|
55
|
+
elasticsearch_query[:from] = current_offset if @current_page || @current_offset
|
56
|
+
end
|
57
|
+
|
58
|
+
def apply_orders
|
59
|
+
return if current_limit.zero?
|
60
|
+
|
61
|
+
order_by(current_order || :default)
|
62
|
+
end
|
63
|
+
|
64
|
+
def apply_aggregations
|
65
|
+
aggregate_with(*current_aggregations) if current_aggregations
|
66
|
+
end
|
67
|
+
|
68
|
+
def apply_suggestions
|
69
|
+
suggest_with(*current_suggestions) if current_suggestions
|
70
|
+
end
|
71
|
+
|
72
|
+
def apply_fields
|
73
|
+
elasticsearch_query[:fields] = current_fields.map(&:to_s) if current_fields
|
74
|
+
end
|
75
|
+
|
76
|
+
def apply_source
|
77
|
+
elasticsearch_query[:_source] = current_source unless current_source.nil?
|
78
|
+
end
|
79
|
+
|
80
|
+
def apply_total_hits_tracking
|
81
|
+
elasticsearch_query[:track_total_hits] = @track_total_hits if @track_total_hits
|
82
|
+
end
|
83
|
+
|
84
|
+
def apply_prepend_hash
|
85
|
+
elasticsearch_query.merge!(@prepend_hash) if @prepend_hash
|
86
|
+
end
|
87
|
+
|
88
|
+
def apply_append_hash
|
89
|
+
elasticsearch_query.merge!(@append_hash) if @append_hash
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,180 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Caoutsearch
|
4
|
+
module Search
|
5
|
+
module Search
|
6
|
+
module QueryMethods
|
7
|
+
delegate :merge, :merge!, :to_h, :to_json, :as_json, :filters,
|
8
|
+
:add_none, :add_filter, :add_sort, :add_aggregation, :add_suggestion,
|
9
|
+
:build_terms, :should_filter_on, :must_filter_on, :must_not_filter_on,
|
10
|
+
:nested_queries, :nested_query,
|
11
|
+
to: :elasticsearch_query
|
12
|
+
|
13
|
+
# Accessors
|
14
|
+
# ------------------------------------------------------------------------
|
15
|
+
def elasticsearch_query
|
16
|
+
@elasticsearch_query ||= Caoutsearch::Search::Query::Base.new
|
17
|
+
end
|
18
|
+
|
19
|
+
# Applying criteria
|
20
|
+
# ------------------------------------------------------------------------
|
21
|
+
def search_by(criteria)
|
22
|
+
case criteria
|
23
|
+
when Array then criteria.each { |criterion| search_by(criterion) }
|
24
|
+
when Hash then criteria.each { |key, value| filter_by(key, value) }
|
25
|
+
when String then apply_dsl_match_all(criteria)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def filter_by(key, value)
|
30
|
+
case key
|
31
|
+
when Array
|
32
|
+
items = key.filter_map { |k| config[:filters][k] }
|
33
|
+
apply_dsl_filters(items, value)
|
34
|
+
when Caoutsearch::Search::DSL::Item
|
35
|
+
apply_dsl_filter(key, value)
|
36
|
+
else
|
37
|
+
if config[:filters].key?(key)
|
38
|
+
apply_dsl_filter(config[:filters][key], value)
|
39
|
+
elsif config[:filters].key?(:__undef__)
|
40
|
+
block = config[:filters][:__undef__]
|
41
|
+
instance_exec(key, value, &block)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# Applying DSL filters items
|
47
|
+
# ------------------------------------------------------------------------
|
48
|
+
def apply_dsl_match_all(value)
|
49
|
+
return unless (block = config[:match_all])
|
50
|
+
|
51
|
+
instance_exec(value, &block)
|
52
|
+
end
|
53
|
+
|
54
|
+
def apply_dsl_filter(item, value)
|
55
|
+
return instance_exec(value, &item.block) if item.has_block?
|
56
|
+
|
57
|
+
terms = []
|
58
|
+
indexes = item.indexes
|
59
|
+
options = item.options.dup
|
60
|
+
query = elasticsearch_query
|
61
|
+
|
62
|
+
if options[:nested_query]
|
63
|
+
query = nested_query(options[:nested_query])
|
64
|
+
options[:nested] = nil
|
65
|
+
end
|
66
|
+
|
67
|
+
indexes.each do |index|
|
68
|
+
options_index = options.dup
|
69
|
+
options_index[:type] = mappings.find_type(index) unless options_index.key?(:type)
|
70
|
+
options_index[:nested] = mappings.nested_path(index) unless options_index.key?(:nested)
|
71
|
+
options_index[:include_in_parent] = mappings.include_in_parent?(index) unless options_index.key?(:include_in_parent) || !options_index[:nested]
|
72
|
+
|
73
|
+
terms += query.build_terms(index, value, **options_index)
|
74
|
+
end
|
75
|
+
|
76
|
+
query.should_filter_on(terms)
|
77
|
+
end
|
78
|
+
|
79
|
+
def apply_dsl_filters(items, value)
|
80
|
+
terms = []
|
81
|
+
|
82
|
+
items.each do |item|
|
83
|
+
terms += isolate_dsl_filter(item, value)
|
84
|
+
end
|
85
|
+
|
86
|
+
should_filter_on(terms)
|
87
|
+
end
|
88
|
+
|
89
|
+
def isolate_dsl_filter(item, value)
|
90
|
+
isolated_instance = clone
|
91
|
+
isolated_instance.apply_dsl_filter(item, value)
|
92
|
+
isolated_instance.elasticsearch_query.dig(:query, :bool, :filter) || []
|
93
|
+
end
|
94
|
+
|
95
|
+
# Applying orders
|
96
|
+
# ------------------------------------------------------------------------
|
97
|
+
def order_by(keys)
|
98
|
+
case keys
|
99
|
+
when Array then keys.each { |key| order_by(key) }
|
100
|
+
when Hash then keys.each { |key, direction| sort_by(key, direction) }
|
101
|
+
when String, Symbol then sort_by(keys)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def sort_by(key, direction = nil)
|
106
|
+
if config[:sorts].key?(key)
|
107
|
+
apply_dsl_sort(config[:sorts][key], direction)
|
108
|
+
elsif config[:sorts].key?(:__undef__)
|
109
|
+
block = config[:sorts][:__undef__]
|
110
|
+
instance_exec(key, direction, &block)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def apply_dsl_sort(item, direction)
|
115
|
+
return instance_exec(direction, &item.block) if item.has_block?
|
116
|
+
|
117
|
+
indexes = item.indexes
|
118
|
+
indexes.each do |index|
|
119
|
+
case index
|
120
|
+
when :default, "default"
|
121
|
+
sort_by(:default, direction)
|
122
|
+
|
123
|
+
when Hash
|
124
|
+
index.map do |key, value|
|
125
|
+
key_direction = (value.to_s == direction.to_s ? :asc : :desc)
|
126
|
+
add_sort(key, key_direction)
|
127
|
+
end
|
128
|
+
|
129
|
+
else
|
130
|
+
add_sort(index, direction)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# Applying aggregations
|
136
|
+
# ------------------------------------------------------------------------
|
137
|
+
def aggregate_with(*args)
|
138
|
+
args.each do |arg|
|
139
|
+
if arg.is_a?(Hash)
|
140
|
+
arg.each do |key, value|
|
141
|
+
next unless (item = config[:aggregations][key])
|
142
|
+
|
143
|
+
apply_dsl_aggregate(item, value)
|
144
|
+
end
|
145
|
+
elsif (item = config[:aggregations][arg])
|
146
|
+
apply_dsl_aggregate(item)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def apply_dsl_aggregate(item, *args)
|
152
|
+
return instance_exec(*args, &item.block) if item.has_block?
|
153
|
+
|
154
|
+
add_aggregation(item.name, item.options)
|
155
|
+
end
|
156
|
+
|
157
|
+
# Applying Suggests
|
158
|
+
# ------------------------------------------------------------------------
|
159
|
+
def suggest_with(*args)
|
160
|
+
args.each do |(hash, options)|
|
161
|
+
raise ArgumentError unless hash.is_a?(Hash)
|
162
|
+
|
163
|
+
hash.each do |key, value|
|
164
|
+
next unless (item = config[:suggestions][key.to_s])
|
165
|
+
|
166
|
+
options ||= {}
|
167
|
+
apply_dsl_suggest(item, value, **options)
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
def apply_dsl_suggest(item, query, **options)
|
173
|
+
return instance_exec(query, **options, &item.block) if item.has_block?
|
174
|
+
|
175
|
+
add_suggestion(item.name, query, **options)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Caoutsearch
|
4
|
+
module Search
|
5
|
+
module Search
|
6
|
+
module Resettable
|
7
|
+
def spawn
|
8
|
+
clone
|
9
|
+
end
|
10
|
+
|
11
|
+
def clone
|
12
|
+
super.reset
|
13
|
+
end
|
14
|
+
|
15
|
+
def dup
|
16
|
+
super.reset
|
17
|
+
end
|
18
|
+
|
19
|
+
def reset
|
20
|
+
reset_variable(:@elasticsearch_query)
|
21
|
+
reset_variable(:@nested_queries)
|
22
|
+
reset_variable(:@response)
|
23
|
+
reset_variable(:@total_count)
|
24
|
+
self
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def reset_variable(key)
|
30
|
+
remove_instance_variable(key) if instance_variable_defined?(key)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Caoutsearch
|
4
|
+
module Search
|
5
|
+
module Search
|
6
|
+
module Response
|
7
|
+
delegate :empty?, :size, :slice, :[], :to_a, :to_ary, to: :hits
|
8
|
+
delegate_missing_to :each
|
9
|
+
|
10
|
+
def response
|
11
|
+
load unless @response
|
12
|
+
@response
|
13
|
+
end
|
14
|
+
|
15
|
+
def took
|
16
|
+
response["took"]
|
17
|
+
end
|
18
|
+
|
19
|
+
def timed_out
|
20
|
+
response["timed_out"]
|
21
|
+
end
|
22
|
+
|
23
|
+
def shards
|
24
|
+
response["_shards"]
|
25
|
+
end
|
26
|
+
|
27
|
+
def hits
|
28
|
+
response.dig("hits", "hits")
|
29
|
+
end
|
30
|
+
|
31
|
+
def max_score
|
32
|
+
response.dig("hits", "max_score")
|
33
|
+
end
|
34
|
+
|
35
|
+
def total_count
|
36
|
+
if response.dig("hits", "total", "relation") == "gte" && !@track_total_hits
|
37
|
+
@total_count ||= spawn.track_total_hits(true).total_count
|
38
|
+
else
|
39
|
+
response.dig("hits", "total", "value")
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def total_pages
|
44
|
+
(total_count.to_f / current_limit).ceil
|
45
|
+
end
|
46
|
+
|
47
|
+
def ids
|
48
|
+
hits.pluck("_id")
|
49
|
+
end
|
50
|
+
|
51
|
+
def aggregations
|
52
|
+
# TODO
|
53
|
+
end
|
54
|
+
|
55
|
+
def suggestions
|
56
|
+
# TODO
|
57
|
+
end
|
58
|
+
|
59
|
+
def records
|
60
|
+
model.where(model.primary_key => ids)
|
61
|
+
end
|
62
|
+
|
63
|
+
def each(&block)
|
64
|
+
return to_enum(:each) { hits.size } unless block
|
65
|
+
|
66
|
+
hits.each(&block)
|
67
|
+
end
|
68
|
+
|
69
|
+
def load
|
70
|
+
@response = perform_search_query(build.to_h)
|
71
|
+
self
|
72
|
+
end
|
73
|
+
|
74
|
+
def perform_search_query(query)
|
75
|
+
request_payload = {
|
76
|
+
index: index_name,
|
77
|
+
body: query
|
78
|
+
}
|
79
|
+
|
80
|
+
instrument(:search) do |event_payload|
|
81
|
+
event_payload[:request] = request_payload
|
82
|
+
event_payload[:response] = client.search(request_payload)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Caoutsearch
|
4
|
+
module Search
|
5
|
+
module Search
|
6
|
+
module ScrollMethods
|
7
|
+
def scroll(scroll: "1h", &block)
|
8
|
+
return to_enum(scroll, scroll: scroll) unless block
|
9
|
+
|
10
|
+
request_payload = {
|
11
|
+
index: index_name,
|
12
|
+
scroll: scroll,
|
13
|
+
body: build.to_h
|
14
|
+
}
|
15
|
+
|
16
|
+
total = 0
|
17
|
+
progress = 0
|
18
|
+
requested_at = nil
|
19
|
+
last_response_time = nil
|
20
|
+
|
21
|
+
results = instrument(:scroll_search) do |event_payload|
|
22
|
+
response = client.search(request_payload)
|
23
|
+
last_response_time = Time.current
|
24
|
+
|
25
|
+
total = response["hits"]["total"]["value"]
|
26
|
+
progress += response["hits"]["hits"].size
|
27
|
+
|
28
|
+
event_payload[:request] = request_payload
|
29
|
+
event_payload[:response] = response
|
30
|
+
event_payload[:total] = total
|
31
|
+
event_payload[:progress] = progress
|
32
|
+
|
33
|
+
response
|
34
|
+
end
|
35
|
+
|
36
|
+
scroll_id = results["_scroll_id"]
|
37
|
+
hits = results["hits"]["hits"]
|
38
|
+
|
39
|
+
yield hits, { progress: progress, total: total, scroll_id: scroll_id }
|
40
|
+
|
41
|
+
while progress < total
|
42
|
+
request_payload = {
|
43
|
+
scroll_id: scroll_id,
|
44
|
+
scroll: scroll
|
45
|
+
}
|
46
|
+
|
47
|
+
requested_at = Time.current
|
48
|
+
|
49
|
+
results = instrument(:scroll, scroll: scroll_id) do |event_payload|
|
50
|
+
response = client.scroll(request_payload)
|
51
|
+
last_response_time = Time.current
|
52
|
+
|
53
|
+
total = response["hits"]["total"]["value"]
|
54
|
+
progress += response["hits"]["hits"].size
|
55
|
+
|
56
|
+
event_payload[:request] = request_payload
|
57
|
+
event_payload[:response] = response
|
58
|
+
event_payload[:total] = total
|
59
|
+
event_payload[:progress] = progress
|
60
|
+
|
61
|
+
response
|
62
|
+
rescue Elastic::Transport::Transport::Errors::NotFound => e
|
63
|
+
raise_enhance_message_when_scroll_failed(e, scroll, requested_at, last_response_time)
|
64
|
+
end
|
65
|
+
|
66
|
+
hits = results["hits"]["hits"]
|
67
|
+
progress += hits.size
|
68
|
+
|
69
|
+
break if hits.empty?
|
70
|
+
|
71
|
+
yield hits, { progress: progress, total: total, scroll_id: scroll_id }
|
72
|
+
end
|
73
|
+
|
74
|
+
total
|
75
|
+
ensure
|
76
|
+
clear_scroll(scroll_id) if scroll_id
|
77
|
+
end
|
78
|
+
|
79
|
+
def clear_scroll(scroll_id)
|
80
|
+
client.clear_scroll(scroll_id: scroll_id)
|
81
|
+
rescue ::Elastic::Transport::Transport::Errors::NotFound
|
82
|
+
# We dont care if the scroll ID is already expired
|
83
|
+
end
|
84
|
+
|
85
|
+
def scroll_records_in_batches(**options)
|
86
|
+
return to_enum(:scroll_records_in_batches, **options) unless block_given?
|
87
|
+
|
88
|
+
scroll(**options) do |hits|
|
89
|
+
ids = hits.map { |doc| doc["_id"] }
|
90
|
+
yield model.where(id: ids)
|
91
|
+
model.connection.clear_query_cache
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def scroll_records(**options, &block)
|
96
|
+
return to_enum(:scroll_records, **options) unless block
|
97
|
+
|
98
|
+
scroll_records_in_batches(**options) do |relation|
|
99
|
+
relation.each(&block)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
private
|
104
|
+
|
105
|
+
def raise_enhance_message_when_scroll_failed(error, scroll, requested_at, last_response_time)
|
106
|
+
elapsed = (requested_at - last_response_time).round(1).seconds
|
107
|
+
|
108
|
+
raise error.exception("Scroll registered for #{scroll}, #{elapsed.inspect} elapsed between. #{error.message}")
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|