caoutsearch 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +22 -0
  3. data/README.md +43 -0
  4. data/lib/caoutsearch/config/client.rb +13 -0
  5. data/lib/caoutsearch/config/mappings.rb +40 -0
  6. data/lib/caoutsearch/config/settings.rb +29 -0
  7. data/lib/caoutsearch/filter/base.rb +101 -0
  8. data/lib/caoutsearch/filter/boolean.rb +19 -0
  9. data/lib/caoutsearch/filter/date.rb +49 -0
  10. data/lib/caoutsearch/filter/default.rb +51 -0
  11. data/lib/caoutsearch/filter/geo_point.rb +11 -0
  12. data/lib/caoutsearch/filter/match.rb +57 -0
  13. data/lib/caoutsearch/filter/none.rb +7 -0
  14. data/lib/caoutsearch/filter/range.rb +28 -0
  15. data/lib/caoutsearch/filter.rb +29 -0
  16. data/lib/caoutsearch/index/base.rb +35 -0
  17. data/lib/caoutsearch/index/document.rb +107 -0
  18. data/lib/caoutsearch/index/indice.rb +55 -0
  19. data/lib/caoutsearch/index/indice_versions.rb +123 -0
  20. data/lib/caoutsearch/index/instrumentation.rb +19 -0
  21. data/lib/caoutsearch/index/internal_dsl.rb +77 -0
  22. data/lib/caoutsearch/index/naming.rb +29 -0
  23. data/lib/caoutsearch/index/reindex.rb +77 -0
  24. data/lib/caoutsearch/index/scoping.rb +54 -0
  25. data/lib/caoutsearch/index/serialization.rb +136 -0
  26. data/lib/caoutsearch/index.rb +7 -0
  27. data/lib/caoutsearch/instrumentation/base.rb +69 -0
  28. data/lib/caoutsearch/instrumentation/index.rb +57 -0
  29. data/lib/caoutsearch/instrumentation/search.rb +41 -0
  30. data/lib/caoutsearch/mappings.rb +79 -0
  31. data/lib/caoutsearch/search/base.rb +27 -0
  32. data/lib/caoutsearch/search/dsl/item.rb +42 -0
  33. data/lib/caoutsearch/search/query/base.rb +16 -0
  34. data/lib/caoutsearch/search/query/boolean.rb +63 -0
  35. data/lib/caoutsearch/search/query/cleaning.rb +29 -0
  36. data/lib/caoutsearch/search/query/getters.rb +35 -0
  37. data/lib/caoutsearch/search/query/merge.rb +27 -0
  38. data/lib/caoutsearch/search/query/nested.rb +23 -0
  39. data/lib/caoutsearch/search/query/setters.rb +68 -0
  40. data/lib/caoutsearch/search/sanitizer.rb +28 -0
  41. data/lib/caoutsearch/search/search/delete_methods.rb +21 -0
  42. data/lib/caoutsearch/search/search/inspect.rb +36 -0
  43. data/lib/caoutsearch/search/search/instrumentation.rb +21 -0
  44. data/lib/caoutsearch/search/search/internal_dsl.rb +77 -0
  45. data/lib/caoutsearch/search/search/naming.rb +47 -0
  46. data/lib/caoutsearch/search/search/query_builder.rb +94 -0
  47. data/lib/caoutsearch/search/search/query_methods.rb +180 -0
  48. data/lib/caoutsearch/search/search/resettable.rb +35 -0
  49. data/lib/caoutsearch/search/search/response.rb +88 -0
  50. data/lib/caoutsearch/search/search/scroll_methods.rb +113 -0
  51. data/lib/caoutsearch/search/search/search_methods.rb +230 -0
  52. data/lib/caoutsearch/search/type_cast.rb +76 -0
  53. data/lib/caoutsearch/search/value.rb +111 -0
  54. data/lib/caoutsearch/search/value_overflow.rb +17 -0
  55. data/lib/caoutsearch/search.rb +6 -0
  56. data/lib/caoutsearch/settings.rb +22 -0
  57. data/lib/caoutsearch/version.rb +5 -0
  58. data/lib/caoutsearch.rb +38 -0
  59. metadata +268 -0
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Caoutsearch
4
+ module Search
5
+ module Search
6
+ module InternalDSL
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ # Do to not make:
11
+ # self.config[:filter][key] = ...
12
+ #
13
+ # Changing only config members will lead to inheritance issue.
14
+ # Instead, make:
15
+ # self.config = config.deep_dup
16
+ # self.config[:filter][key] = ...
17
+ #
18
+ class_attribute :config, default: {
19
+ contexts: ActiveSupport::HashWithIndifferentAccess.new,
20
+ filters: ActiveSupport::HashWithIndifferentAccess.new,
21
+ defaults: ActiveSupport::HashWithIndifferentAccess.new,
22
+ aggregations: ActiveSupport::HashWithIndifferentAccess.new,
23
+ suggestions: ActiveSupport::HashWithIndifferentAccess.new,
24
+ sorts: ActiveSupport::HashWithIndifferentAccess.new
25
+ }
26
+ end
27
+
28
+ class_methods do
29
+ def match_all(&block)
30
+ self.config = config.deep_dup
31
+ config[:match_all] = block
32
+ end
33
+
34
+ %w[context default].each do |method|
35
+ config_attribute = method.pluralize.to_sym
36
+
37
+ define_method method do |name = nil, &block|
38
+ self.config = config.deep_dup
39
+
40
+ if name
41
+ config[config_attribute][name] = Caoutsearch::Search::DSL::Item.new(name, &block)
42
+ else
43
+ config[config_attribute][:__undef__] = block
44
+ end
45
+ end
46
+ end
47
+
48
+ %w[filter aggregation sort suggestion].each do |method|
49
+ config_attribute = method.pluralize.to_sym
50
+
51
+ define_method method do |name = nil, **options, &block|
52
+ self.config = config.deep_dup
53
+
54
+ if name
55
+ config[config_attribute][name.to_s] = Caoutsearch::Search::DSL::Item.new(name, options, &block)
56
+ else
57
+ config[config_attribute][:__undef__] = block
58
+ end
59
+ end
60
+ end
61
+
62
+ def alias_filter(new_name, old_name)
63
+ filter(new_name) { |value| search_by(old_name => value) }
64
+ end
65
+
66
+ def alias_sort(new_name, old_name)
67
+ sort(new_name) { |direction| sort_by(old_name, direction) }
68
+ end
69
+
70
+ def alias_aggregation(new_name, old_name)
71
+ aggregation(new_name) { aggregate_with(old_name) }
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Caoutsearch
4
+ module Search
5
+ module Search
6
+ module Naming
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ delegate :model, :model_name, :index_name, to: :class
11
+ end
12
+
13
+ class_methods do
14
+ def model
15
+ @model ||= model_name.constantize
16
+ end
17
+
18
+ def model_name
19
+ @model_name ||= defaut_model_name
20
+ end
21
+
22
+ def model_name=(name)
23
+ @model_name = name
24
+ end
25
+
26
+ def index_name
27
+ @index_name ||= default_index_name
28
+ end
29
+
30
+ def index_name=(name)
31
+ @index_name = name
32
+ end
33
+
34
+ private
35
+
36
+ def default_index_name
37
+ name.gsub(/Search$/, "").tableize.tr("/", "_")
38
+ end
39
+
40
+ def defaut_model_name
41
+ name.gsub(/Search$/, "")
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Caoutsearch
4
+ module Search
5
+ module Search
6
+ module QueryBuilder
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ include ActiveSupport::Callbacks
11
+ define_callbacks :build
12
+ end
13
+
14
+ def build
15
+ run_callbacks :build do
16
+ apply_prepend_hash
17
+ apply_search_criteria
18
+ apply_context
19
+ apply_defaults
20
+ apply_limits
21
+ apply_orders
22
+ apply_aggregations
23
+ apply_suggestions
24
+ apply_fields
25
+ apply_source
26
+ apply_total_hits_tracking
27
+ apply_append_hash
28
+ end
29
+
30
+ elasticsearch_query.clean
31
+ elasticsearch_query
32
+ end
33
+
34
+ def apply_search_criteria
35
+ search_by(search_criteria)
36
+ end
37
+
38
+ def apply_context
39
+ return unless current_context
40
+
41
+ item = config[:contexts][current_context.to_s]
42
+ instance_exec(&item.block) if item
43
+ end
44
+
45
+ def apply_defaults
46
+ keys = search_criteria_keys.map(&:to_s)
47
+
48
+ config[:defaults].each do |key, item|
49
+ instance_exec(&item.block) unless keys.include?(key.to_s)
50
+ end
51
+ end
52
+
53
+ def apply_limits
54
+ elasticsearch_query[:size] = current_limit.to_i if @current_page || @current_limit
55
+ elasticsearch_query[:from] = current_offset if @current_page || @current_offset
56
+ end
57
+
58
+ def apply_orders
59
+ return if current_limit.zero?
60
+
61
+ order_by(current_order || :default)
62
+ end
63
+
64
+ def apply_aggregations
65
+ aggregate_with(*current_aggregations) if current_aggregations
66
+ end
67
+
68
+ def apply_suggestions
69
+ suggest_with(*current_suggestions) if current_suggestions
70
+ end
71
+
72
+ def apply_fields
73
+ elasticsearch_query[:fields] = current_fields.map(&:to_s) if current_fields
74
+ end
75
+
76
+ def apply_source
77
+ elasticsearch_query[:_source] = current_source unless current_source.nil?
78
+ end
79
+
80
+ def apply_total_hits_tracking
81
+ elasticsearch_query[:track_total_hits] = @track_total_hits if @track_total_hits
82
+ end
83
+
84
+ def apply_prepend_hash
85
+ elasticsearch_query.merge!(@prepend_hash) if @prepend_hash
86
+ end
87
+
88
+ def apply_append_hash
89
+ elasticsearch_query.merge!(@append_hash) if @append_hash
90
+ end
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,180 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Caoutsearch
4
+ module Search
5
+ module Search
6
+ module QueryMethods
7
+ delegate :merge, :merge!, :to_h, :to_json, :as_json, :filters,
8
+ :add_none, :add_filter, :add_sort, :add_aggregation, :add_suggestion,
9
+ :build_terms, :should_filter_on, :must_filter_on, :must_not_filter_on,
10
+ :nested_queries, :nested_query,
11
+ to: :elasticsearch_query
12
+
13
+ # Accessors
14
+ # ------------------------------------------------------------------------
15
+ def elasticsearch_query
16
+ @elasticsearch_query ||= Caoutsearch::Search::Query::Base.new
17
+ end
18
+
19
+ # Applying criteria
20
+ # ------------------------------------------------------------------------
21
+ def search_by(criteria)
22
+ case criteria
23
+ when Array then criteria.each { |criterion| search_by(criterion) }
24
+ when Hash then criteria.each { |key, value| filter_by(key, value) }
25
+ when String then apply_dsl_match_all(criteria)
26
+ end
27
+ end
28
+
29
+ def filter_by(key, value)
30
+ case key
31
+ when Array
32
+ items = key.filter_map { |k| config[:filters][k] }
33
+ apply_dsl_filters(items, value)
34
+ when Caoutsearch::Search::DSL::Item
35
+ apply_dsl_filter(key, value)
36
+ else
37
+ if config[:filters].key?(key)
38
+ apply_dsl_filter(config[:filters][key], value)
39
+ elsif config[:filters].key?(:__undef__)
40
+ block = config[:filters][:__undef__]
41
+ instance_exec(key, value, &block)
42
+ end
43
+ end
44
+ end
45
+
46
+ # Applying DSL filters items
47
+ # ------------------------------------------------------------------------
48
+ def apply_dsl_match_all(value)
49
+ return unless (block = config[:match_all])
50
+
51
+ instance_exec(value, &block)
52
+ end
53
+
54
+ def apply_dsl_filter(item, value)
55
+ return instance_exec(value, &item.block) if item.has_block?
56
+
57
+ terms = []
58
+ indexes = item.indexes
59
+ options = item.options.dup
60
+ query = elasticsearch_query
61
+
62
+ if options[:nested_query]
63
+ query = nested_query(options[:nested_query])
64
+ options[:nested] = nil
65
+ end
66
+
67
+ indexes.each do |index|
68
+ options_index = options.dup
69
+ options_index[:type] = mappings.find_type(index) unless options_index.key?(:type)
70
+ options_index[:nested] = mappings.nested_path(index) unless options_index.key?(:nested)
71
+ options_index[:include_in_parent] = mappings.include_in_parent?(index) unless options_index.key?(:include_in_parent) || !options_index[:nested]
72
+
73
+ terms += query.build_terms(index, value, **options_index)
74
+ end
75
+
76
+ query.should_filter_on(terms)
77
+ end
78
+
79
+ def apply_dsl_filters(items, value)
80
+ terms = []
81
+
82
+ items.each do |item|
83
+ terms += isolate_dsl_filter(item, value)
84
+ end
85
+
86
+ should_filter_on(terms)
87
+ end
88
+
89
+ def isolate_dsl_filter(item, value)
90
+ isolated_instance = clone
91
+ isolated_instance.apply_dsl_filter(item, value)
92
+ isolated_instance.elasticsearch_query.dig(:query, :bool, :filter) || []
93
+ end
94
+
95
+ # Applying orders
96
+ # ------------------------------------------------------------------------
97
+ def order_by(keys)
98
+ case keys
99
+ when Array then keys.each { |key| order_by(key) }
100
+ when Hash then keys.each { |key, direction| sort_by(key, direction) }
101
+ when String, Symbol then sort_by(keys)
102
+ end
103
+ end
104
+
105
+ def sort_by(key, direction = nil)
106
+ if config[:sorts].key?(key)
107
+ apply_dsl_sort(config[:sorts][key], direction)
108
+ elsif config[:sorts].key?(:__undef__)
109
+ block = config[:sorts][:__undef__]
110
+ instance_exec(key, direction, &block)
111
+ end
112
+ end
113
+
114
+ def apply_dsl_sort(item, direction)
115
+ return instance_exec(direction, &item.block) if item.has_block?
116
+
117
+ indexes = item.indexes
118
+ indexes.each do |index|
119
+ case index
120
+ when :default, "default"
121
+ sort_by(:default, direction)
122
+
123
+ when Hash
124
+ index.map do |key, value|
125
+ key_direction = (value.to_s == direction.to_s ? :asc : :desc)
126
+ add_sort(key, key_direction)
127
+ end
128
+
129
+ else
130
+ add_sort(index, direction)
131
+ end
132
+ end
133
+ end
134
+
135
+ # Applying aggregations
136
+ # ------------------------------------------------------------------------
137
+ def aggregate_with(*args)
138
+ args.each do |arg|
139
+ if arg.is_a?(Hash)
140
+ arg.each do |key, value|
141
+ next unless (item = config[:aggregations][key])
142
+
143
+ apply_dsl_aggregate(item, value)
144
+ end
145
+ elsif (item = config[:aggregations][arg])
146
+ apply_dsl_aggregate(item)
147
+ end
148
+ end
149
+ end
150
+
151
+ def apply_dsl_aggregate(item, *args)
152
+ return instance_exec(*args, &item.block) if item.has_block?
153
+
154
+ add_aggregation(item.name, item.options)
155
+ end
156
+
157
+ # Applying Suggests
158
+ # ------------------------------------------------------------------------
159
+ def suggest_with(*args)
160
+ args.each do |(hash, options)|
161
+ raise ArgumentError unless hash.is_a?(Hash)
162
+
163
+ hash.each do |key, value|
164
+ next unless (item = config[:suggestions][key.to_s])
165
+
166
+ options ||= {}
167
+ apply_dsl_suggest(item, value, **options)
168
+ end
169
+ end
170
+ end
171
+
172
+ def apply_dsl_suggest(item, query, **options)
173
+ return instance_exec(query, **options, &item.block) if item.has_block?
174
+
175
+ add_suggestion(item.name, query, **options)
176
+ end
177
+ end
178
+ end
179
+ end
180
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Caoutsearch
4
+ module Search
5
+ module Search
6
+ module Resettable
7
+ def spawn
8
+ clone
9
+ end
10
+
11
+ def clone
12
+ super.reset
13
+ end
14
+
15
+ def dup
16
+ super.reset
17
+ end
18
+
19
+ def reset
20
+ reset_variable(:@elasticsearch_query)
21
+ reset_variable(:@nested_queries)
22
+ reset_variable(:@response)
23
+ reset_variable(:@total_count)
24
+ self
25
+ end
26
+
27
+ private
28
+
29
+ def reset_variable(key)
30
+ remove_instance_variable(key) if instance_variable_defined?(key)
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Caoutsearch
4
+ module Search
5
+ module Search
6
+ module Response
7
+ delegate :empty?, :size, :slice, :[], :to_a, :to_ary, to: :hits
8
+ delegate_missing_to :each
9
+
10
+ def response
11
+ load unless @response
12
+ @response
13
+ end
14
+
15
+ def took
16
+ response["took"]
17
+ end
18
+
19
+ def timed_out
20
+ response["timed_out"]
21
+ end
22
+
23
+ def shards
24
+ response["_shards"]
25
+ end
26
+
27
+ def hits
28
+ response.dig("hits", "hits")
29
+ end
30
+
31
+ def max_score
32
+ response.dig("hits", "max_score")
33
+ end
34
+
35
+ def total_count
36
+ if response.dig("hits", "total", "relation") == "gte" && !@track_total_hits
37
+ @total_count ||= spawn.track_total_hits(true).total_count
38
+ else
39
+ response.dig("hits", "total", "value")
40
+ end
41
+ end
42
+
43
+ def total_pages
44
+ (total_count.to_f / current_limit).ceil
45
+ end
46
+
47
+ def ids
48
+ hits.pluck("_id")
49
+ end
50
+
51
+ def aggregations
52
+ # TODO
53
+ end
54
+
55
+ def suggestions
56
+ # TODO
57
+ end
58
+
59
+ def records
60
+ model.where(model.primary_key => ids)
61
+ end
62
+
63
+ def each(&block)
64
+ return to_enum(:each) { hits.size } unless block
65
+
66
+ hits.each(&block)
67
+ end
68
+
69
+ def load
70
+ @response = perform_search_query(build.to_h)
71
+ self
72
+ end
73
+
74
+ def perform_search_query(query)
75
+ request_payload = {
76
+ index: index_name,
77
+ body: query
78
+ }
79
+
80
+ instrument(:search) do |event_payload|
81
+ event_payload[:request] = request_payload
82
+ event_payload[:response] = client.search(request_payload)
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,113 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Caoutsearch
4
+ module Search
5
+ module Search
6
+ module ScrollMethods
7
+ def scroll(scroll: "1h", &block)
8
+ return to_enum(scroll, scroll: scroll) unless block
9
+
10
+ request_payload = {
11
+ index: index_name,
12
+ scroll: scroll,
13
+ body: build.to_h
14
+ }
15
+
16
+ total = 0
17
+ progress = 0
18
+ requested_at = nil
19
+ last_response_time = nil
20
+
21
+ results = instrument(:scroll_search) do |event_payload|
22
+ response = client.search(request_payload)
23
+ last_response_time = Time.current
24
+
25
+ total = response["hits"]["total"]["value"]
26
+ progress += response["hits"]["hits"].size
27
+
28
+ event_payload[:request] = request_payload
29
+ event_payload[:response] = response
30
+ event_payload[:total] = total
31
+ event_payload[:progress] = progress
32
+
33
+ response
34
+ end
35
+
36
+ scroll_id = results["_scroll_id"]
37
+ hits = results["hits"]["hits"]
38
+
39
+ yield hits, { progress: progress, total: total, scroll_id: scroll_id }
40
+
41
+ while progress < total
42
+ request_payload = {
43
+ scroll_id: scroll_id,
44
+ scroll: scroll
45
+ }
46
+
47
+ requested_at = Time.current
48
+
49
+ results = instrument(:scroll, scroll: scroll_id) do |event_payload|
50
+ response = client.scroll(request_payload)
51
+ last_response_time = Time.current
52
+
53
+ total = response["hits"]["total"]["value"]
54
+ progress += response["hits"]["hits"].size
55
+
56
+ event_payload[:request] = request_payload
57
+ event_payload[:response] = response
58
+ event_payload[:total] = total
59
+ event_payload[:progress] = progress
60
+
61
+ response
62
+ rescue Elastic::Transport::Transport::Errors::NotFound => e
63
+ raise_enhance_message_when_scroll_failed(e, scroll, requested_at, last_response_time)
64
+ end
65
+
66
+ hits = results["hits"]["hits"]
67
+ progress += hits.size
68
+
69
+ break if hits.empty?
70
+
71
+ yield hits, { progress: progress, total: total, scroll_id: scroll_id }
72
+ end
73
+
74
+ total
75
+ ensure
76
+ clear_scroll(scroll_id) if scroll_id
77
+ end
78
+
79
+ def clear_scroll(scroll_id)
80
+ client.clear_scroll(scroll_id: scroll_id)
81
+ rescue ::Elastic::Transport::Transport::Errors::NotFound
82
+ # We dont care if the scroll ID is already expired
83
+ end
84
+
85
+ def scroll_records_in_batches(**options)
86
+ return to_enum(:scroll_records_in_batches, **options) unless block_given?
87
+
88
+ scroll(**options) do |hits|
89
+ ids = hits.map { |doc| doc["_id"] }
90
+ yield model.where(id: ids)
91
+ model.connection.clear_query_cache
92
+ end
93
+ end
94
+
95
+ def scroll_records(**options, &block)
96
+ return to_enum(:scroll_records, **options) unless block
97
+
98
+ scroll_records_in_batches(**options) do |relation|
99
+ relation.each(&block)
100
+ end
101
+ end
102
+
103
+ private
104
+
105
+ def raise_enhance_message_when_scroll_failed(error, scroll, requested_at, last_response_time)
106
+ elapsed = (requested_at - last_response_time).round(1).seconds
107
+
108
+ raise error.exception("Scroll registered for #{scroll}, #{elapsed.inspect} elapsed between. #{error.message}")
109
+ end
110
+ end
111
+ end
112
+ end
113
+ end