jay_api 28.2.0 → 28.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/lib/jay_api/elasticsearch/async.rb +2 -2
- data/lib/jay_api/elasticsearch/index.rb +12 -181
- data/lib/jay_api/elasticsearch/indexable.rb +225 -0
- data/lib/jay_api/elasticsearch/indexes.rb +34 -0
- data/lib/jay_api/elasticsearch/query_builder/aggregations/composite.rb +77 -0
- data/lib/jay_api/elasticsearch/query_builder/aggregations/sources/sources.rb +46 -0
- data/lib/jay_api/elasticsearch/query_builder/aggregations/sources/terms.rb +56 -0
- data/lib/jay_api/elasticsearch/query_builder/aggregations.rb +11 -0
- data/lib/jay_api/elasticsearch/query_results.rb +2 -2
- data/lib/jay_api/elasticsearch.rb +1 -0
- data/lib/jay_api/version.rb +1 -1
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c1af59670e04b1508c6773538b8d9d0478f1971e80755ea526012eaa44f4717c
|
4
|
+
data.tar.gz: d23445b944fd8ef0cea8fd628536132fe2cd024ec315b286223b08ecb2c57452
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f6805dc6330cdb960f50af2321112572f9959da7789e02c2810231df24b7a5176686754bb488234e4f64659dbdbdc078985564e5987780cd66e93aaca0e5d0e4
|
7
|
+
data.tar.gz: ad80fac10a256c1ff8061cc6822fe9d89763e86f9e710cf0b9e7a8f103f864d4dfd24051c90727fb7fc664a0251503a97515beb1d2cd890b3a4cd43a50066490
|
data/CHANGELOG.md
CHANGED
@@ -8,6 +8,18 @@ Please mark backwards incompatible changes with an exclamation mark at the start
|
|
8
8
|
|
9
9
|
## [Unreleased]
|
10
10
|
|
11
|
+
## [28.4.0] - 2025-08-26
|
12
|
+
|
13
|
+
### Added
|
14
|
+
- The `Elasticsearch::Indexes` class. A class which allows multiple indexes to
|
15
|
+
be used (fed or queried) at the same time.
|
16
|
+
|
17
|
+
## [28.3.0] - 2025-06-05
|
18
|
+
|
19
|
+
### Added
|
20
|
+
- The `Aggregations::Composite` class and the `Aggregations#composite` method.
|
21
|
+
They make it possible to use Elasticsearch's `composite` aggregations.
|
22
|
+
|
11
23
|
## [28.2.0] - 2025-05-30
|
12
24
|
|
13
25
|
### Added
|
@@ -19,8 +19,8 @@ module JayAPI
|
|
19
19
|
|
20
20
|
def_delegators :index, :index_name
|
21
21
|
|
22
|
-
# @param [JayAPI::Elasticsearch::
|
23
|
-
# which to execute asynchronous operations
|
22
|
+
# @param [JayAPI::Elasticsearch::Indexable] index The elasticsearch
|
23
|
+
# index or indexes on which to execute asynchronous operations
|
24
24
|
def initialize(index)
|
25
25
|
@index = index
|
26
26
|
end
|
@@ -1,57 +1,28 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
4
|
-
require 'active_support/json' # Needed because ActiveSupport 6 doesn't include it's own JSON module. 🤦
|
5
|
-
require 'active_support/core_ext/hash/indifferent_access'
|
6
|
-
require 'active_support/core_ext/hash/keys'
|
7
|
-
require 'elasticsearch'
|
8
|
-
require 'logging'
|
9
|
-
|
10
|
-
require_relative 'errors/elasticsearch_error'
|
11
|
-
require_relative 'async'
|
12
|
-
require_relative 'query_results'
|
13
|
-
require_relative 'response'
|
14
|
-
require_relative 'batch_counter'
|
15
|
-
require_relative 'search_after_results'
|
3
|
+
require_relative 'indexable'
|
16
4
|
|
17
5
|
module JayAPI
|
18
6
|
module Elasticsearch
|
19
7
|
# Represents an Elasticsearch index. Allows data to be pushed to it one
|
20
8
|
# record at a time or in batches of the specified size.
|
21
9
|
class Index
|
22
|
-
|
23
|
-
|
24
|
-
# Default type for documents indexed with the #index method.
|
25
|
-
DEFAULT_DOC_TYPE = 'nested'
|
10
|
+
include ::JayAPI::Elasticsearch::Indexable
|
26
11
|
|
27
|
-
# Supported document types (for the #index method)
|
28
|
-
SUPPORTED_TYPES = [DEFAULT_DOC_TYPE, nil].freeze
|
29
|
-
|
30
|
-
# :reek:ControlParameter (want to avoid the creating of the logger on method definition)
|
31
|
-
# Creates a new instance of the class.
|
32
12
|
# @param [JayAPI::Elasticsearch::Client] client The Elasticsearch Client object.
|
33
13
|
# @param [String] index_name The name of the Elasticsearch index.
|
34
|
-
# @param [Integer] batch_size The size of the batch. When this
|
35
|
-
#
|
36
|
-
#
|
14
|
+
# @param [Integer] batch_size The size of the batch. When this many items
|
15
|
+
# are pushed into the index they are flushed to the Elasticsearch
|
16
|
+
# instance.
|
37
17
|
# @param [Logging::Logger, nil] logger The logger object to use, if
|
38
18
|
# none is given a new one will be created.
|
39
19
|
def initialize(client:, index_name:, batch_size: 100, logger: nil)
|
40
|
-
|
41
|
-
|
42
|
-
@client = client
|
43
|
-
@index_name = index_name
|
44
|
-
@batch_size = batch_size
|
45
|
-
|
46
|
-
@batch = []
|
20
|
+
super(client: client, index_names: [index_name], batch_size: batch_size, logger: logger)
|
47
21
|
end
|
48
22
|
|
49
|
-
#
|
50
|
-
|
51
|
-
|
52
|
-
def push(data)
|
53
|
-
@batch << { index: { _index: index_name, _type: 'nested', data: data } }
|
54
|
-
flush! if @batch.size >= batch_size
|
23
|
+
# @return [String] The name of the Elasticsearch index.
|
24
|
+
def index_name
|
25
|
+
@index_name ||= index_names.first
|
55
26
|
end
|
56
27
|
|
57
28
|
# Sends a record to the Elasticsearch instance right away.
|
@@ -59,8 +30,8 @@ module JayAPI
|
|
59
30
|
# @param [String, nil] type The type of the document. When set to +nil+
|
60
31
|
# the decision is left to Elasticsearch's API. Which will normally
|
61
32
|
# default to +_doc+.
|
62
|
-
# @return [Hash] A
|
63
|
-
# example of such Hash is:
|
33
|
+
# @return [Hash] A Hash containing information about the created document.
|
34
|
+
# An example of such Hash is:
|
64
35
|
#
|
65
36
|
# {
|
66
37
|
# "_index" => "xyz01_unit_test",
|
@@ -76,147 +47,7 @@ module JayAPI
|
|
76
47
|
# For information on the contents of this Hash please see:
|
77
48
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#docs-index-api-response-body
|
78
49
|
def index(data, type: DEFAULT_DOC_TYPE)
|
79
|
-
|
80
|
-
|
81
|
-
client.index index: index_name, type: type, body: data
|
82
|
-
end
|
83
|
-
|
84
|
-
# Performs a query on the index.
|
85
|
-
# For more information on how to build the query please refer to the
|
86
|
-
# Elasticsearch DSL query:
|
87
|
-
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
88
|
-
# @param [Hash] query The query to perform.
|
89
|
-
# @param [JayAPI::Elasticsearch::BatchCounter, nil] batch_counter Object keeping track of batches.
|
90
|
-
# @param [Symbol, nil] type Type of query, at the moment either nil or :search_after.
|
91
|
-
# @return [JayAPI::Elasticsearch::QueryResults] The query results.
|
92
|
-
# @raise [Elasticsearch::Transport::Transport::ServerError] If the
|
93
|
-
# query fails.
|
94
|
-
def search(query, batch_counter: nil, type: nil)
|
95
|
-
begin
|
96
|
-
response = Response.new(client.search(index: index_name, body: query))
|
97
|
-
rescue ::Elasticsearch::Transport::Transport::Errors::BadRequest
|
98
|
-
logger.error "The 'search' query is invalid: #{JSON.pretty_generate(query)}"
|
99
|
-
raise
|
100
|
-
end
|
101
|
-
query_results(query, response, batch_counter, type)
|
102
|
-
end
|
103
|
-
|
104
|
-
# Sends whatever is currently in the send queue to the Elasticsearch
|
105
|
-
# instance and clears the queue.
|
106
|
-
def flush
|
107
|
-
return unless @batch.any?
|
108
|
-
|
109
|
-
flush!
|
110
|
-
end
|
111
|
-
|
112
|
-
# Returns the number of elements currently on the send queue.
|
113
|
-
# @return [Integer] The number of items in the send queue.
|
114
|
-
def queue_size
|
115
|
-
@batch.size
|
116
|
-
end
|
117
|
-
|
118
|
-
# Delete the documents matching the given query from the Index.
|
119
|
-
# For more information on how to build the query please refer to the
|
120
|
-
# Elasticsearch DSL documentation:
|
121
|
-
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
122
|
-
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
|
123
|
-
# @param [Hash] query The delete query
|
124
|
-
# @param [Integer] slices Number of slices to cut the operation into for
|
125
|
-
# faster processing (i.e., run the operation in parallel)
|
126
|
-
# @param [Boolean] wait_for_completion False if Elasticsearch should not
|
127
|
-
# wait for completion and perform the request asynchronously, true if it
|
128
|
-
# should wait for completion (i.e., run the operation asynchronously)
|
129
|
-
# @return [Hash] A Hash that details the results of the operation
|
130
|
-
# @example Returned Hash (with `wait_for_completion: true`):
|
131
|
-
# {
|
132
|
-
# took: 103,
|
133
|
-
# timed_out: false,
|
134
|
-
# total: 76,
|
135
|
-
# deleted: 76,
|
136
|
-
# batches: 1,
|
137
|
-
# version_conflicts: 0,
|
138
|
-
# noops: 0,
|
139
|
-
# retries: { bulk: 0, search: 0 },
|
140
|
-
# throttled_millis: 0,
|
141
|
-
# requests_per_second: 1.0,
|
142
|
-
# throttled_until_millis: 0,
|
143
|
-
# failures: []
|
144
|
-
# }
|
145
|
-
# @example Returned Hash (with `wait_for_completion: false`):
|
146
|
-
# {
|
147
|
-
# task: "B5oDyEsHQu2Q-wpbaMSMTg:577388264"
|
148
|
-
# }
|
149
|
-
# @raise [Elasticsearch::Transport::Transport::ServerError] If the
|
150
|
-
# query fails.
|
151
|
-
def delete_by_query(query, slices: nil, wait_for_completion: true)
|
152
|
-
request_params = { index: index_name, body: query }.tap do |params|
|
153
|
-
params.merge!(slices: slices) if slices
|
154
|
-
params.merge!(wait_for_completion: false) unless wait_for_completion
|
155
|
-
end
|
156
|
-
|
157
|
-
client.delete_by_query(**request_params).deep_symbolize_keys
|
158
|
-
end
|
159
|
-
|
160
|
-
# Deletes asynchronously the documents matching the given query from the
|
161
|
-
# Index.
|
162
|
-
# For more information on how to build the query please refer to the
|
163
|
-
# Elasticsearch DSL documentation:
|
164
|
-
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
165
|
-
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
|
166
|
-
# @param [Hash] query The delete query
|
167
|
-
# @param [Integer, String] slices Number of slices to cut the operation
|
168
|
-
# into for faster processing (i.e., run the operation in parallel). Use
|
169
|
-
# "auto" to make elasticsearch decide how many slices to divide into
|
170
|
-
# @return [Concurrent::Promise] The eventual value returned from the single
|
171
|
-
# completion of the delete operation
|
172
|
-
def delete_by_query_async(query, slices: nil)
|
173
|
-
async.delete_by_query(query, slices: slices)
|
174
|
-
end
|
175
|
-
|
176
|
-
private
|
177
|
-
|
178
|
-
attr_reader :logger, :batch
|
179
|
-
|
180
|
-
# Flushes the current send queue to the Elasticsearch instance and
|
181
|
-
# clears the queue.
|
182
|
-
def flush!
|
183
|
-
logger.info "Pushing data to Elasticsearch (#{batch.size} records)..."
|
184
|
-
response = client.bulk body: batch
|
185
|
-
|
186
|
-
handle_errors(response) if response['errors']
|
187
|
-
|
188
|
-
logger.info 'Done'
|
189
|
-
@batch = []
|
190
|
-
end
|
191
|
-
|
192
|
-
# @param [Hash] query The elastic search query.
|
193
|
-
# @param [JayAPI::Elasticsearch::Response] response The response to the query.
|
194
|
-
# @param [JayAPI::Elasticsearch::BatchCounter, nil] batch_counter Object keeping track of batches.
|
195
|
-
# @param [Symbol, nil] type Type of query, at the moment either nil or :search_after.
|
196
|
-
# @return [QueryResults]
|
197
|
-
def query_results(query, response, batch_counter, type)
|
198
|
-
(type == :search_after ? SearchAfterResults : QueryResults).new(
|
199
|
-
index: self,
|
200
|
-
query: query,
|
201
|
-
response: response,
|
202
|
-
batch_counter: BatchCounter.create_or_update(batch_counter, query, response.size)
|
203
|
-
)
|
204
|
-
end
|
205
|
-
|
206
|
-
# Scans the Elasticsearch response in search for the first item that has
|
207
|
-
# en erroneous state and raises an error including the error details.
|
208
|
-
# @param [Hash] response The response returned by the Elasticsearch client.
|
209
|
-
# @raise [Errors::ElasticsearchError] Is always raised.
|
210
|
-
def handle_errors(response)
|
211
|
-
error_item = response['items'].find { |item| item['index']['error'] }
|
212
|
-
|
213
|
-
raise Errors::ElasticsearchError,
|
214
|
-
"An error occurred when pushing the data to Elasticsearch:\n#{error_item['index']['error'].inspect}"
|
215
|
-
end
|
216
|
-
|
217
|
-
# @return [JayAPI::Elasticsearch::Async]
|
218
|
-
def async
|
219
|
-
@async ||= JayAPI::Elasticsearch::Async.new(self)
|
50
|
+
super.first
|
220
51
|
end
|
221
52
|
end
|
222
53
|
end
|
@@ -0,0 +1,225 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support'
|
4
|
+
require 'active_support/json' # Needed because ActiveSupport 6 doesn't include it's own JSON module. 🤦
|
5
|
+
require 'active_support/core_ext/hash/keys'
|
6
|
+
require 'elasticsearch'
|
7
|
+
require 'logging'
|
8
|
+
|
9
|
+
require_relative 'async'
|
10
|
+
require_relative 'batch_counter'
|
11
|
+
require_relative 'errors/elasticsearch_error'
|
12
|
+
require_relative 'query_results'
|
13
|
+
require_relative 'response'
|
14
|
+
require_relative 'search_after_results'
|
15
|
+
|
16
|
+
module JayAPI
|
17
|
+
module Elasticsearch
|
18
|
+
# This module houses the Elasticsearch methods that can be used with a
|
19
|
+
# single or with multiple indexes. Its main purpose is to avoid code
|
20
|
+
# repetition between classes.
|
21
|
+
module Indexable
|
22
|
+
# Default type for documents indexed with the #index method.
|
23
|
+
DEFAULT_DOC_TYPE = 'nested'
|
24
|
+
|
25
|
+
# Supported document types (for the #index method)
|
26
|
+
SUPPORTED_TYPES = [DEFAULT_DOC_TYPE, nil].freeze
|
27
|
+
|
28
|
+
attr_reader :client, :batch_size
|
29
|
+
|
30
|
+
# :reek:ControlParameter (want to avoid the creating of the logger on method definition)
|
31
|
+
# @param [JayAPI::Elasticsearch::Client] client The Elasticsearch Client object.
|
32
|
+
# @param [Array<String>] index_names The names of the Elasticsearch indexes.
|
33
|
+
# @param [Integer] batch_size The size of the batch. When this many items
|
34
|
+
# are pushed into the index they are flushed to the Elasticsearch
|
35
|
+
# instance.
|
36
|
+
# @param [Logging::Logger, nil] logger The logger object to use, if
|
37
|
+
# none is given a new one will be created.
|
38
|
+
def initialize(client:, index_names:, batch_size: 100, logger: nil)
|
39
|
+
@logger = logger || Logging.logger[self]
|
40
|
+
|
41
|
+
@client = client
|
42
|
+
@index_names = index_names
|
43
|
+
@batch_size = batch_size
|
44
|
+
|
45
|
+
@batch = []
|
46
|
+
end
|
47
|
+
|
48
|
+
# Pushes a record into the index. (This does not send the record to the
|
49
|
+
# Elasticsearch instance, only puts it into the send queue).
|
50
|
+
# @param [Hash] data The data to be pushed to the index.
|
51
|
+
def push(data)
|
52
|
+
index_names.each do |index_name|
|
53
|
+
batch << { index: { _index: index_name, _type: 'nested', data: data } }
|
54
|
+
end
|
55
|
+
|
56
|
+
flush! if batch.size >= batch_size
|
57
|
+
end
|
58
|
+
|
59
|
+
# Sends a record to the Elasticsearch instance right away.
|
60
|
+
# @param [Hash] data The data to be sent.
|
61
|
+
# @param [String, nil] type The type of the document. When set to +nil+
|
62
|
+
# the decision is left to Elasticsearch's API. Which will normally
|
63
|
+
# default to +_doc+.
|
64
|
+
# @return [Array<Hash>] An array with hashes containing information about
|
65
|
+
# the created documents. An example of such Hashes is:
|
66
|
+
#
|
67
|
+
# {
|
68
|
+
# "_index" => "xyz01_unit_test",
|
69
|
+
# "_type" => "nested",
|
70
|
+
# "_id" => "SVY1mJEBQ5CNFZM8Lodt",
|
71
|
+
# "_version" => 1,
|
72
|
+
# "result" => "created",
|
73
|
+
# "_shards" => { "total" => 2, "successful" => 1, "failed" => 0 },
|
74
|
+
# "_seq_no" => 0,
|
75
|
+
# "_primary_term" => 1
|
76
|
+
# }
|
77
|
+
#
|
78
|
+
# For information on the contents of this Hash please see:
|
79
|
+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#docs-index-api-response-body
|
80
|
+
def index(data, type: DEFAULT_DOC_TYPE)
|
81
|
+
raise ArgumentError, "Unsupported type: '#{type}'" unless SUPPORTED_TYPES.include?(type)
|
82
|
+
|
83
|
+
index_names.map { |index_name| client.index index: index_name, type: type, body: data }
|
84
|
+
end
|
85
|
+
|
86
|
+
# Performs a query on the index.
|
87
|
+
# For more information on how to build the query please refer to the
|
88
|
+
# Elasticsearch DSL query:
|
89
|
+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
90
|
+
# @param [Hash] query The query to perform.
|
91
|
+
# @param [JayAPI::Elasticsearch::BatchCounter, nil] batch_counter Object keeping track of batches.
|
92
|
+
# @param [Symbol, nil] type Type of query, at the moment either nil or :search_after.
|
93
|
+
# @return [JayAPI::Elasticsearch::QueryResults] The query results.
|
94
|
+
# @raise [Elasticsearch::Transport::Transport::ServerError] If the
|
95
|
+
# query fails.
|
96
|
+
def search(query, batch_counter: nil, type: nil)
|
97
|
+
begin
|
98
|
+
response = Response.new(client.search(index: index_names, body: query))
|
99
|
+
rescue ::Elasticsearch::Transport::Transport::Errors::BadRequest
|
100
|
+
logger.error "The 'search' query is invalid: #{JSON.pretty_generate(query)}"
|
101
|
+
raise
|
102
|
+
end
|
103
|
+
query_results(query, response, batch_counter, type)
|
104
|
+
end
|
105
|
+
|
106
|
+
# Sends whatever is currently in the send queue to the Elasticsearch
|
107
|
+
# instance and clears the queue.
|
108
|
+
def flush
|
109
|
+
return unless @batch.any?
|
110
|
+
|
111
|
+
flush!
|
112
|
+
end
|
113
|
+
|
114
|
+
# Returns the number of elements currently on the send queue.
|
115
|
+
# @return [Integer] The number of items in the send queue.
|
116
|
+
def queue_size
|
117
|
+
batch.size
|
118
|
+
end
|
119
|
+
|
120
|
+
# Delete the documents matching the given query from the Index.
|
121
|
+
# For more information on how to build the query please refer to the
|
122
|
+
# Elasticsearch DSL documentation:
|
123
|
+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
124
|
+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
|
125
|
+
# @param [Hash] query The delete query
|
126
|
+
# @param [Integer] slices Number of slices to cut the operation into for
|
127
|
+
# faster processing (i.e., run the operation in parallel)
|
128
|
+
# @param [Boolean] wait_for_completion False if Elasticsearch should not
|
129
|
+
# wait for completion and perform the request asynchronously, true if it
|
130
|
+
# should wait for completion (i.e., run the operation synchronously)
|
131
|
+
# @return [Hash] A Hash that details the results of the operation
|
132
|
+
# @example Returned Hash (with `wait_for_completion: true`):
|
133
|
+
# {
|
134
|
+
# took: 103,
|
135
|
+
# timed_out: false,
|
136
|
+
# total: 76,
|
137
|
+
# deleted: 76,
|
138
|
+
# batches: 1,
|
139
|
+
# version_conflicts: 0,
|
140
|
+
# noops: 0,
|
141
|
+
# retries: { bulk: 0, search: 0 },
|
142
|
+
# throttled_millis: 0,
|
143
|
+
# requests_per_second: 1.0,
|
144
|
+
# throttled_until_millis: 0,
|
145
|
+
# failures: []
|
146
|
+
# }
|
147
|
+
# @example Returned Hash (with `wait_for_completion: false`):
|
148
|
+
# {
|
149
|
+
# task: "B5oDyEsHQu2Q-wpbaMSMTg:577388264"
|
150
|
+
# }
|
151
|
+
# @raise [Elasticsearch::Transport::Transport::ServerError] If the
|
152
|
+
# query fails.
|
153
|
+
def delete_by_query(query, slices: nil, wait_for_completion: true)
|
154
|
+
request_params = { index: index_names, body: query }.tap do |params|
|
155
|
+
params.merge!(slices: slices) if slices
|
156
|
+
params.merge!(wait_for_completion: false) unless wait_for_completion
|
157
|
+
end
|
158
|
+
|
159
|
+
client.delete_by_query(**request_params).deep_symbolize_keys
|
160
|
+
end
|
161
|
+
|
162
|
+
# Deletes asynchronously the documents matching the given query from the
|
163
|
+
# Index.
|
164
|
+
# For more information on how to build the query please refer to the
|
165
|
+
# Elasticsearch DSL documentation:
|
166
|
+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
167
|
+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
|
168
|
+
# @param [Hash] query The delete query
|
169
|
+
# @param [Integer, String] slices Number of slices to cut the operation
|
170
|
+
# into for faster processing (i.e., run the operation in parallel). Use
|
171
|
+
# "auto" to make Elasticsearch decide how many slices to divide into
|
172
|
+
# @return [Concurrent::Promise] The eventual value returned from the single
|
173
|
+
# completion of the delete operation
|
174
|
+
def delete_by_query_async(query, slices: nil)
|
175
|
+
async.delete_by_query(query, slices: slices)
|
176
|
+
end
|
177
|
+
|
178
|
+
private
|
179
|
+
|
180
|
+
attr_reader :index_names, :logger, :batch
|
181
|
+
|
182
|
+
# Scans the Elasticsearch response in search for the first item that has
|
183
|
+
# an erroneous state and raises an error including the error details.
|
184
|
+
# @param [Hash] response The response returned by the Elasticsearch client.
|
185
|
+
# @raise [Errors::ElasticsearchError] Is always raised.
|
186
|
+
def handle_errors(response)
|
187
|
+
error_item = response['items'].find { |item| item['index']['error'] }
|
188
|
+
|
189
|
+
raise Errors::ElasticsearchError,
|
190
|
+
"An error occurred when pushing the data to Elasticsearch:\n#{error_item['index']['error'].inspect}"
|
191
|
+
end
|
192
|
+
|
193
|
+
# Flushes the current send queue to the Elasticsearch instance and
|
194
|
+
# clears the queue.
|
195
|
+
def flush!
|
196
|
+
logger.info "Pushing data to Elasticsearch (#{batch.size} records)..."
|
197
|
+
response = client.bulk body: batch
|
198
|
+
|
199
|
+
handle_errors(response) if response['errors']
|
200
|
+
|
201
|
+
logger.info 'Done'
|
202
|
+
@batch = []
|
203
|
+
end
|
204
|
+
|
205
|
+
# @param [Hash] query The elastic search query.
|
206
|
+
# @param [JayAPI::Elasticsearch::Response] response The response to the query.
|
207
|
+
# @param [JayAPI::Elasticsearch::BatchCounter, nil] batch_counter Object keeping track of batches.
|
208
|
+
# @param [Symbol, nil] type Type of query, at the moment either nil or :search_after.
|
209
|
+
# @return [QueryResults]
|
210
|
+
def query_results(query, response, batch_counter, type)
|
211
|
+
(type == :search_after ? SearchAfterResults : QueryResults).new(
|
212
|
+
index: self,
|
213
|
+
query: query,
|
214
|
+
response: response,
|
215
|
+
batch_counter: BatchCounter.create_or_update(batch_counter, query, response.size)
|
216
|
+
)
|
217
|
+
end
|
218
|
+
|
219
|
+
# @return [JayAPI::Elasticsearch::Async]
|
220
|
+
def async
|
221
|
+
@async ||= JayAPI::Elasticsearch::Async.new(self)
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'indexable'
|
4
|
+
|
5
|
+
module JayAPI
|
6
|
+
module Elasticsearch
|
7
|
+
# Represents a group of Elasticsearch indexes. Allows the execution of
|
8
|
+
# searches over all of the specified indexes or push data to all of them
|
9
|
+
# at the same time.
|
10
|
+
class Indexes
|
11
|
+
include ::JayAPI::Elasticsearch::Indexable
|
12
|
+
|
13
|
+
# @param [JayAPI::Elasticsearch::Client] client The Elasticsearch Client object.
|
14
|
+
# @param [Array<String>] index_names The names of the Elasticsearch indexes.
|
15
|
+
# @param [Integer] batch_size The size of the batch. When this many items
|
16
|
+
# are pushed into the indexes they are flushed to the Elasticsearch
|
17
|
+
# instance.
|
18
|
+
# @param [Logging::Logger, nil] logger The logger object to use, if
|
19
|
+
# none is given a new one will be created.
|
20
|
+
def initialize(client:, index_names:, batch_size: 100, logger: nil)
|
21
|
+
super
|
22
|
+
|
23
|
+
return if (batch_size % index_names.size).zero?
|
24
|
+
|
25
|
+
self.logger.warn(
|
26
|
+
"'batch_size' is not a multiple of the number of elements in 'index_names'. " \
|
27
|
+
"This can lead to a _bulk size slightly bigger than 'batch_size'"
|
28
|
+
)
|
29
|
+
end
|
30
|
+
|
31
|
+
attr_reader :index_names
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support'
|
4
|
+
require 'active_support/core_ext/string/inflections'
|
5
|
+
|
6
|
+
require_relative 'aggregation'
|
7
|
+
require_relative 'sources/sources'
|
8
|
+
require_relative 'errors/aggregations_error'
|
9
|
+
|
10
|
+
module JayAPI
|
11
|
+
module Elasticsearch
|
12
|
+
class QueryBuilder
|
13
|
+
class Aggregations
|
14
|
+
# Represents a Composite aggregation in Elasticsearch. For more
|
15
|
+
# information about this type of aggregation:
|
16
|
+
# @see https://www.elastic.co/docs/reference/aggregations/search-aggregations-bucket-composite-aggregation
|
17
|
+
class Composite < ::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Aggregation
|
18
|
+
attr_reader :size
|
19
|
+
|
20
|
+
# @param [String] name The name of the composite aggregation.
|
21
|
+
# @param [Integer] size The number of composite buckets to return.
|
22
|
+
# @yieldparam [JayAPI::Elasticsearch::QueryBuilder::Aggregations::Sources::Sources]
|
23
|
+
# The collection of sources for the composite aggregation. This
|
24
|
+
# should be used by the caller to add sources to the composite
|
25
|
+
# aggregation.
|
26
|
+
# @raise [JayAPI::Elasticsearch::QueryBuilder::Aggregations::Errors::AggregationsError]
|
27
|
+
# If the method is called without a block.
|
28
|
+
def initialize(name, size: nil, &block)
|
29
|
+
unless block
|
30
|
+
raise(::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Errors::AggregationsError,
|
31
|
+
"The #{self.class.name.demodulize} aggregation must be initialized with a block")
|
32
|
+
end
|
33
|
+
|
34
|
+
super(name)
|
35
|
+
@size = size
|
36
|
+
block.call(sources)
|
37
|
+
end
|
38
|
+
|
39
|
+
# @return [self] A copy of the receiver. Sources and nested
|
40
|
+
# aggregations are also cloned.
|
41
|
+
def clone
|
42
|
+
# rubocop:disable Lint/EmptyBlock (The sources will be assigned later)
|
43
|
+
copy = self.class.new(name, size: size) {}
|
44
|
+
# rubocop:enable Lint/EmptyBlock
|
45
|
+
|
46
|
+
copy.aggregations = aggregations.clone
|
47
|
+
copy.sources = sources.clone
|
48
|
+
copy
|
49
|
+
end
|
50
|
+
|
51
|
+
# @return [Hash] The Hash representation of the +Aggregation+.
|
52
|
+
# Properly formatted for Elasticsearch.
|
53
|
+
def to_h
|
54
|
+
super do
|
55
|
+
{
|
56
|
+
composite: {
|
57
|
+
sources: sources.to_a,
|
58
|
+
size: size
|
59
|
+
}.compact
|
60
|
+
}
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
protected
|
65
|
+
|
66
|
+
attr_writer :sources # Used by the #clone method
|
67
|
+
|
68
|
+
# @return [JayAPI::Elasticsearch::QueryBuilder::Aggregations::Sources::Sources]
|
69
|
+
# The collection of sources of the composite aggregation.
|
70
|
+
def sources
|
71
|
+
@sources ||= ::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Sources::Sources.new
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'terms'
|
4
|
+
|
5
|
+
module JayAPI
|
6
|
+
module Elasticsearch
|
7
|
+
class QueryBuilder
|
8
|
+
class Aggregations
|
9
|
+
module Sources
|
10
|
+
# Represents the collection of sources for a Composite aggregation in
|
11
|
+
# Elasticsearch
|
12
|
+
class Sources
|
13
|
+
# Adds a +terms+ source to the collection.
|
14
|
+
# For information about the parameters:
|
15
|
+
# @see Sources::Terms#initialize
|
16
|
+
def terms(name, **kw_args)
|
17
|
+
sources.push(::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Sources::Terms.new(name, **kw_args))
|
18
|
+
end
|
19
|
+
|
20
|
+
# @return [Array<Hash>] Array representation of the collection of
|
21
|
+
# sources of the composite aggregation.
|
22
|
+
def to_a
|
23
|
+
sources.map(&:to_h)
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [self] A copy of the receiver (not a shallow clone, it
|
27
|
+
# clones all of the elements of the collection).
|
28
|
+
def clone
|
29
|
+
self.class.new.tap do |copy|
|
30
|
+
copy.sources.concat(sources.map(&:clone))
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
protected
|
35
|
+
|
36
|
+
# @return [Array<Object>] The array used to hold the collection of
|
37
|
+
# sources.
|
38
|
+
def sources
|
39
|
+
@sources ||= []
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module JayAPI
|
4
|
+
module Elasticsearch
|
5
|
+
class QueryBuilder
|
6
|
+
class Aggregations
|
7
|
+
module Sources
|
8
|
+
# Represents a "Terms" value source for a Composite aggregation.
|
9
|
+
# More information about this type of value source can be found here:
|
10
|
+
# https://www.elastic.co/docs/reference/aggregations/search-aggregations-bucket-composite-aggregation#_terms
|
11
|
+
class Terms
|
12
|
+
attr_reader :name, :field, :order, :missing_bucket, :missing_order
|
13
|
+
|
14
|
+
# @param [String] name The name for the value source.
|
15
|
+
# @param [String] field The field for the value source.
|
16
|
+
# @param [String, nil] order The order in which the values coming
|
17
|
+
# from this data source should be ordered, this can be either
|
18
|
+
# "asc" or "desc"
|
19
|
+
# @param [Boolean] missing_bucket Whether or not a bucket for the
|
20
|
+
# documents without a value in +field+ should be created.
|
21
|
+
# @param [String] missing_order Where to put the bucket for the
|
22
|
+
# documents with a missing value, either "first" or "last".
|
23
|
+
def initialize(name, field:, order: nil, missing_bucket: nil, missing_order: nil)
|
24
|
+
@name = name
|
25
|
+
@field = field
|
26
|
+
@order = order
|
27
|
+
@missing_bucket = missing_bucket
|
28
|
+
@missing_order = missing_order
|
29
|
+
end
|
30
|
+
|
31
|
+
# @return [self] A copy of the receiver.
|
32
|
+
def clone
|
33
|
+
self.class.new(
|
34
|
+
name, field: field, order: order, missing_bucket: missing_bucket, missing_order: missing_order
|
35
|
+
)
|
36
|
+
end
|
37
|
+
|
38
|
+
# @return [Hash] The hash representation for the value source.
|
39
|
+
def to_h
|
40
|
+
{
|
41
|
+
name => {
|
42
|
+
terms: {
|
43
|
+
field: field,
|
44
|
+
order: order,
|
45
|
+
missing_bucket: missing_bucket,
|
46
|
+
missing_order: missing_order
|
47
|
+
}.compact
|
48
|
+
}
|
49
|
+
}
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -5,6 +5,7 @@ require 'forwardable'
|
|
5
5
|
require_relative 'aggregations/aggregation'
|
6
6
|
require_relative 'aggregations/avg'
|
7
7
|
require_relative 'aggregations/cardinality'
|
8
|
+
require_relative 'aggregations/composite'
|
8
9
|
require_relative 'aggregations/date_histogram'
|
9
10
|
require_relative 'aggregations/filter'
|
10
11
|
require_relative 'aggregations/scripted_metric'
|
@@ -121,6 +122,16 @@ module JayAPI
|
|
121
122
|
)
|
122
123
|
end
|
123
124
|
|
125
|
+
# Adds a +composite+ aggregation. For more information about the parameters:
|
126
|
+
# @see JayAPI::Elasticsearch::QueryBuilder::Aggregations::Composite#initialize
|
127
|
+
def composite(name, size: nil, &block)
|
128
|
+
add(
|
129
|
+
::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Composite.new(
|
130
|
+
name, size: size, &block
|
131
|
+
)
|
132
|
+
)
|
133
|
+
end
|
134
|
+
|
124
135
|
# Returns a Hash with the correct format for the current list of
|
125
136
|
# aggregations. For example:
|
126
137
|
#
|
@@ -21,8 +21,8 @@ module JayAPI
|
|
21
21
|
def_delegators :response, :hits, :total, :size, :count, :first, :last, :any?, :empty?, :aggregations
|
22
22
|
|
23
23
|
# Creates a new instance of the class.
|
24
|
-
# @param [JayAPI::Elasticsearch::
|
25
|
-
# index
|
24
|
+
# @param [JayAPI::Elasticsearch::Indexable] index The Elasticsearch
|
25
|
+
# index or indexes over which the query should be performed.
|
26
26
|
# @param [Hash] query The query that produced the results.
|
27
27
|
# @param [JayAPI::Elasticsearch::Results] response An object containing Docs retrieved from Elasticsearch.
|
28
28
|
# @param [JayAPI::Elasticsearch::BatchCounter] batch_counter An object keeping track of the current batch.
|
@@ -6,6 +6,7 @@ require_relative 'elasticsearch/client'
|
|
6
6
|
require_relative 'elasticsearch/client_factory'
|
7
7
|
require_relative 'elasticsearch/errors'
|
8
8
|
require_relative 'elasticsearch/index'
|
9
|
+
require_relative 'elasticsearch/indexes'
|
9
10
|
require_relative 'elasticsearch/query_builder'
|
10
11
|
require_relative 'elasticsearch/query_results'
|
11
12
|
require_relative 'elasticsearch/response'
|
data/lib/jay_api/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jay_api
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 28.
|
4
|
+
version: 28.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Accenture-Industry X
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2025-
|
12
|
+
date: 2025-08-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -135,17 +135,22 @@ files:
|
|
135
135
|
- lib/jay_api/elasticsearch/errors/query_execution_timeout.rb
|
136
136
|
- lib/jay_api/elasticsearch/errors/search_after_error.rb
|
137
137
|
- lib/jay_api/elasticsearch/index.rb
|
138
|
+
- lib/jay_api/elasticsearch/indexable.rb
|
139
|
+
- lib/jay_api/elasticsearch/indexes.rb
|
138
140
|
- lib/jay_api/elasticsearch/query_builder.rb
|
139
141
|
- lib/jay_api/elasticsearch/query_builder/aggregations.rb
|
140
142
|
- lib/jay_api/elasticsearch/query_builder/aggregations/aggregation.rb
|
141
143
|
- lib/jay_api/elasticsearch/query_builder/aggregations/avg.rb
|
142
144
|
- lib/jay_api/elasticsearch/query_builder/aggregations/cardinality.rb
|
145
|
+
- lib/jay_api/elasticsearch/query_builder/aggregations/composite.rb
|
143
146
|
- lib/jay_api/elasticsearch/query_builder/aggregations/date_histogram.rb
|
144
147
|
- lib/jay_api/elasticsearch/query_builder/aggregations/errors.rb
|
145
148
|
- lib/jay_api/elasticsearch/query_builder/aggregations/errors/aggregations_error.rb
|
146
149
|
- lib/jay_api/elasticsearch/query_builder/aggregations/filter.rb
|
147
150
|
- lib/jay_api/elasticsearch/query_builder/aggregations/max.rb
|
148
151
|
- lib/jay_api/elasticsearch/query_builder/aggregations/scripted_metric.rb
|
152
|
+
- lib/jay_api/elasticsearch/query_builder/aggregations/sources/sources.rb
|
153
|
+
- lib/jay_api/elasticsearch/query_builder/aggregations/sources/terms.rb
|
149
154
|
- lib/jay_api/elasticsearch/query_builder/aggregations/sum.rb
|
150
155
|
- lib/jay_api/elasticsearch/query_builder/aggregations/terms.rb
|
151
156
|
- lib/jay_api/elasticsearch/query_builder/aggregations/top_hits.rb
|