elastomer-client 3.2.2 → 6.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.devcontainer/devcontainer.json +46 -0
- data/.devcontainer/postCreateCommand.sh +4 -0
- data/.github/dependabot.yaml +11 -0
- data/.github/workflows/main.yml +45 -0
- data/.github/workflows/rubocop.yml +15 -0
- data/.gitignore +1 -1
- data/.rubocop.yml +13 -65
- data/.ruby-version +1 -0
- data/CHANGELOG.md +76 -0
- data/Gemfile +18 -1
- data/README.md +110 -51
- data/Rakefile +3 -1
- data/docker/compose.yaml +71 -0
- data/docker/elasticsearch8plus.yml +13 -0
- data/docs/README.md +4 -5
- data/docs/bulk_indexing.md +1 -1
- data/docs/client.md +20 -33
- data/docs/cluster.md +8 -8
- data/docs/docs.md +5 -5
- data/docs/index.md +4 -4
- data/docs/multi_search.md +1 -1
- data/docs/notifications.md +3 -3
- data/docs/scan_scroll.md +1 -1
- data/docs/snapshots.md +1 -1
- data/docs/templates.md +1 -1
- data/elastomer-client.gemspec +7 -16
- data/lib/{elastomer → elastomer_client}/client/bulk.rb +70 -47
- data/lib/{elastomer → elastomer_client}/client/cluster.rb +18 -16
- data/lib/{elastomer → elastomer_client}/client/delete_by_query.rb +6 -4
- data/lib/{elastomer → elastomer_client}/client/docs.rb +82 -72
- data/lib/{elastomer → elastomer_client}/client/errors.rb +7 -17
- data/lib/{elastomer → elastomer_client}/client/index.rb +55 -79
- data/lib/{elastomer → elastomer_client}/client/multi_percolate.rb +7 -5
- data/lib/{elastomer → elastomer_client}/client/multi_search.rb +5 -3
- data/lib/{elastomer → elastomer_client}/client/native_delete_by_query.rb +6 -6
- data/lib/{elastomer → elastomer_client}/client/nodes.rb +11 -10
- data/lib/{elastomer → elastomer_client}/client/percolator.rb +9 -10
- data/lib/elastomer_client/client/reindex.rb +29 -0
- data/lib/{elastomer → elastomer_client}/client/repository.rb +7 -5
- data/lib/{elastomer → elastomer_client}/client/rest_api_spec/api_spec.rb +7 -6
- data/lib/{elastomer → elastomer_client}/client/rest_api_spec/api_spec_v5_6.rb +1 -1
- data/lib/elastomer_client/client/rest_api_spec/api_spec_v8_13.rb +7567 -0
- data/lib/elastomer_client/client/rest_api_spec/api_spec_v8_7.rb +6553 -0
- data/lib/{elastomer → elastomer_client}/client/rest_api_spec/rest_api.rb +6 -4
- data/lib/{elastomer → elastomer_client}/client/rest_api_spec.rb +3 -2
- data/lib/{elastomer → elastomer_client}/client/scroller.rb +17 -16
- data/lib/{elastomer → elastomer_client}/client/snapshot.rb +10 -8
- data/lib/{elastomer → elastomer_client}/client/tasks.rb +9 -13
- data/lib/{elastomer → elastomer_client}/client/template.rb +10 -9
- data/lib/elastomer_client/client/update_by_query.rb +50 -0
- data/lib/{elastomer → elastomer_client}/client.rb +51 -62
- data/lib/{elastomer → elastomer_client}/core_ext/time.rb +2 -0
- data/lib/{elastomer → elastomer_client}/middleware/compress.rb +2 -2
- data/lib/{elastomer → elastomer_client}/middleware/encode_json.rb +4 -2
- data/lib/{elastomer → elastomer_client}/middleware/limit_size.rb +5 -3
- data/lib/{elastomer → elastomer_client}/middleware/opaque_id.rb +10 -7
- data/lib/{elastomer → elastomer_client}/middleware/parse_json.rb +5 -3
- data/lib/{elastomer → elastomer_client}/notifications.rb +17 -15
- data/lib/elastomer_client/version.rb +9 -0
- data/lib/elastomer_client/version_support.rb +24 -0
- data/script/bootstrap +4 -2
- data/script/console +3 -1
- data/script/generate-rest-api-spec +77 -22
- data/test/assertions.rb +32 -39
- data/test/client/bulk_test.rb +165 -143
- data/test/client/cluster_test.rb +35 -13
- data/test/client/docs_test.rb +387 -274
- data/test/client/errors_test.rb +38 -40
- data/test/client/index_test.rb +243 -202
- data/test/client/multi_percolate_test.rb +46 -41
- data/test/client/multi_search_test.rb +122 -67
- data/test/client/native_delete_by_query_test.rb +96 -88
- data/test/client/nodes_test.rb +21 -10
- data/test/client/percolator_test.rb +19 -14
- data/test/client/reindex_test.rb +76 -0
- data/test/client/repository_test.rb +31 -19
- data/test/client/rest_api_spec/api_spec_test.rb +13 -11
- data/test/client/rest_api_spec/rest_api_test.rb +9 -7
- data/test/client/scroller_test.rb +44 -70
- data/test/client/snapshot_test.rb +38 -21
- data/test/client/stubbed_client_test.rb +7 -4
- data/test/client/tasks_test.rb +12 -17
- data/test/client/template_test.rb +34 -13
- data/test/client/update_by_query_test.rb +137 -0
- data/test/client_test.rb +158 -92
- data/test/core_ext/time_test.rb +14 -12
- data/test/middleware/encode_json_test.rb +18 -7
- data/test/middleware/opaque_id_test.rb +18 -14
- data/test/middleware/parse_json_test.rb +17 -9
- data/test/mock_response.rb +30 -0
- data/test/notifications_test.rb +15 -8
- data/test/test_helper.rb +40 -97
- data/test/version_support_test.rb +13 -78
- metadata +60 -208
- data/.overcommit.yml +0 -5
- data/.travis.yml +0 -34
- data/docker/docker-compose.cibuild.yml +0 -8
- data/docker/docker-compose.es24.yml +0 -34
- data/docker/docker-compose.es56.yml +0 -37
- data/docs/warmers.md +0 -3
- data/lib/elastomer/client/app_delete_by_query.rb +0 -144
- data/lib/elastomer/client/rest_api_spec/api_spec_v2_3.rb +0 -2232
- data/lib/elastomer/client/rest_api_spec/api_spec_v2_4.rb +0 -2250
- data/lib/elastomer/client/warmer.rb +0 -98
- data/lib/elastomer/version.rb +0 -7
- data/lib/elastomer/version_support.rb +0 -182
- data/script/cibuild +0 -103
- data/script/cibuild-elastomer-client +0 -1
- data/script/cibuild-elastomer-client-es24 +0 -8
- data/script/cibuild-elastomer-client-es56 +0 -8
- data/test/client/app_delete_by_query_test.rb +0 -192
- data/test/client/es_5_x_warmer_test.rb +0 -13
- data/test/client/warmer_test.rb +0 -60
data/docs/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient in Depth
|
2
2
|
|
3
|
-
We first started building the
|
3
|
+
We first started building the ElastomerClient gem when an
|
4
4
|
[official client](https://github.com/elasticsearch/elasticsearch-ruby)
|
5
5
|
was not yet available from Elasticsearch. We were looking for a client that
|
6
6
|
provided a one-to-one mapping of the Elasticsearch APIs and avoided higher level
|
@@ -16,10 +16,10 @@ To that end we have tried to be as faithful as possible to the Elasticsearch API
|
|
16
16
|
with our implementation. There are a few places where it made sense to wrap the
|
17
17
|
Elasticsearch API inside Ruby idioms. One notable location is the
|
18
18
|
[scan-scroll](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html)
|
19
|
-
search type; the
|
19
|
+
search type; the ElastomerClient provides a Ruby iterator to work with these
|
20
20
|
types of queries.
|
21
21
|
|
22
|
-
Below are links to documents describing the various components of the
|
22
|
+
Below are links to documents describing the various components of the ElastomerClient
|
23
23
|
Client library. Start with the core components - specifically the **Client**
|
24
24
|
document. All the other components are built atop the client.
|
25
25
|
|
@@ -30,7 +30,6 @@ document. All the other components are built atop the client.
|
|
30
30
|
* [Documents](docs.md)
|
31
31
|
* [Cluster](cluster.md)
|
32
32
|
* [Templates](templates.md)
|
33
|
-
* [Warmers](warmers.md)
|
34
33
|
|
35
34
|
**Bulk Components**
|
36
35
|
|
data/docs/bulk_indexing.md
CHANGED
data/docs/client.md
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient Client Component
|
2
2
|
|
3
|
-
All methods in the
|
4
|
-
Elasticsearch. The [`
|
3
|
+
All methods in the ElastomerClient gem eventually make an HTTP request to
|
4
|
+
Elasticsearch. The [`ElastomerClient::Client`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client.rb)
|
5
5
|
class is responsible for connecting to an Elasticsearch instance, making HTTP
|
6
6
|
requests, processing the response, and handling errors. Let's look at the
|
7
|
-
details of how `
|
7
|
+
details of how `ElastomerClient::Client` handles HTTP communication.
|
8
8
|
|
9
9
|
### Connecting
|
10
10
|
|
@@ -15,36 +15,23 @@ the concept of *middlewares* that operate on the HTTP request and response. We
|
|
15
15
|
use Faraday middleware to encode and decode JSON messages exchanged with
|
16
16
|
Elasticsearch.
|
17
17
|
|
18
|
-
Without any options the `
|
18
|
+
Without any options the `ElastomerClient::Client` will connect to the default
|
19
19
|
Elasticsearch URL `http://localhost:9200`. The `Net:HTTP` client from the Ruby
|
20
20
|
standard library will be used.
|
21
21
|
|
22
22
|
```ruby
|
23
|
-
client =
|
23
|
+
client = ElastomerClient::Client.new
|
24
24
|
client.host #=> 'localhost'
|
25
25
|
client.port #=> 9200
|
26
26
|
client.url #=> 'http://localhost:9200'
|
27
27
|
```
|
28
28
|
|
29
|
-
[Boxen](https://boxen.github.com) configures Elasticsearch to listen on port
|
30
|
-
`19200` instead of the standard port. We can provide either the full URL or just
|
31
|
-
a different port number if Elasticsearch is running on `localhost`.
|
32
|
-
|
33
|
-
```ruby
|
34
|
-
client = Elastomer::Client.new :port => 19200
|
35
|
-
client.host #=> 'localhost'
|
36
|
-
client.port #=> 19200
|
37
|
-
client.url #=> 'http://localhost:19200'
|
38
|
-
|
39
|
-
client = Elastomer::Client.new :url => "http://localhost:19200"
|
40
|
-
```
|
41
|
-
|
42
29
|
Elasticsearch works best with persistent connections. We can use the
|
43
30
|
`Net::HTTP::Persistent` adapter with Faraday.
|
44
31
|
|
45
32
|
```ruby
|
46
|
-
client =
|
47
|
-
:port =>
|
33
|
+
client = ElastomerClient::Client.new \
|
34
|
+
:port => 9200,
|
48
35
|
:adapter => :net_http_persistent
|
49
36
|
```
|
50
37
|
|
@@ -58,8 +45,8 @@ The open timeout is configured once when the client is first created. The read
|
|
58
45
|
timeout can be set for each request.
|
59
46
|
|
60
47
|
```ruby
|
61
|
-
client =
|
62
|
-
:url => "http://localhost:
|
48
|
+
client = ElastomerClient::Client.new \
|
49
|
+
:url => "http://localhost:9200",
|
63
50
|
:adapter => :net_http_persistent,
|
64
51
|
:open_timeout => 1,
|
65
52
|
:read_timeout => 5
|
@@ -77,14 +64,14 @@ Elasticsearch provides an `X-Opaque-Id` request header. Any value set in this
|
|
77
64
|
request header will be returned in the corresponding response header. This
|
78
65
|
allows the client to correlate the response from Elasticsearch with the request
|
79
66
|
that was submitted. We have written an
|
80
|
-
[OpaqueId](https://github.com/github/elastomer-client/blob/
|
67
|
+
[OpaqueId](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/middleware/opaque_id.rb)
|
81
68
|
middleware that will abort any request if the `X-Opaque-Id` headers disagree
|
82
69
|
between the request and the response. You can use this feature by setting
|
83
70
|
the `:opaque_id` flag.
|
84
71
|
|
85
72
|
```ruby
|
86
|
-
client =
|
87
|
-
:url => "http://localhost:
|
73
|
+
client = ElastomerClient::Client.new \
|
74
|
+
:url => "http://localhost:9200",
|
88
75
|
:adapter => :net_http_persistent,
|
89
76
|
:opaque_id => true
|
90
77
|
```
|
@@ -95,7 +82,7 @@ If you are not using persistent connections, then you do not need to worry about
|
|
95
82
|
### HTTP Methods
|
96
83
|
|
97
84
|
The standard HTTP verbs - `head`, `get`, `put`, `post`, `delete` - are exposed
|
98
|
-
as methods on the `
|
85
|
+
as methods on the `ElastomerClient::Client` class. Each method accepts a path and a
|
99
86
|
Hash of parameters. Some parameters are applied as path expansions, some are
|
100
87
|
reserved, and the remainder are used as URL parameters. We'll look at the
|
101
88
|
reserved parameters first.
|
@@ -140,11 +127,11 @@ Elasticsearch has responded.
|
|
140
127
|
|
141
128
|
**:action** and **:context**
|
142
129
|
|
143
|
-
Each method in the
|
130
|
+
Each method in the ElastomerClient gem has its own `:action` value that is
|
144
131
|
used in conjunction with the [notifications](notifications.md) layer. The
|
145
132
|
`:action` parameter cannot be changed by the user. Instead you can provide a
|
146
133
|
`:context` value to each method call. This will be passed unchanged to the
|
147
|
-
notifications layer, and it is useful for tracking where an
|
134
|
+
notifications layer, and it is useful for tracking where an ElastomerClient
|
148
135
|
method is called from within your application.
|
149
136
|
|
150
137
|
#### URL Handling
|
@@ -157,7 +144,7 @@ With the [`Addressable::Template`](https://github.com/sporkmonger/addressable#ur
|
|
157
144
|
a typical search URL takes the form `{/index}{/type}/_search`. The `:index` and
|
158
145
|
`:type` values are taken from the parameters Hash and combined with the template
|
159
146
|
to generate the URL. The internal
|
160
|
-
[`expand_path`](https://github.com/github/elastomer-client/blob/
|
147
|
+
[`expand_path`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client.rb#L245)
|
161
148
|
method handles the URL generation.
|
162
149
|
|
163
150
|
Here are a few examples to better illustrate the concept.
|
@@ -212,17 +199,17 @@ client.expand_path("/{index}/{type}/_search", {
|
|
212
199
|
```
|
213
200
|
|
214
201
|
And that is the basic concept of the `expand_path` method. The URL template
|
215
|
-
pattern is used extensively in the
|
202
|
+
pattern is used extensively in the ElastomerClient code, so it is definitely
|
216
203
|
worth knowing about.
|
217
204
|
|
218
205
|
### Errors
|
219
206
|
|
220
207
|
Invariably things will go wrong where computers and networks are involved. The
|
221
|
-
|
208
|
+
ElastomerClient code makes no attempt to retry an operation in the face of an
|
222
209
|
error. However, it does classify errors into *fatal* and *retryable* exceptions.
|
223
210
|
|
224
211
|
Each class that inherits from
|
225
|
-
[`
|
212
|
+
[`ElastomerClient::Client::Error`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client/errors.rb)
|
226
213
|
has a `fatal?` method (and the inverse `retry?` method). If an exception is
|
227
214
|
fatal, then the request is fundamentally flawed and should not be retried.
|
228
215
|
Passing a malformed search query or trying to search an index that does not
|
data/docs/cluster.md
CHANGED
@@ -1,17 +1,17 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient Cluster Component
|
2
2
|
|
3
3
|
The cluster component deals with commands for managing cluster state and
|
4
4
|
monitoring cluster health. All the commands found under the
|
5
5
|
[cluster API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html)
|
6
6
|
section of the Elasticsearch documentation are implemented by the
|
7
|
-
[`cluster.rb`](https://github.com/github/elastomer-client/blob/
|
8
|
-
module and the [`nodes.rb`](https://github.com/github/elastomer-client/blob/
|
7
|
+
[`cluster.rb`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client/cluster.rb)
|
8
|
+
module and the [`nodes.rb`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client/nodes.rb)
|
9
9
|
module.
|
10
10
|
|
11
11
|
## Cluster
|
12
12
|
|
13
13
|
API endpoints dealing with cluster level information and settings are found in
|
14
|
-
the [`Cluster`](lib/
|
14
|
+
the [`Cluster`](lib/elastomer_client/client/cluster.rb) class. Each of these methods
|
15
15
|
corresponds to an API endpoint described in the Elasticsearch documentation
|
16
16
|
(linked to above). The params listed in the documentation can be passed to these
|
17
17
|
methods, so we do not take too much trouble to enumerate them all.
|
@@ -22,8 +22,8 @@ The cluster [health API](https://www.elastic.co/guide/en/elasticsearch/reference
|
|
22
22
|
returns a very simple cluster health status report.
|
23
23
|
|
24
24
|
```ruby
|
25
|
-
require '
|
26
|
-
client =
|
25
|
+
require 'elastomer_client/client'
|
26
|
+
client = ElastomerClient::Client.new :port => 9200
|
27
27
|
|
28
28
|
# the current health summary
|
29
29
|
client.cluster.health
|
@@ -125,8 +125,8 @@ individual (or multiple) nodes in the cluster. We expose these via the `nodes`
|
|
125
125
|
module in elastomer-client.
|
126
126
|
|
127
127
|
```ruby
|
128
|
-
require '
|
129
|
-
client =
|
128
|
+
require 'elastomer_client/client'
|
129
|
+
client = ElastomerClient::Client.new :port => 9200
|
130
130
|
|
131
131
|
# gather OS, JVM, and process information from the local node
|
132
132
|
client.nodes("_local").info(:info => %w[os jvm process])
|
data/docs/docs.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient Documents Component
|
2
2
|
|
3
3
|
The documents components handles all API calls related to
|
4
4
|
[indexing documents](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs.html)
|
@@ -11,8 +11,8 @@ that is the only difference between the two. In the example below, the resulting
|
|
11
11
|
documents components are equivalent.
|
12
12
|
|
13
13
|
```ruby
|
14
|
-
require '
|
15
|
-
client =
|
14
|
+
require 'elastomer_client/client'
|
15
|
+
client = ElastomerClient::Client.new :port => 9200
|
16
16
|
|
17
17
|
docs1 = client.index("blog").docs("post")
|
18
18
|
docs2 = client.docs("blog", "post")
|
@@ -94,7 +94,7 @@ locations enables you to reconcile documents between the two.
|
|
94
94
|
The `:_id` field is only one of several special fields that control document
|
95
95
|
indexing in Elasticsearch. The full list of supported fields are enumerated in
|
96
96
|
the `index`
|
97
|
-
[method documentation](https://github.com/github/elastomer-client/blob/
|
97
|
+
[method documentation](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client/docs.rb#L45-56).
|
98
98
|
|
99
99
|
As a parting note, you can also provide the index name and document type as part
|
100
100
|
of the document itself. These fields will be extracted from the document before
|
@@ -163,7 +163,7 @@ client.docs.search \
|
|
163
163
|
The `search` method returns the query response from Elasticsearch as a ruby
|
164
164
|
Hash. All the keys are represented as Strings. The [hashie](https://github.com/intridea/hashie)
|
165
165
|
project has some useful transforms and wrappers for working with these result
|
166
|
-
sets, but that is left to the user to implement if they so desire.
|
166
|
+
sets, but that is left to the user to implement if they so desire. ElastomerClient
|
167
167
|
client returns only ruby Hashes.
|
168
168
|
|
169
169
|
Searches can be executed against multiple indices and multiple types. Again,
|
data/docs/index.md
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient Index Component
|
2
2
|
|
3
3
|
The index component provides access to the
|
4
4
|
[indices API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html)
|
5
5
|
used for index management, settings, mappings, and aliases. Index
|
6
|
-
[
|
6
|
+
[templates](templates.md) are handled via their own
|
7
7
|
components. Methods for adding documents to the index and searching those
|
8
8
|
documents are found in the [documents](documents.md) component. The index
|
9
9
|
component deals solely with management of the indices themselves.
|
@@ -14,8 +14,8 @@ However, you can omit the index name and pass it along with each API method
|
|
14
14
|
called.
|
15
15
|
|
16
16
|
```ruby
|
17
|
-
require '
|
18
|
-
client =
|
17
|
+
require 'elastomer_client/client'
|
18
|
+
client = ElastomerClient::Client.new :port => 9200
|
19
19
|
|
20
20
|
# you can provide an index name
|
21
21
|
index = client.index "blog"
|
data/docs/multi_search.md
CHANGED
data/docs/notifications.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# Notifications Support
|
2
2
|
|
3
|
-
Requiring `
|
4
|
-
|
3
|
+
Requiring `elastomer_client/notifications` enables support for broadcasting
|
4
|
+
ElastomerClient events through ActiveSupport::Notifications.
|
5
5
|
|
6
6
|
The event namespace is `request.client.elastomer`.
|
7
7
|
|
@@ -14,7 +14,7 @@ The event namespace is `request.client.elastomer`.
|
|
14
14
|
:action => "docs.search",
|
15
15
|
:context => nil,
|
16
16
|
:body => "{\"query\":{\"match_all\":{}}}",
|
17
|
-
:url => #<URI::HTTP:0x007fb6f3e98b60 URL:http://localhost:
|
17
|
+
:url => #<URI::HTTP:0x007fb6f3e98b60 URL:http://localhost:9200/index-test/_search?size=0>,
|
18
18
|
:method => :get,
|
19
19
|
:status => 200
|
20
20
|
}
|
data/docs/scan_scroll.md
CHANGED
data/docs/snapshots.md
CHANGED
data/docs/templates.md
CHANGED
data/elastomer-client.gemspec
CHANGED
@@ -1,15 +1,17 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
2
4
|
lib = File.expand_path("../lib", __FILE__)
|
3
5
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require "
|
6
|
+
require "elastomer_client/version"
|
5
7
|
|
6
8
|
Gem::Specification.new do |spec|
|
7
9
|
spec.name = "elastomer-client"
|
8
|
-
spec.version =
|
10
|
+
spec.version = ElastomerClient::VERSION
|
9
11
|
spec.authors = ["Tim Pease", "Grant Rodgers"]
|
10
12
|
spec.email = ["tim.pease@github.com", "grant.rodgers@github.com"]
|
11
13
|
spec.summary = %q{A library for interacting with Elasticsearch}
|
12
|
-
spec.description = %q{
|
14
|
+
spec.description = %q{ElastomerClient is a low level API client for the
|
13
15
|
Elasticsearch HTTP interface.}
|
14
16
|
spec.homepage = "https://github.com/github/elastomer-client"
|
15
17
|
spec.license = "MIT"
|
@@ -20,19 +22,8 @@ Gem::Specification.new do |spec|
|
|
20
22
|
spec.require_paths = ["lib"]
|
21
23
|
|
22
24
|
spec.add_dependency "addressable", "~> 2.5"
|
23
|
-
spec.add_dependency "faraday", "
|
24
|
-
spec.add_dependency "faraday_middleware", "
|
25
|
+
spec.add_dependency "faraday", ">= 0.17"
|
26
|
+
spec.add_dependency "faraday_middleware", ">= 0.14"
|
25
27
|
spec.add_dependency "multi_json", "~> 1.12"
|
26
28
|
spec.add_dependency "semantic", "~> 1.6"
|
27
|
-
|
28
|
-
spec.add_development_dependency "bundler", "~> 2.0"
|
29
|
-
spec.add_development_dependency "activesupport", ">= 3.0"
|
30
|
-
spec.add_development_dependency "minitest", "~> 5.10"
|
31
|
-
spec.add_development_dependency "minitest-fail-fast", "~> 0.1.0"
|
32
|
-
spec.add_development_dependency "minitest-focus", "~> 1.1", ">= 1.1.2"
|
33
|
-
spec.add_development_dependency "webmock", "~> 3.5"
|
34
|
-
spec.add_development_dependency "awesome_print", "~> 1.8"
|
35
|
-
spec.add_development_dependency "pry-byebug", "~> 3.4"
|
36
|
-
spec.add_development_dependency "spy", "~> 1.0"
|
37
|
-
spec.add_development_dependency "rake"
|
38
29
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ElastomerClient
|
2
4
|
class Client
|
3
5
|
|
4
6
|
# The `bulk` method can be used in two ways. Without a block the method
|
@@ -28,7 +30,7 @@ module Elastomer
|
|
28
30
|
# end
|
29
31
|
#
|
30
32
|
# Returns the response body as a Hash
|
31
|
-
def bulk(
|
33
|
+
def bulk(body = nil, params = nil)
|
32
34
|
if block_given?
|
33
35
|
params, body = (body || {}), nil
|
34
36
|
yield bulk_obj = Bulk.new(self, params)
|
@@ -37,8 +39,10 @@ module Elastomer
|
|
37
39
|
else
|
38
40
|
raise "bulk request body cannot be nil" if body.nil?
|
39
41
|
params ||= {}
|
42
|
+
updated_params = params.merge(body:, action: "bulk", rest_api: "bulk")
|
43
|
+
updated_params.delete(:type) if version_support.es_version_8_plus?
|
40
44
|
|
41
|
-
response = self.post "{/index}{/type}/_bulk",
|
45
|
+
response = self.post "{/index}{/type}/_bulk", updated_params
|
42
46
|
response.body
|
43
47
|
end
|
44
48
|
end
|
@@ -103,7 +107,7 @@ module Elastomer
|
|
103
107
|
# # "failure" => 0
|
104
108
|
# # }
|
105
109
|
#
|
106
|
-
# # sample response item:
|
110
|
+
# # sample response item for ES5:
|
107
111
|
# # {
|
108
112
|
# # "delete": {
|
109
113
|
# # "_index": "foo",
|
@@ -115,6 +119,17 @@ module Elastomer
|
|
115
119
|
# # }
|
116
120
|
# # }
|
117
121
|
#
|
122
|
+
# # sample response item for ES8:
|
123
|
+
# # {
|
124
|
+
# # "delete": {
|
125
|
+
# # "_index": "foo",
|
126
|
+
# # "_id": "42",
|
127
|
+
# # "_version": 3,
|
128
|
+
# # "status": 200,
|
129
|
+
# # "result": "deleted"
|
130
|
+
# # }
|
131
|
+
# # }
|
132
|
+
#
|
118
133
|
# Returns a Hash of stats about items from the responses.
|
119
134
|
def bulk_stream_items(ops, params = {})
|
120
135
|
stats = {
|
@@ -158,17 +173,17 @@ module Elastomer
|
|
158
173
|
# immediately.
|
159
174
|
#
|
160
175
|
class Bulk
|
161
|
-
DEFAULT_REQUEST_SIZE =
|
176
|
+
DEFAULT_REQUEST_SIZE = 2**20 * 10 # 10 MB
|
162
177
|
|
163
178
|
# Create a new bulk client for handling some of the details of
|
164
179
|
# accumulating documents to index and then formatting them properly for
|
165
180
|
# the bulk API command.
|
166
181
|
#
|
167
|
-
# client -
|
182
|
+
# client - ElastomerClient::Client used for HTTP requests to the server
|
168
183
|
# params - Parameters Hash to pass to the Client#bulk method
|
169
184
|
# :request_size - the maximum request size in bytes
|
170
185
|
# :action_count - the maximum number of actions
|
171
|
-
def initialize(
|
186
|
+
def initialize(client, params = {})
|
172
187
|
@client = client
|
173
188
|
@params = params
|
174
189
|
|
@@ -190,7 +205,7 @@ module Elastomer
|
|
190
205
|
# buffered until the request size is met or exceeded. When this happens a
|
191
206
|
# bulk request is issued, queued actions are cleared, and the response
|
192
207
|
# from Elasticsearch is returned.
|
193
|
-
def request_size=(
|
208
|
+
def request_size=(value)
|
194
209
|
if value.nil?
|
195
210
|
@request_size = nil
|
196
211
|
else
|
@@ -232,9 +247,9 @@ module Elastomer
|
|
232
247
|
# index("foo" => "bar", "_id" => 1, "_type" => "foo")
|
233
248
|
#
|
234
249
|
# Returns the response from the bulk call if one was made or nil.
|
235
|
-
def index(
|
250
|
+
def index(document, params = {})
|
236
251
|
params = prepare_params(document, params)
|
237
|
-
add_to_actions({:
|
252
|
+
add_to_actions({index: params}, document)
|
238
253
|
end
|
239
254
|
|
240
255
|
# Add a create action to the list of bulk actions to be performed when
|
@@ -251,9 +266,9 @@ module Elastomer
|
|
251
266
|
# create("foo" => "bar", "_id" => 1)
|
252
267
|
#
|
253
268
|
# Returns the response from the bulk call if one was made or nil.
|
254
|
-
def create(
|
269
|
+
def create(document, params)
|
255
270
|
params = prepare_params(document, params)
|
256
|
-
add_to_actions({:
|
271
|
+
add_to_actions({create: params}, document)
|
257
272
|
end
|
258
273
|
|
259
274
|
# Add an update action to the list of bulk actions to be performed when
|
@@ -265,14 +280,14 @@ module Elastomer
|
|
265
280
|
# params - Parameters for the update action (as a Hash) (optional)
|
266
281
|
#
|
267
282
|
# Examples
|
268
|
-
# update({"foo" => "bar"}, {:_id => 1}
|
269
|
-
# update({"foo" => "bar"}, {:id => 1}
|
270
|
-
# update("foo" => "bar", "_id" => 1)
|
283
|
+
# update({"doc" => {"foo" => "bar"}}, {:_id => 1})
|
284
|
+
# update({"doc" => {"foo" => "bar"}}, {:id => 1})
|
285
|
+
# update({"doc" => {"foo" => "bar"}}, "_id" => 1)
|
271
286
|
#
|
272
287
|
# Returns the response from the bulk call if one was made or nil.
|
273
|
-
def update(
|
288
|
+
def update(document, params)
|
274
289
|
params = prepare_params(document, params)
|
275
|
-
add_to_actions({:
|
290
|
+
add_to_actions({update: params}, document)
|
276
291
|
end
|
277
292
|
|
278
293
|
# Add a delete action to the list of bulk actions to be performed when
|
@@ -284,9 +299,9 @@ module Elastomer
|
|
284
299
|
# delete(:_id => 1, :_type => 'foo')
|
285
300
|
#
|
286
301
|
# Returns the response from the bulk call if one was made or nil.
|
287
|
-
def delete(
|
302
|
+
def delete(params)
|
288
303
|
params = prepare_params(nil, params)
|
289
|
-
add_to_actions({:
|
304
|
+
add_to_actions({delete: params})
|
290
305
|
end
|
291
306
|
|
292
307
|
# Immediately execute a bulk API call with the currently accumulated
|
@@ -307,37 +322,23 @@ module Elastomer
|
|
307
322
|
@actions.clear
|
308
323
|
end
|
309
324
|
|
310
|
-
SPECIAL_KEYS = %w[id type index version version_type routing parent
|
311
|
-
|
325
|
+
SPECIAL_KEYS = %w[id type index version version_type routing parent consistency refresh retry_on_conflict]
|
326
|
+
UNPREFIXED_SPECIAL_KEYS = %w[parent retry_on_conflict routing version version_type]
|
312
327
|
|
313
328
|
# Internal: convert special key parameters to their wire representation
|
314
329
|
# and apply any override document parameters.
|
315
330
|
def prepare_params(document, params)
|
316
331
|
params = convert_special_keys(params)
|
317
|
-
|
318
|
-
params = from_document(document).merge(params)
|
319
|
-
end
|
332
|
+
|
320
333
|
params.delete(:_id) if params[:_id].nil? || params[:_id].to_s.empty?
|
321
|
-
params
|
322
|
-
end
|
334
|
+
params.delete("_id") if params["_id"].nil? || params["_id"].to_s.empty?
|
323
335
|
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
#
|
328
|
-
# document - The document Hash
|
329
|
-
#
|
330
|
-
# Returns extracted key/value pairs as a Hash.
|
331
|
-
def from_document( document )
|
332
|
-
opts = {}
|
333
|
-
|
334
|
-
SPECIAL_KEYS_HASH.values.each do |field|
|
335
|
-
key = field.to_sym
|
336
|
-
opts[key] = document.delete field if document.key? field
|
337
|
-
opts[key] = document.delete key if document.key? key
|
336
|
+
if client.version_support.es_version_8_plus?
|
337
|
+
params.delete(:_type)
|
338
|
+
params.delete("_type")
|
338
339
|
end
|
339
340
|
|
340
|
-
|
341
|
+
params
|
341
342
|
end
|
342
343
|
|
343
344
|
# Internal: Convert incoming Ruby symbol keys to their special underscore
|
@@ -353,9 +354,30 @@ module Elastomer
|
|
353
354
|
def convert_special_keys(params)
|
354
355
|
new_params = params.dup
|
355
356
|
|
356
|
-
|
357
|
-
|
358
|
-
|
357
|
+
SPECIAL_KEYS.each do |key|
|
358
|
+
omit_prefix = (
|
359
|
+
client.version_support.es_version_8_plus? &&
|
360
|
+
UNPREFIXED_SPECIAL_KEYS.include?(key)
|
361
|
+
)
|
362
|
+
|
363
|
+
prefixed_key = "_" + key
|
364
|
+
converted_key = (omit_prefix ? "" : "_") + key
|
365
|
+
|
366
|
+
if new_params.key?(prefixed_key)
|
367
|
+
new_params[converted_key] = new_params.delete(prefixed_key)
|
368
|
+
end
|
369
|
+
|
370
|
+
if new_params.key?(prefixed_key.to_sym)
|
371
|
+
new_params[converted_key.to_sym] = new_params.delete(prefixed_key.to_sym)
|
372
|
+
end
|
373
|
+
|
374
|
+
if new_params.key?(key)
|
375
|
+
new_params[converted_key] = new_params.delete(key)
|
376
|
+
end
|
377
|
+
|
378
|
+
if new_params.key?(key.to_sym)
|
379
|
+
new_params[converted_key.to_sym] = new_params.delete(key.to_sym)
|
380
|
+
end
|
359
381
|
end
|
360
382
|
|
361
383
|
new_params
|
@@ -375,7 +397,7 @@ module Elastomer
|
|
375
397
|
# Returns the response from the bulk call if one was made or nil.
|
376
398
|
# Raises RequestSizeError if the given action is larger than the
|
377
399
|
# configured requst size or the client.max_request_size
|
378
|
-
def add_to_actions(
|
400
|
+
def add_to_actions(action, document = nil)
|
379
401
|
action = MultiJson.dump(action)
|
380
402
|
size = action.bytesize
|
381
403
|
|
@@ -389,6 +411,7 @@ module Elastomer
|
|
389
411
|
response = nil
|
390
412
|
begin
|
391
413
|
response = call if ready_to_send?(size)
|
414
|
+
# rubocop:disable Lint/UselessRescue
|
392
415
|
rescue StandardError => err
|
393
416
|
raise err
|
394
417
|
ensure
|
@@ -407,7 +430,7 @@ module Elastomer
|
|
407
430
|
# dispatch the bulk request.
|
408
431
|
#
|
409
432
|
# Returns `true` of `false`
|
410
|
-
def ready_to_send?(
|
433
|
+
def ready_to_send?(size)
|
411
434
|
total_request_size = @current_request_size + size
|
412
435
|
total_action_count = @current_action_count + 1
|
413
436
|
|
@@ -417,7 +440,7 @@ module Elastomer
|
|
417
440
|
|
418
441
|
# Internal: Raises a RequestSizeError if the given size is larger than
|
419
442
|
# the configured client.max_request_size
|
420
|
-
def check_action_size!(
|
443
|
+
def check_action_size!(size)
|
421
444
|
return unless size > client.max_request_size
|
422
445
|
raise RequestSizeError, "Bulk action of size `#{size}` exceeds the maximum requst size: #{client.max_request_size}"
|
423
446
|
end
|