elastomer-client 3.2.3 → 6.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.devcontainer/devcontainer.json +46 -0
- data/.devcontainer/postCreateCommand.sh +4 -0
- data/.github/dependabot.yaml +11 -0
- data/.github/workflows/main.yml +45 -0
- data/.github/workflows/rubocop.yml +15 -0
- data/.gitignore +1 -1
- data/.rubocop.yml +13 -65
- data/.ruby-version +1 -0
- data/CHANGELOG.md +76 -0
- data/Gemfile +18 -1
- data/README.md +110 -51
- data/Rakefile +3 -1
- data/docker/compose.yaml +71 -0
- data/docker/elasticsearch8plus.yml +13 -0
- data/docs/README.md +4 -5
- data/docs/bulk_indexing.md +1 -1
- data/docs/client.md +20 -33
- data/docs/cluster.md +8 -8
- data/docs/docs.md +5 -5
- data/docs/index.md +4 -4
- data/docs/multi_search.md +1 -1
- data/docs/notifications.md +3 -3
- data/docs/scan_scroll.md +1 -1
- data/docs/snapshots.md +1 -1
- data/docs/templates.md +1 -1
- data/elastomer-client.gemspec +7 -16
- data/lib/{elastomer → elastomer_client}/client/bulk.rb +70 -47
- data/lib/{elastomer → elastomer_client}/client/cluster.rb +18 -16
- data/lib/{elastomer → elastomer_client}/client/delete_by_query.rb +6 -4
- data/lib/{elastomer → elastomer_client}/client/docs.rb +82 -72
- data/lib/{elastomer → elastomer_client}/client/errors.rb +7 -17
- data/lib/{elastomer → elastomer_client}/client/index.rb +55 -79
- data/lib/{elastomer → elastomer_client}/client/multi_percolate.rb +7 -5
- data/lib/{elastomer → elastomer_client}/client/multi_search.rb +5 -3
- data/lib/{elastomer → elastomer_client}/client/native_delete_by_query.rb +6 -6
- data/lib/{elastomer → elastomer_client}/client/nodes.rb +11 -10
- data/lib/{elastomer → elastomer_client}/client/percolator.rb +9 -10
- data/lib/elastomer_client/client/reindex.rb +34 -0
- data/lib/{elastomer → elastomer_client}/client/repository.rb +7 -5
- data/lib/{elastomer → elastomer_client}/client/rest_api_spec/api_spec.rb +7 -6
- data/lib/{elastomer → elastomer_client}/client/rest_api_spec/api_spec_v5_6.rb +1 -1
- data/lib/elastomer_client/client/rest_api_spec/api_spec_v8_13.rb +7567 -0
- data/lib/elastomer_client/client/rest_api_spec/api_spec_v8_7.rb +6553 -0
- data/lib/{elastomer → elastomer_client}/client/rest_api_spec/rest_api.rb +5 -3
- data/lib/{elastomer → elastomer_client}/client/rest_api_spec.rb +3 -2
- data/lib/{elastomer → elastomer_client}/client/scroller.rb +17 -16
- data/lib/{elastomer → elastomer_client}/client/snapshot.rb +10 -8
- data/lib/{elastomer → elastomer_client}/client/tasks.rb +9 -13
- data/lib/{elastomer → elastomer_client}/client/template.rb +10 -9
- data/lib/elastomer_client/client/update_by_query.rb +50 -0
- data/lib/{elastomer → elastomer_client}/client.rb +51 -62
- data/lib/{elastomer → elastomer_client}/core_ext/time.rb +2 -0
- data/lib/{elastomer → elastomer_client}/middleware/compress.rb +2 -2
- data/lib/{elastomer → elastomer_client}/middleware/encode_json.rb +4 -2
- data/lib/{elastomer → elastomer_client}/middleware/limit_size.rb +5 -3
- data/lib/{elastomer → elastomer_client}/middleware/opaque_id.rb +10 -7
- data/lib/{elastomer → elastomer_client}/middleware/parse_json.rb +5 -3
- data/lib/{elastomer → elastomer_client}/notifications.rb +17 -15
- data/lib/elastomer_client/version.rb +9 -0
- data/lib/elastomer_client/version_support.rb +24 -0
- data/script/bootstrap +4 -2
- data/script/console +3 -1
- data/script/generate-rest-api-spec +77 -22
- data/test/assertions.rb +32 -39
- data/test/client/bulk_test.rb +166 -141
- data/test/client/cluster_test.rb +35 -13
- data/test/client/docs_test.rb +387 -274
- data/test/client/errors_test.rb +38 -40
- data/test/client/index_test.rb +243 -202
- data/test/client/multi_percolate_test.rb +46 -41
- data/test/client/multi_search_test.rb +122 -67
- data/test/client/native_delete_by_query_test.rb +96 -88
- data/test/client/nodes_test.rb +21 -10
- data/test/client/percolator_test.rb +19 -14
- data/test/client/reindex_test.rb +100 -0
- data/test/client/repository_test.rb +31 -19
- data/test/client/rest_api_spec/api_spec_test.rb +13 -11
- data/test/client/rest_api_spec/rest_api_test.rb +9 -7
- data/test/client/scroller_test.rb +44 -70
- data/test/client/snapshot_test.rb +38 -21
- data/test/client/stubbed_client_test.rb +7 -4
- data/test/client/tasks_test.rb +12 -17
- data/test/client/template_test.rb +34 -13
- data/test/client/update_by_query_test.rb +137 -0
- data/test/client_test.rb +158 -92
- data/test/core_ext/time_test.rb +14 -12
- data/test/middleware/encode_json_test.rb +18 -7
- data/test/middleware/opaque_id_test.rb +18 -14
- data/test/middleware/parse_json_test.rb +17 -9
- data/test/mock_response.rb +30 -0
- data/test/notifications_test.rb +15 -8
- data/test/test_helper.rb +40 -97
- data/test/version_support_test.rb +13 -78
- metadata +60 -208
- data/.overcommit.yml +0 -5
- data/.travis.yml +0 -34
- data/docker/docker-compose.cibuild.yml +0 -8
- data/docker/docker-compose.es24.yml +0 -34
- data/docker/docker-compose.es56.yml +0 -37
- data/docs/warmers.md +0 -3
- data/lib/elastomer/client/app_delete_by_query.rb +0 -144
- data/lib/elastomer/client/rest_api_spec/api_spec_v2_3.rb +0 -2232
- data/lib/elastomer/client/rest_api_spec/api_spec_v2_4.rb +0 -2250
- data/lib/elastomer/client/warmer.rb +0 -98
- data/lib/elastomer/version.rb +0 -7
- data/lib/elastomer/version_support.rb +0 -182
- data/script/cibuild +0 -103
- data/script/cibuild-elastomer-client +0 -1
- data/script/cibuild-elastomer-client-es24 +0 -8
- data/script/cibuild-elastomer-client-es56 +0 -8
- data/test/client/app_delete_by_query_test.rb +0 -192
- data/test/client/es_5_x_warmer_test.rb +0 -13
- data/test/client/warmer_test.rb +0 -60
data/docs/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient in Depth
|
2
2
|
|
3
|
-
We first started building the
|
3
|
+
We first started building the ElastomerClient gem when an
|
4
4
|
[official client](https://github.com/elasticsearch/elasticsearch-ruby)
|
5
5
|
was not yet available from Elasticsearch. We were looking for a client that
|
6
6
|
provided a one-to-one mapping of the Elasticsearch APIs and avoided higher level
|
@@ -16,10 +16,10 @@ To that end we have tried to be as faithful as possible to the Elasticsearch API
|
|
16
16
|
with our implementation. There are a few places where it made sense to wrap the
|
17
17
|
Elasticsearch API inside Ruby idioms. One notable location is the
|
18
18
|
[scan-scroll](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html)
|
19
|
-
search type; the
|
19
|
+
search type; the ElastomerClient provides a Ruby iterator to work with these
|
20
20
|
types of queries.
|
21
21
|
|
22
|
-
Below are links to documents describing the various components of the
|
22
|
+
Below are links to documents describing the various components of the ElastomerClient
|
23
23
|
Client library. Start with the core components - specifically the **Client**
|
24
24
|
document. All the other components are built atop the client.
|
25
25
|
|
@@ -30,7 +30,6 @@ document. All the other components are built atop the client.
|
|
30
30
|
* [Documents](docs.md)
|
31
31
|
* [Cluster](cluster.md)
|
32
32
|
* [Templates](templates.md)
|
33
|
-
* [Warmers](warmers.md)
|
34
33
|
|
35
34
|
**Bulk Components**
|
36
35
|
|
data/docs/bulk_indexing.md
CHANGED
data/docs/client.md
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient Client Component
|
2
2
|
|
3
|
-
All methods in the
|
4
|
-
Elasticsearch. The [`
|
3
|
+
All methods in the ElastomerClient gem eventually make an HTTP request to
|
4
|
+
Elasticsearch. The [`ElastomerClient::Client`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client.rb)
|
5
5
|
class is responsible for connecting to an Elasticsearch instance, making HTTP
|
6
6
|
requests, processing the response, and handling errors. Let's look at the
|
7
|
-
details of how `
|
7
|
+
details of how `ElastomerClient::Client` handles HTTP communication.
|
8
8
|
|
9
9
|
### Connecting
|
10
10
|
|
@@ -15,36 +15,23 @@ the concept of *middlewares* that operate on the HTTP request and response. We
|
|
15
15
|
use Faraday middleware to encode and decode JSON messages exchanged with
|
16
16
|
Elasticsearch.
|
17
17
|
|
18
|
-
Without any options the `
|
18
|
+
Without any options the `ElastomerClient::Client` will connect to the default
|
19
19
|
Elasticsearch URL `http://localhost:9200`. The `Net:HTTP` client from the Ruby
|
20
20
|
standard library will be used.
|
21
21
|
|
22
22
|
```ruby
|
23
|
-
client =
|
23
|
+
client = ElastomerClient::Client.new
|
24
24
|
client.host #=> 'localhost'
|
25
25
|
client.port #=> 9200
|
26
26
|
client.url #=> 'http://localhost:9200'
|
27
27
|
```
|
28
28
|
|
29
|
-
[Boxen](https://boxen.github.com) configures Elasticsearch to listen on port
|
30
|
-
`19200` instead of the standard port. We can provide either the full URL or just
|
31
|
-
a different port number if Elasticsearch is running on `localhost`.
|
32
|
-
|
33
|
-
```ruby
|
34
|
-
client = Elastomer::Client.new :port => 19200
|
35
|
-
client.host #=> 'localhost'
|
36
|
-
client.port #=> 19200
|
37
|
-
client.url #=> 'http://localhost:19200'
|
38
|
-
|
39
|
-
client = Elastomer::Client.new :url => "http://localhost:19200"
|
40
|
-
```
|
41
|
-
|
42
29
|
Elasticsearch works best with persistent connections. We can use the
|
43
30
|
`Net::HTTP::Persistent` adapter with Faraday.
|
44
31
|
|
45
32
|
```ruby
|
46
|
-
client =
|
47
|
-
:port =>
|
33
|
+
client = ElastomerClient::Client.new \
|
34
|
+
:port => 9200,
|
48
35
|
:adapter => :net_http_persistent
|
49
36
|
```
|
50
37
|
|
@@ -58,8 +45,8 @@ The open timeout is configured once when the client is first created. The read
|
|
58
45
|
timeout can be set for each request.
|
59
46
|
|
60
47
|
```ruby
|
61
|
-
client =
|
62
|
-
:url => "http://localhost:
|
48
|
+
client = ElastomerClient::Client.new \
|
49
|
+
:url => "http://localhost:9200",
|
63
50
|
:adapter => :net_http_persistent,
|
64
51
|
:open_timeout => 1,
|
65
52
|
:read_timeout => 5
|
@@ -77,14 +64,14 @@ Elasticsearch provides an `X-Opaque-Id` request header. Any value set in this
|
|
77
64
|
request header will be returned in the corresponding response header. This
|
78
65
|
allows the client to correlate the response from Elasticsearch with the request
|
79
66
|
that was submitted. We have written an
|
80
|
-
[OpaqueId](https://github.com/github/elastomer-client/blob/
|
67
|
+
[OpaqueId](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/middleware/opaque_id.rb)
|
81
68
|
middleware that will abort any request if the `X-Opaque-Id` headers disagree
|
82
69
|
between the request and the response. You can use this feature by setting
|
83
70
|
the `:opaque_id` flag.
|
84
71
|
|
85
72
|
```ruby
|
86
|
-
client =
|
87
|
-
:url => "http://localhost:
|
73
|
+
client = ElastomerClient::Client.new \
|
74
|
+
:url => "http://localhost:9200",
|
88
75
|
:adapter => :net_http_persistent,
|
89
76
|
:opaque_id => true
|
90
77
|
```
|
@@ -95,7 +82,7 @@ If you are not using persistent connections, then you do not need to worry about
|
|
95
82
|
### HTTP Methods
|
96
83
|
|
97
84
|
The standard HTTP verbs - `head`, `get`, `put`, `post`, `delete` - are exposed
|
98
|
-
as methods on the `
|
85
|
+
as methods on the `ElastomerClient::Client` class. Each method accepts a path and a
|
99
86
|
Hash of parameters. Some parameters are applied as path expansions, some are
|
100
87
|
reserved, and the remainder are used as URL parameters. We'll look at the
|
101
88
|
reserved parameters first.
|
@@ -140,11 +127,11 @@ Elasticsearch has responded.
|
|
140
127
|
|
141
128
|
**:action** and **:context**
|
142
129
|
|
143
|
-
Each method in the
|
130
|
+
Each method in the ElastomerClient gem has its own `:action` value that is
|
144
131
|
used in conjunction with the [notifications](notifications.md) layer. The
|
145
132
|
`:action` parameter cannot be changed by the user. Instead you can provide a
|
146
133
|
`:context` value to each method call. This will be passed unchanged to the
|
147
|
-
notifications layer, and it is useful for tracking where an
|
134
|
+
notifications layer, and it is useful for tracking where an ElastomerClient
|
148
135
|
method is called from within your application.
|
149
136
|
|
150
137
|
#### URL Handling
|
@@ -157,7 +144,7 @@ With the [`Addressable::Template`](https://github.com/sporkmonger/addressable#ur
|
|
157
144
|
a typical search URL takes the form `{/index}{/type}/_search`. The `:index` and
|
158
145
|
`:type` values are taken from the parameters Hash and combined with the template
|
159
146
|
to generate the URL. The internal
|
160
|
-
[`expand_path`](https://github.com/github/elastomer-client/blob/
|
147
|
+
[`expand_path`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client.rb#L245)
|
161
148
|
method handles the URL generation.
|
162
149
|
|
163
150
|
Here are a few examples to better illustrate the concept.
|
@@ -212,17 +199,17 @@ client.expand_path("/{index}/{type}/_search", {
|
|
212
199
|
```
|
213
200
|
|
214
201
|
And that is the basic concept of the `expand_path` method. The URL template
|
215
|
-
pattern is used extensively in the
|
202
|
+
pattern is used extensively in the ElastomerClient code, so it is definitely
|
216
203
|
worth knowing about.
|
217
204
|
|
218
205
|
### Errors
|
219
206
|
|
220
207
|
Invariably things will go wrong where computers and networks are involved. The
|
221
|
-
|
208
|
+
ElastomerClient code makes no attempt to retry an operation in the face of an
|
222
209
|
error. However, it does classify errors into *fatal* and *retryable* exceptions.
|
223
210
|
|
224
211
|
Each class that inherits from
|
225
|
-
[`
|
212
|
+
[`ElastomerClient::Client::Error`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client/errors.rb)
|
226
213
|
has a `fatal?` method (and the inverse `retry?` method). If an exception is
|
227
214
|
fatal, then the request is fundamentally flawed and should not be retried.
|
228
215
|
Passing a malformed search query or trying to search an index that does not
|
data/docs/cluster.md
CHANGED
@@ -1,17 +1,17 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient Cluster Component
|
2
2
|
|
3
3
|
The cluster component deals with commands for managing cluster state and
|
4
4
|
monitoring cluster health. All the commands found under the
|
5
5
|
[cluster API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html)
|
6
6
|
section of the Elasticsearch documentation are implemented by the
|
7
|
-
[`cluster.rb`](https://github.com/github/elastomer-client/blob/
|
8
|
-
module and the [`nodes.rb`](https://github.com/github/elastomer-client/blob/
|
7
|
+
[`cluster.rb`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client/cluster.rb)
|
8
|
+
module and the [`nodes.rb`](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client/nodes.rb)
|
9
9
|
module.
|
10
10
|
|
11
11
|
## Cluster
|
12
12
|
|
13
13
|
API endpoints dealing with cluster level information and settings are found in
|
14
|
-
the [`Cluster`](lib/
|
14
|
+
the [`Cluster`](lib/elastomer_client/client/cluster.rb) class. Each of these methods
|
15
15
|
corresponds to an API endpoint described in the Elasticsearch documentation
|
16
16
|
(linked to above). The params listed in the documentation can be passed to these
|
17
17
|
methods, so we do not take too much trouble to enumerate them all.
|
@@ -22,8 +22,8 @@ The cluster [health API](https://www.elastic.co/guide/en/elasticsearch/reference
|
|
22
22
|
returns a very simple cluster health status report.
|
23
23
|
|
24
24
|
```ruby
|
25
|
-
require '
|
26
|
-
client =
|
25
|
+
require 'elastomer_client/client'
|
26
|
+
client = ElastomerClient::Client.new :port => 9200
|
27
27
|
|
28
28
|
# the current health summary
|
29
29
|
client.cluster.health
|
@@ -125,8 +125,8 @@ individual (or multiple) nodes in the cluster. We expose these via the `nodes`
|
|
125
125
|
module in elastomer-client.
|
126
126
|
|
127
127
|
```ruby
|
128
|
-
require '
|
129
|
-
client =
|
128
|
+
require 'elastomer_client/client'
|
129
|
+
client = ElastomerClient::Client.new :port => 9200
|
130
130
|
|
131
131
|
# gather OS, JVM, and process information from the local node
|
132
132
|
client.nodes("_local").info(:info => %w[os jvm process])
|
data/docs/docs.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient Documents Component
|
2
2
|
|
3
3
|
The documents components handles all API calls related to
|
4
4
|
[indexing documents](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs.html)
|
@@ -11,8 +11,8 @@ that is the only difference between the two. In the example below, the resulting
|
|
11
11
|
documents components are equivalent.
|
12
12
|
|
13
13
|
```ruby
|
14
|
-
require '
|
15
|
-
client =
|
14
|
+
require 'elastomer_client/client'
|
15
|
+
client = ElastomerClient::Client.new :port => 9200
|
16
16
|
|
17
17
|
docs1 = client.index("blog").docs("post")
|
18
18
|
docs2 = client.docs("blog", "post")
|
@@ -94,7 +94,7 @@ locations enables you to reconcile documents between the two.
|
|
94
94
|
The `:_id` field is only one of several special fields that control document
|
95
95
|
indexing in Elasticsearch. The full list of supported fields are enumerated in
|
96
96
|
the `index`
|
97
|
-
[method documentation](https://github.com/github/elastomer-client/blob/
|
97
|
+
[method documentation](https://github.com/github/elastomer-client/blob/main/lib/elastomer_client/client/docs.rb#L45-56).
|
98
98
|
|
99
99
|
As a parting note, you can also provide the index name and document type as part
|
100
100
|
of the document itself. These fields will be extracted from the document before
|
@@ -163,7 +163,7 @@ client.docs.search \
|
|
163
163
|
The `search` method returns the query response from Elasticsearch as a ruby
|
164
164
|
Hash. All the keys are represented as Strings. The [hashie](https://github.com/intridea/hashie)
|
165
165
|
project has some useful transforms and wrappers for working with these result
|
166
|
-
sets, but that is left to the user to implement if they so desire.
|
166
|
+
sets, but that is left to the user to implement if they so desire. ElastomerClient
|
167
167
|
client returns only ruby Hashes.
|
168
168
|
|
169
169
|
Searches can be executed against multiple indices and multiple types. Again,
|
data/docs/index.md
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
#
|
1
|
+
# ElastomerClient Index Component
|
2
2
|
|
3
3
|
The index component provides access to the
|
4
4
|
[indices API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html)
|
5
5
|
used for index management, settings, mappings, and aliases. Index
|
6
|
-
[
|
6
|
+
[templates](templates.md) are handled via their own
|
7
7
|
components. Methods for adding documents to the index and searching those
|
8
8
|
documents are found in the [documents](documents.md) component. The index
|
9
9
|
component deals solely with management of the indices themselves.
|
@@ -14,8 +14,8 @@ However, you can omit the index name and pass it along with each API method
|
|
14
14
|
called.
|
15
15
|
|
16
16
|
```ruby
|
17
|
-
require '
|
18
|
-
client =
|
17
|
+
require 'elastomer_client/client'
|
18
|
+
client = ElastomerClient::Client.new :port => 9200
|
19
19
|
|
20
20
|
# you can provide an index name
|
21
21
|
index = client.index "blog"
|
data/docs/multi_search.md
CHANGED
data/docs/notifications.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# Notifications Support
|
2
2
|
|
3
|
-
Requiring `
|
4
|
-
|
3
|
+
Requiring `elastomer_client/notifications` enables support for broadcasting
|
4
|
+
ElastomerClient events through ActiveSupport::Notifications.
|
5
5
|
|
6
6
|
The event namespace is `request.client.elastomer`.
|
7
7
|
|
@@ -14,7 +14,7 @@ The event namespace is `request.client.elastomer`.
|
|
14
14
|
:action => "docs.search",
|
15
15
|
:context => nil,
|
16
16
|
:body => "{\"query\":{\"match_all\":{}}}",
|
17
|
-
:url => #<URI::HTTP:0x007fb6f3e98b60 URL:http://localhost:
|
17
|
+
:url => #<URI::HTTP:0x007fb6f3e98b60 URL:http://localhost:9200/index-test/_search?size=0>,
|
18
18
|
:method => :get,
|
19
19
|
:status => 200
|
20
20
|
}
|
data/docs/scan_scroll.md
CHANGED
data/docs/snapshots.md
CHANGED
data/docs/templates.md
CHANGED
data/elastomer-client.gemspec
CHANGED
@@ -1,15 +1,17 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
2
4
|
lib = File.expand_path("../lib", __FILE__)
|
3
5
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require "
|
6
|
+
require "elastomer_client/version"
|
5
7
|
|
6
8
|
Gem::Specification.new do |spec|
|
7
9
|
spec.name = "elastomer-client"
|
8
|
-
spec.version =
|
10
|
+
spec.version = ElastomerClient::VERSION
|
9
11
|
spec.authors = ["Tim Pease", "Grant Rodgers"]
|
10
12
|
spec.email = ["tim.pease@github.com", "grant.rodgers@github.com"]
|
11
13
|
spec.summary = %q{A library for interacting with Elasticsearch}
|
12
|
-
spec.description = %q{
|
14
|
+
spec.description = %q{ElastomerClient is a low level API client for the
|
13
15
|
Elasticsearch HTTP interface.}
|
14
16
|
spec.homepage = "https://github.com/github/elastomer-client"
|
15
17
|
spec.license = "MIT"
|
@@ -20,19 +22,8 @@ Gem::Specification.new do |spec|
|
|
20
22
|
spec.require_paths = ["lib"]
|
21
23
|
|
22
24
|
spec.add_dependency "addressable", "~> 2.5"
|
23
|
-
spec.add_dependency "faraday", "
|
24
|
-
spec.add_dependency "faraday_middleware", "
|
25
|
+
spec.add_dependency "faraday", ">= 0.17"
|
26
|
+
spec.add_dependency "faraday_middleware", ">= 0.14"
|
25
27
|
spec.add_dependency "multi_json", "~> 1.12"
|
26
28
|
spec.add_dependency "semantic", "~> 1.6"
|
27
|
-
|
28
|
-
spec.add_development_dependency "bundler", "~> 2.0"
|
29
|
-
spec.add_development_dependency "activesupport", ">= 3.0"
|
30
|
-
spec.add_development_dependency "minitest", "~> 5.10"
|
31
|
-
spec.add_development_dependency "minitest-fail-fast", "~> 0.1.0"
|
32
|
-
spec.add_development_dependency "minitest-focus", "~> 1.1", ">= 1.1.2"
|
33
|
-
spec.add_development_dependency "webmock", "~> 3.5"
|
34
|
-
spec.add_development_dependency "awesome_print", "~> 1.8"
|
35
|
-
spec.add_development_dependency "pry-byebug", "~> 3.4"
|
36
|
-
spec.add_development_dependency "spy", "~> 1.0"
|
37
|
-
spec.add_development_dependency "rake"
|
38
29
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ElastomerClient
|
2
4
|
class Client
|
3
5
|
|
4
6
|
# The `bulk` method can be used in two ways. Without a block the method
|
@@ -28,7 +30,7 @@ module Elastomer
|
|
28
30
|
# end
|
29
31
|
#
|
30
32
|
# Returns the response body as a Hash
|
31
|
-
def bulk(
|
33
|
+
def bulk(body = nil, params = nil)
|
32
34
|
if block_given?
|
33
35
|
params, body = (body || {}), nil
|
34
36
|
yield bulk_obj = Bulk.new(self, params)
|
@@ -37,8 +39,10 @@ module Elastomer
|
|
37
39
|
else
|
38
40
|
raise "bulk request body cannot be nil" if body.nil?
|
39
41
|
params ||= {}
|
42
|
+
updated_params = params.merge(body:, action: "bulk", rest_api: "bulk")
|
43
|
+
updated_params.delete(:type) if version_support.es_version_8_plus?
|
40
44
|
|
41
|
-
response = self.post "{/index}{/type}/_bulk",
|
45
|
+
response = self.post "{/index}{/type}/_bulk", updated_params
|
42
46
|
response.body
|
43
47
|
end
|
44
48
|
end
|
@@ -103,7 +107,7 @@ module Elastomer
|
|
103
107
|
# # "failure" => 0
|
104
108
|
# # }
|
105
109
|
#
|
106
|
-
# # sample response item:
|
110
|
+
# # sample response item for ES5:
|
107
111
|
# # {
|
108
112
|
# # "delete": {
|
109
113
|
# # "_index": "foo",
|
@@ -115,6 +119,17 @@ module Elastomer
|
|
115
119
|
# # }
|
116
120
|
# # }
|
117
121
|
#
|
122
|
+
# # sample response item for ES8:
|
123
|
+
# # {
|
124
|
+
# # "delete": {
|
125
|
+
# # "_index": "foo",
|
126
|
+
# # "_id": "42",
|
127
|
+
# # "_version": 3,
|
128
|
+
# # "status": 200,
|
129
|
+
# # "result": "deleted"
|
130
|
+
# # }
|
131
|
+
# # }
|
132
|
+
#
|
118
133
|
# Returns a Hash of stats about items from the responses.
|
119
134
|
def bulk_stream_items(ops, params = {})
|
120
135
|
stats = {
|
@@ -158,17 +173,17 @@ module Elastomer
|
|
158
173
|
# immediately.
|
159
174
|
#
|
160
175
|
class Bulk
|
161
|
-
DEFAULT_REQUEST_SIZE =
|
176
|
+
DEFAULT_REQUEST_SIZE = 2**20 * 10 # 10 MB
|
162
177
|
|
163
178
|
# Create a new bulk client for handling some of the details of
|
164
179
|
# accumulating documents to index and then formatting them properly for
|
165
180
|
# the bulk API command.
|
166
181
|
#
|
167
|
-
# client -
|
182
|
+
# client - ElastomerClient::Client used for HTTP requests to the server
|
168
183
|
# params - Parameters Hash to pass to the Client#bulk method
|
169
184
|
# :request_size - the maximum request size in bytes
|
170
185
|
# :action_count - the maximum number of actions
|
171
|
-
def initialize(
|
186
|
+
def initialize(client, params = {})
|
172
187
|
@client = client
|
173
188
|
@params = params
|
174
189
|
|
@@ -190,7 +205,7 @@ module Elastomer
|
|
190
205
|
# buffered until the request size is met or exceeded. When this happens a
|
191
206
|
# bulk request is issued, queued actions are cleared, and the response
|
192
207
|
# from Elasticsearch is returned.
|
193
|
-
def request_size=(
|
208
|
+
def request_size=(value)
|
194
209
|
if value.nil?
|
195
210
|
@request_size = nil
|
196
211
|
else
|
@@ -232,9 +247,9 @@ module Elastomer
|
|
232
247
|
# index("foo" => "bar", "_id" => 1, "_type" => "foo")
|
233
248
|
#
|
234
249
|
# Returns the response from the bulk call if one was made or nil.
|
235
|
-
def index(
|
250
|
+
def index(document, params = {})
|
236
251
|
params = prepare_params(document, params)
|
237
|
-
add_to_actions({:
|
252
|
+
add_to_actions({index: params}, document)
|
238
253
|
end
|
239
254
|
|
240
255
|
# Add a create action to the list of bulk actions to be performed when
|
@@ -251,9 +266,9 @@ module Elastomer
|
|
251
266
|
# create("foo" => "bar", "_id" => 1)
|
252
267
|
#
|
253
268
|
# Returns the response from the bulk call if one was made or nil.
|
254
|
-
def create(
|
269
|
+
def create(document, params)
|
255
270
|
params = prepare_params(document, params)
|
256
|
-
add_to_actions({:
|
271
|
+
add_to_actions({create: params}, document)
|
257
272
|
end
|
258
273
|
|
259
274
|
# Add an update action to the list of bulk actions to be performed when
|
@@ -265,14 +280,14 @@ module Elastomer
|
|
265
280
|
# params - Parameters for the update action (as a Hash) (optional)
|
266
281
|
#
|
267
282
|
# Examples
|
268
|
-
# update({"foo" => "bar"}, {:_id => 1}
|
269
|
-
# update({"foo" => "bar"}, {:id => 1}
|
270
|
-
# update("foo" => "bar", "_id" => 1)
|
283
|
+
# update({"doc" => {"foo" => "bar"}}, {:_id => 1})
|
284
|
+
# update({"doc" => {"foo" => "bar"}}, {:id => 1})
|
285
|
+
# update({"doc" => {"foo" => "bar"}}, "_id" => 1)
|
271
286
|
#
|
272
287
|
# Returns the response from the bulk call if one was made or nil.
|
273
|
-
def update(
|
288
|
+
def update(document, params)
|
274
289
|
params = prepare_params(document, params)
|
275
|
-
add_to_actions({:
|
290
|
+
add_to_actions({update: params}, document)
|
276
291
|
end
|
277
292
|
|
278
293
|
# Add a delete action to the list of bulk actions to be performed when
|
@@ -284,9 +299,9 @@ module Elastomer
|
|
284
299
|
# delete(:_id => 1, :_type => 'foo')
|
285
300
|
#
|
286
301
|
# Returns the response from the bulk call if one was made or nil.
|
287
|
-
def delete(
|
302
|
+
def delete(params)
|
288
303
|
params = prepare_params(nil, params)
|
289
|
-
add_to_actions({:
|
304
|
+
add_to_actions({delete: params})
|
290
305
|
end
|
291
306
|
|
292
307
|
# Immediately execute a bulk API call with the currently accumulated
|
@@ -307,37 +322,23 @@ module Elastomer
|
|
307
322
|
@actions.clear
|
308
323
|
end
|
309
324
|
|
310
|
-
SPECIAL_KEYS = %w[id type index version version_type routing parent
|
311
|
-
|
325
|
+
SPECIAL_KEYS = %w[id type index version version_type routing parent consistency refresh retry_on_conflict]
|
326
|
+
UNPREFIXED_SPECIAL_KEYS = %w[parent retry_on_conflict routing version version_type]
|
312
327
|
|
313
328
|
# Internal: convert special key parameters to their wire representation
|
314
329
|
# and apply any override document parameters.
|
315
330
|
def prepare_params(document, params)
|
316
331
|
params = convert_special_keys(params)
|
317
|
-
|
318
|
-
params = from_document(document).merge(params)
|
319
|
-
end
|
332
|
+
|
320
333
|
params.delete(:_id) if params[:_id].nil? || params[:_id].to_s.empty?
|
321
|
-
params
|
322
|
-
end
|
334
|
+
params.delete("_id") if params["_id"].nil? || params["_id"].to_s.empty?
|
323
335
|
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
#
|
328
|
-
# document - The document Hash
|
329
|
-
#
|
330
|
-
# Returns extracted key/value pairs as a Hash.
|
331
|
-
def from_document( document )
|
332
|
-
opts = {}
|
333
|
-
|
334
|
-
SPECIAL_KEYS_HASH.values.each do |field|
|
335
|
-
key = field.to_sym
|
336
|
-
opts[key] = document.delete field if document.key? field
|
337
|
-
opts[key] = document.delete key if document.key? key
|
336
|
+
if client.version_support.es_version_8_plus?
|
337
|
+
params.delete(:_type)
|
338
|
+
params.delete("_type")
|
338
339
|
end
|
339
340
|
|
340
|
-
|
341
|
+
params
|
341
342
|
end
|
342
343
|
|
343
344
|
# Internal: Convert incoming Ruby symbol keys to their special underscore
|
@@ -353,9 +354,30 @@ module Elastomer
|
|
353
354
|
def convert_special_keys(params)
|
354
355
|
new_params = params.dup
|
355
356
|
|
356
|
-
|
357
|
-
|
358
|
-
|
357
|
+
SPECIAL_KEYS.each do |key|
|
358
|
+
omit_prefix = (
|
359
|
+
client.version_support.es_version_8_plus? &&
|
360
|
+
UNPREFIXED_SPECIAL_KEYS.include?(key)
|
361
|
+
)
|
362
|
+
|
363
|
+
prefixed_key = "_" + key
|
364
|
+
converted_key = (omit_prefix ? "" : "_") + key
|
365
|
+
|
366
|
+
if new_params.key?(prefixed_key)
|
367
|
+
new_params[converted_key] = new_params.delete(prefixed_key)
|
368
|
+
end
|
369
|
+
|
370
|
+
if new_params.key?(prefixed_key.to_sym)
|
371
|
+
new_params[converted_key.to_sym] = new_params.delete(prefixed_key.to_sym)
|
372
|
+
end
|
373
|
+
|
374
|
+
if new_params.key?(key)
|
375
|
+
new_params[converted_key] = new_params.delete(key)
|
376
|
+
end
|
377
|
+
|
378
|
+
if new_params.key?(key.to_sym)
|
379
|
+
new_params[converted_key.to_sym] = new_params.delete(key.to_sym)
|
380
|
+
end
|
359
381
|
end
|
360
382
|
|
361
383
|
new_params
|
@@ -375,7 +397,7 @@ module Elastomer
|
|
375
397
|
# Returns the response from the bulk call if one was made or nil.
|
376
398
|
# Raises RequestSizeError if the given action is larger than the
|
377
399
|
# configured requst size or the client.max_request_size
|
378
|
-
def add_to_actions(
|
400
|
+
def add_to_actions(action, document = nil)
|
379
401
|
action = MultiJson.dump(action)
|
380
402
|
size = action.bytesize
|
381
403
|
|
@@ -389,6 +411,7 @@ module Elastomer
|
|
389
411
|
response = nil
|
390
412
|
begin
|
391
413
|
response = call if ready_to_send?(size)
|
414
|
+
# rubocop:disable Lint/UselessRescue
|
392
415
|
rescue StandardError => err
|
393
416
|
raise err
|
394
417
|
ensure
|
@@ -407,7 +430,7 @@ module Elastomer
|
|
407
430
|
# dispatch the bulk request.
|
408
431
|
#
|
409
432
|
# Returns `true` of `false`
|
410
|
-
def ready_to_send?(
|
433
|
+
def ready_to_send?(size)
|
411
434
|
total_request_size = @current_request_size + size
|
412
435
|
total_action_count = @current_action_count + 1
|
413
436
|
|
@@ -417,7 +440,7 @@ module Elastomer
|
|
417
440
|
|
418
441
|
# Internal: Raises a RequestSizeError if the given size is larger than
|
419
442
|
# the configured client.max_request_size
|
420
|
-
def check_action_size!(
|
443
|
+
def check_action_size!(size)
|
421
444
|
return unless size > client.max_request_size
|
422
445
|
raise RequestSizeError, "Bulk action of size `#{size}` exceeds the maximum requst size: #{client.max_request_size}"
|
423
446
|
end
|