elasticsearch 7.13.3 → 8.14.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +8 -6
- data/README.md +71 -33
- data/Rakefile +15 -20
- data/bin/elastic_ruby_console +2 -18
- data/elasticsearch.gemspec +24 -35
- data/lib/elasticsearch/helpers/bulk_helper.rb +129 -0
- data/lib/elasticsearch/helpers/esql_helper.rb +72 -0
- data/lib/elasticsearch/helpers/scroll_helper.rb +95 -0
- data/lib/elasticsearch/version.rb +1 -1
- data/lib/elasticsearch.rb +167 -7
- data/spec/integration/characters_escaping_spec.rb +94 -0
- data/spec/integration/client_integration_spec.rb +64 -0
- data/spec/integration/helpers/bulk_helper_spec.rb +206 -0
- data/spec/integration/helpers/esql_helper_spec.rb +118 -0
- data/spec/integration/helpers/helpers_spec_helper.rb +29 -0
- data/spec/integration/helpers/scroll_helper_spec.rb +81 -0
- data/spec/integration/opentelemetry_spec.rb +55 -0
- data/spec/spec_helper.rb +43 -0
- data/spec/unit/api_key_spec.rb +138 -0
- data/spec/unit/cloud_credentials_spec.rb +167 -0
- data/spec/unit/custom_transport_implementation_spec.rb +43 -0
- data/spec/unit/elasticsearch_product_validation_spec.rb +148 -0
- data/spec/unit/headers_spec.rb +55 -0
- data/spec/unit/opaque_id_spec.rb +48 -0
- data/spec/unit/user_agent_spec.rb +69 -0
- data/{test/unit/wrapper_gem_test.rb → spec/unit/wrapper_gem_spec.rb} +11 -21
- metadata +64 -111
- data/test/integration/client_integration_test.rb +0 -80
- data/test/test_helper.rb +0 -115
- /data/{LICENSE → LICENSE.txt} +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 829c510c4fb0664f58d4be87a7d0d8eed2906b68e659f63b33a7e082cba6c1ee
|
4
|
+
data.tar.gz: bafb973a40c4eda2a5de4fb0abe5a7f4308efbe8105f531545953a405d2cb83f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4c98e9e0c6cc219b281da17019021ce7bd7e28120c9390b05dd71550bf2527c99570002f779f504f10820effe1fc2cd0b2824213bb445184fb7b77eabc69532b
|
7
|
+
data.tar.gz: 6178b702bd1077554debcf12783ffd1cda149c507c87284cbe60028a0f06fe0db8f2c140f3c29fd19fca23ec456012087ff25f6dcec34553e63d550146248ede
|
data/Gemfile
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Licensed to Elasticsearch B.V. under one or more contributor
|
2
4
|
# license agreements. See the NOTICE file distributed with
|
3
5
|
# this work for additional information regarding copyright
|
@@ -20,14 +22,14 @@ source 'https://rubygems.org'
|
|
20
22
|
# Specify your gem's dependencies in elasticsearch.gemspec
|
21
23
|
gemspec
|
22
24
|
|
23
|
-
if File.exist? File.expand_path(
|
24
|
-
gem 'elasticsearch-api', :
|
25
|
+
if File.exist? File.expand_path('../elasticsearch-api/elasticsearch-api.gemspec', __dir__)
|
26
|
+
gem 'elasticsearch-api', path: File.expand_path('../elasticsearch-api', __dir__), require: false
|
25
27
|
end
|
26
28
|
|
27
|
-
if
|
28
|
-
gem '
|
29
|
+
if ENV['TRANSPORT_VERSION'] == 'main'
|
30
|
+
gem 'elastic-transport', git: 'https://github.com/elastic/elastic-transport-ruby.git', branch: 'main'
|
29
31
|
end
|
30
32
|
|
31
|
-
if
|
32
|
-
gem '
|
33
|
+
if RUBY_VERSION >= '3.0'
|
34
|
+
gem 'opentelemetry-sdk', require: false
|
33
35
|
end
|
data/README.md
CHANGED
@@ -2,6 +2,29 @@
|
|
2
2
|
|
3
3
|
The `elasticsearch` library provides a Ruby client and API for [Elasticsearch](http://elasticsearch.com).
|
4
4
|
|
5
|
+
## Usage
|
6
|
+
|
7
|
+
This gem is a wrapper for two separate libraries:
|
8
|
+
|
9
|
+
* [`elastic-transport`](https://github.com/elastic/elastic-transport-ruby/), which provides a low-level Ruby client for connecting to [Elastic](http://elasticsearch.com) services.
|
10
|
+
* [`elasticsearch-api`](https://github.com/elasticsearch/elasticsearch-ruby/tree/main/elasticsearch-api), which provides a Ruby API for the Elasticsearch RESTful API.
|
11
|
+
|
12
|
+
Install the `elasticsearch` package and use the API directly:
|
13
|
+
|
14
|
+
```ruby
|
15
|
+
require 'elasticsearch'
|
16
|
+
|
17
|
+
client = Elasticsearch::Client.new(log: true)
|
18
|
+
|
19
|
+
client.cluster.health
|
20
|
+
|
21
|
+
client.transport.reload_connections!
|
22
|
+
|
23
|
+
client.search(q: 'test')
|
24
|
+
|
25
|
+
# etc.
|
26
|
+
```
|
27
|
+
|
5
28
|
Features overview:
|
6
29
|
|
7
30
|
* Pluggable logging and tracing
|
@@ -14,25 +37,13 @@ Features overview:
|
|
14
37
|
* Extensive documentation and examples
|
15
38
|
* Emphasis on modularity and extendability of both the client and API libraries
|
16
39
|
|
17
|
-
(For integration with Ruby models and Rails applications,
|
18
|
-
see the <https://github.com/elasticsearch/elasticsearch-rails> project.)
|
40
|
+
(For integration with Ruby models and Rails applications, see the <https://github.com/elasticsearch/elasticsearch-rails> project.)
|
19
41
|
|
20
42
|
## Compatibility
|
21
43
|
|
22
|
-
|
44
|
+
We follow Ruby’s own maintenance policy and officially support all currently maintained versions per [Ruby Maintenance Branches](https://www.ruby-lang.org/en/downloads/branches/).
|
23
45
|
|
24
|
-
|
25
|
-
just use a release matching major version of Elasticsearch.
|
26
|
-
|
27
|
-
| Ruby | | Elasticsearch |
|
28
|
-
|:-------------:|:-:| :-----------: |
|
29
|
-
| 0.90 | → | 0.90 |
|
30
|
-
| 1.x | → | 1.x |
|
31
|
-
| 2.x | → | 2.x |
|
32
|
-
| 5.x | → | 5.x |
|
33
|
-
| 6.x | → | 6.x |
|
34
|
-
| 7.x | → | 7.x |
|
35
|
-
| master | → | master |
|
46
|
+
Language clients are forward compatible; meaning that clients support communicating with greater minor versions of Elasticsearch. Elastic language clients are also backwards compatible with lesser supported minor Elasticsearch versions.
|
36
47
|
|
37
48
|
## Installation
|
38
49
|
|
@@ -51,41 +62,68 @@ or install it from a source code checkout:
|
|
51
62
|
bundle install
|
52
63
|
rake install
|
53
64
|
|
54
|
-
##
|
65
|
+
## Configuration
|
55
66
|
|
56
|
-
|
67
|
+
* [Identifying running tasks with X-Opaque-Id](#identifying-running-tasks-with-x-opaque-id)
|
68
|
+
* [Api Key Authentication](#api-key-authentication)
|
57
69
|
|
58
|
-
|
59
|
-
which provides a low-level Ruby client for connecting to an [Elasticsearch](http://elasticsearch.com) cluster
|
60
|
-
* [`elasticsearch-api`](https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch-api),
|
61
|
-
which provides a Ruby API for the Elasticsearch RESTful API
|
70
|
+
### Identifying running tasks with X-Opaque-Id
|
62
71
|
|
63
|
-
|
72
|
+
The X-Opaque-Id header allows to track certain calls, or associate certain tasks with the client that started them ([more on the Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html#_identifying_running_tasks)). To use this feature, you need to set an id for `opaque_id` on the client on each request. Example:
|
64
73
|
|
65
74
|
```ruby
|
66
|
-
|
75
|
+
client = Elasticsearch::Client.new
|
76
|
+
client.search(index: 'myindex', q: 'title:test', opaque_id: '123456')
|
77
|
+
```
|
78
|
+
The search request will include the following HTTP Header:
|
79
|
+
```
|
80
|
+
X-Opaque-Id: 123456
|
81
|
+
```
|
67
82
|
|
68
|
-
client
|
83
|
+
You can also set a prefix for X-Opaque-Id when initializing the client. This will be prepended to the id you set before each request if you're using X-Opaque-Id. Example:
|
84
|
+
```ruby
|
85
|
+
client = Elastic::Transport::Client.new(opaque_id_prefix: 'eu-west1_')
|
86
|
+
client.search(index: 'myindex', q: 'title:test', opaque_id: '123456')
|
87
|
+
```
|
88
|
+
The request will include the following HTTP Header:
|
89
|
+
```
|
90
|
+
X-Opaque-Id: eu-west1_123456
|
91
|
+
```
|
69
92
|
|
70
|
-
|
93
|
+
### Api Key Authentication
|
71
94
|
|
72
|
-
|
95
|
+
You can use [**API Key authentication**](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html):
|
73
96
|
|
74
|
-
|
97
|
+
``` ruby
|
98
|
+
Elasticsearch::Client.new(
|
99
|
+
host: host,
|
100
|
+
transport_options: transport_options,
|
101
|
+
api_key: credentials
|
102
|
+
)
|
103
|
+
```
|
75
104
|
|
76
|
-
|
105
|
+
Where credentials is either the base64 encoding of `id` and `api_key` joined by a colon or a hash with the `id` and `api_key`:
|
106
|
+
|
107
|
+
``` ruby
|
108
|
+
Elasticsearch::Client.new(
|
109
|
+
host: host,
|
110
|
+
transport_options: transport_options,
|
111
|
+
api_key: {id: 'my_id', api_key: 'my_api_key'}
|
112
|
+
)
|
77
113
|
```
|
78
114
|
|
115
|
+
## API and Transport
|
116
|
+
|
79
117
|
Please refer to the specific library documentation for details:
|
80
118
|
|
81
119
|
* **Transport**:
|
82
|
-
[[README]](https://github.com/
|
83
|
-
[[Documentation]](
|
120
|
+
[[README]](https://github.com/elastic/elastic-transport-ruby#elastic-transport)
|
121
|
+
[[Documentation]](https://rubydoc.info/github/elastic/elastic-transport-ruby/)
|
84
122
|
|
85
123
|
* **API**:
|
86
|
-
[[README]](https://github.com/
|
87
|
-
[[Documentation]](
|
124
|
+
[[README]](https://github.com/elastic/elasticsearch-ruby/tree/main/elasticsearch-api#elasticsearchapi)
|
125
|
+
[[Documentation]](https://rubydoc.info/gems/elasticsearch-api)
|
88
126
|
|
89
127
|
## License
|
90
128
|
|
91
|
-
This software is licensed under the [Apache 2 license](./LICENSE).
|
129
|
+
This software is licensed under the [Apache 2 license](./LICENSE).
|
data/Rakefile
CHANGED
@@ -15,40 +15,35 @@
|
|
15
15
|
# specific language governing permissions and limitations
|
16
16
|
# under the License.
|
17
17
|
|
18
|
-
require
|
18
|
+
require 'bundler/gem_tasks'
|
19
19
|
|
20
|
-
|
21
|
-
|
20
|
+
task(:default) { system 'rake --tasks' }
|
21
|
+
|
22
|
+
desc 'Run unit tests'
|
23
|
+
task test: 'test:spec'
|
22
24
|
|
23
25
|
# ----- Test tasks ------------------------------------------------------------
|
26
|
+
require 'rspec/core/rake_task'
|
24
27
|
|
25
|
-
require 'rake/testtask'
|
26
28
|
namespace :test do
|
27
|
-
desc
|
29
|
+
desc 'Wait for Elasticsearch to be in a green state'
|
28
30
|
task :wait_for_green do
|
29
31
|
sh '../scripts/wait-cluster.sh'
|
30
32
|
end
|
31
33
|
|
32
|
-
|
33
|
-
|
34
|
-
test.test_files = FileList["test/unit/**/*_test.rb"]
|
35
|
-
test.verbose = false
|
36
|
-
test.warning = false
|
34
|
+
RSpec::Core::RakeTask.new(:integration) do |t|
|
35
|
+
t.pattern = 'spec/integration/**{,/*/**}/*_spec.rb'
|
37
36
|
end
|
38
37
|
|
39
|
-
|
40
|
-
|
41
|
-
test.libs << 'lib' << 'test'
|
42
|
-
test.test_files = FileList["test/integration/**/*_test.rb"]
|
43
|
-
test.verbose = false
|
44
|
-
test.warning = false
|
38
|
+
RSpec::Core::RakeTask.new(:unit) do |t|
|
39
|
+
t.pattern = 'spec/unit/**{,/*/**}/*_spec.rb'
|
45
40
|
end
|
46
41
|
|
47
|
-
|
48
|
-
|
49
|
-
|
42
|
+
desc 'Run unit and integration tests'
|
43
|
+
task :all do
|
44
|
+
Rake::Task['test:unit'].invoke
|
45
|
+
Rake::Task['test:integration'].invoke
|
50
46
|
end
|
51
|
-
|
52
47
|
end
|
53
48
|
|
54
49
|
# ----- Documentation tasks ---------------------------------------------------
|
data/bin/elastic_ruby_console
CHANGED
@@ -1,30 +1,14 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch/lib', __dir__))
|
4
|
-
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch-transport/lib', __dir__))
|
5
|
-
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch-dsl/lib', __dir__))
|
6
4
|
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch-api/lib', __dir__))
|
7
|
-
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch
|
8
|
-
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch-extensions/lib', __dir__))
|
5
|
+
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch/lib/elasticsearch/helpers', __dir__))
|
9
6
|
|
10
7
|
require 'elasticsearch'
|
11
|
-
require 'elasticsearch-transport'
|
12
8
|
require 'elasticsearch-api'
|
13
|
-
|
14
|
-
gems_not_loaded = ['elasticsearch-dsl', 'elasticsearch/xpack', 'elasticsearch-extensions'].reject do |gem|
|
15
|
-
begin
|
16
|
-
(require gem) || true
|
17
|
-
rescue LoadError
|
18
|
-
false
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
unless gems_not_loaded.empty?
|
23
|
-
warn "The following gems were not loaded: [#{gems_not_loaded.join(', ')}]. Please install and require them explicitly."
|
24
|
-
end
|
9
|
+
require 'elasticsearch/helpers/bulk_helper'
|
25
10
|
|
26
11
|
include Elasticsearch
|
27
|
-
include Elasticsearch::DSL if defined?(Elasticsearch::DSL)
|
28
12
|
|
29
13
|
begin
|
30
14
|
require 'pry'
|
data/elasticsearch.gemspec
CHANGED
@@ -22,54 +22,43 @@ require 'elasticsearch/version'
|
|
22
22
|
Gem::Specification.new do |s|
|
23
23
|
s.name = 'elasticsearch'
|
24
24
|
s.version = Elasticsearch::VERSION
|
25
|
-
s.authors = ['
|
26
|
-
s.email = ['
|
25
|
+
s.authors = ['Elastic Client Library Maintainers']
|
26
|
+
s.email = ['client-libs@elastic.co']
|
27
27
|
s.summary = 'Ruby integrations for Elasticsearch'
|
28
|
-
s.homepage = 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/
|
28
|
+
s.homepage = 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html'
|
29
29
|
s.license = 'Apache-2.0'
|
30
30
|
s.metadata = {
|
31
|
-
'homepage_uri' => 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/
|
32
|
-
'changelog_uri' => 'https://github.com/elastic/elasticsearch-ruby/blob/
|
33
|
-
'source_code_uri' => 'https://github.com/elastic/elasticsearch-ruby/tree/
|
31
|
+
'homepage_uri' => 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html',
|
32
|
+
'changelog_uri' => 'https://github.com/elastic/elasticsearch-ruby/blob/main/CHANGELOG.md',
|
33
|
+
'source_code_uri' => 'https://github.com/elastic/elasticsearch-ruby/tree/main',
|
34
34
|
'bug_tracker_uri' => 'https://github.com/elastic/elasticsearch-ruby/issues'
|
35
35
|
}
|
36
|
-
s.files
|
37
|
-
s.executables
|
38
|
-
s.executables
|
39
|
-
s.test_files
|
36
|
+
s.files = `git ls-files`.split($/)
|
37
|
+
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
38
|
+
s.executables << 'elastic_ruby_console'
|
39
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
40
40
|
s.require_paths = ['lib']
|
41
41
|
s.bindir = 'bin'
|
42
42
|
|
43
|
-
s.extra_rdoc_files = [
|
44
|
-
s.rdoc_options = [
|
43
|
+
s.extra_rdoc_files = ['README.md', 'LICENSE.txt']
|
44
|
+
s.rdoc_options = ['--charset=UTF-8']
|
45
45
|
|
46
|
-
s.required_ruby_version = '>= 2.
|
47
|
-
|
48
|
-
s.add_dependency 'elasticsearch-transport', '7.13.3'
|
49
|
-
s.add_dependency 'elasticsearch-api', '7.13.3'
|
46
|
+
s.required_ruby_version = '>= 2.5'
|
50
47
|
|
48
|
+
s.add_dependency 'elastic-transport', '~> 8.3'
|
49
|
+
s.add_dependency 'elasticsearch-api', '8.14.0'
|
50
|
+
|
51
|
+
s.add_development_dependency 'base64'
|
51
52
|
s.add_development_dependency 'bundler'
|
52
|
-
|
53
|
-
s.add_development_dependency 'rake', '~> 13'
|
54
|
-
|
55
|
-
s.add_development_dependency 'elasticsearch-extensions'
|
56
|
-
|
57
|
-
s.add_development_dependency 'ansi'
|
58
|
-
s.add_development_dependency 'shoulda-context'
|
59
|
-
s.add_development_dependency 'mocha'
|
60
|
-
s.add_development_dependency 'yard'
|
53
|
+
s.add_development_dependency 'debug' unless defined?(JRUBY_VERSION)
|
61
54
|
s.add_development_dependency 'pry'
|
62
|
-
|
63
|
-
|
64
|
-
s.add_development_dependency 'minitest'
|
65
|
-
s.add_development_dependency 'minitest-reporters'
|
66
|
-
s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
|
55
|
+
s.add_development_dependency 'rake'
|
67
56
|
s.add_development_dependency 'require-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
|
68
|
-
s.add_development_dependency '
|
69
|
-
s.add_development_dependency '
|
70
|
-
s.add_development_dependency '
|
71
|
-
|
72
|
-
s.add_development_dependency '
|
57
|
+
s.add_development_dependency 'rspec'
|
58
|
+
s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
|
59
|
+
s.add_development_dependency 'simplecov'
|
60
|
+
s.add_development_dependency 'webmock'
|
61
|
+
s.add_development_dependency 'yard'
|
73
62
|
|
74
63
|
s.description = <<-DESC.gsub(/^ /, '')
|
75
64
|
Ruby integrations for Elasticsearch (client, API, etc.)
|
@@ -0,0 +1,129 @@
|
|
1
|
+
# Licensed to Elasticsearch B.V. under one or more contributor
|
2
|
+
# license agreements. See the NOTICE file distributed with
|
3
|
+
# this work for additional information regarding copyright
|
4
|
+
# ownership. Elasticsearch B.V. licenses this file to you under
|
5
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
6
|
+
# not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Elasticsearch
|
19
|
+
module Helpers
|
20
|
+
# Elasticsearch Client Helper for the Bulk API
|
21
|
+
#
|
22
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-bulk.html
|
23
|
+
#
|
24
|
+
class BulkHelper
|
25
|
+
attr_accessor :index
|
26
|
+
|
27
|
+
# Create a BulkHelper
|
28
|
+
#
|
29
|
+
# @param [Elasticsearch::Client] client Instance of Elasticsearch client to use.
|
30
|
+
# @param [String] index Index on which to perform the Bulk actions.
|
31
|
+
# @param [Hash] params Parameters to re-use in every bulk call
|
32
|
+
#
|
33
|
+
def initialize(client, index, params = {})
|
34
|
+
@client = client
|
35
|
+
@index = index
|
36
|
+
@params = params
|
37
|
+
end
|
38
|
+
|
39
|
+
# Index documents using the Bulk API.
|
40
|
+
#
|
41
|
+
# @param [Array<Hash>] docs The documents to be indexed.
|
42
|
+
# @param [Hash] params Parameters to use in the bulk ingestion. See the official Elastic documentation for Bulk API for parameters to send to the Bulk API.
|
43
|
+
# @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of ingestion.
|
44
|
+
# @param block [Block] Optional block to run after ingesting a batch of documents.
|
45
|
+
# @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
|
46
|
+
# @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
|
47
|
+
#
|
48
|
+
def ingest(docs, params = {}, body = {}, &block)
|
49
|
+
ingest_docs = docs.map { |doc| { index: { _index: @index, data: doc} } }
|
50
|
+
if (slice = params.delete(:slice))
|
51
|
+
ingest_docs.each_slice(slice) do |items|
|
52
|
+
ingest(items.map { |item| item[:index][:data] }, params, &block)
|
53
|
+
end
|
54
|
+
else
|
55
|
+
bulk_request(ingest_docs, params, &block)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Delete documents using the Bulk API
|
60
|
+
#
|
61
|
+
# @param [Array] ids Array of id's for documents to delete.
|
62
|
+
# @param [Hash] params Parameters to send to bulk delete.
|
63
|
+
#
|
64
|
+
def delete(ids, params = {}, body = {})
|
65
|
+
delete_docs = ids.map { |id| { delete: { _index: @index, _id: id} } }
|
66
|
+
@client.bulk({ body: delete_docs }.merge(params.merge(@params)))
|
67
|
+
end
|
68
|
+
|
69
|
+
# Update documents using the Bulk API
|
70
|
+
#
|
71
|
+
# @param [Array<Hash>] docs (Required) The documents to be updated.
|
72
|
+
# @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of updates.
|
73
|
+
# @param block [Block] Optional block to run after ingesting a batch of documents.
|
74
|
+
#
|
75
|
+
# @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
|
76
|
+
# @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
|
77
|
+
#
|
78
|
+
def update(docs, params = {}, body = {}, &block)
|
79
|
+
ingest_docs = docs.map do |doc|
|
80
|
+
{ update: { _index: @index, _id: doc.delete('id'), data: { doc: doc } } }
|
81
|
+
end
|
82
|
+
if (slice = params.delete(:slice))
|
83
|
+
ingest_docs.each_slice(slice) { |items| update(items, params, &block) }
|
84
|
+
else
|
85
|
+
bulk_request(ingest_docs, params, &block)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Ingest data directly from a JSON file
|
90
|
+
#
|
91
|
+
# @param [String] file (Required) The file path.
|
92
|
+
# @param [Hash] params Parameters to use in the bulk ingestion.
|
93
|
+
# @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of updates.
|
94
|
+
# @option params [Array|String] keys If the data needs to be digged from the JSON file, the
|
95
|
+
# keys can be passed in with this parameter to find it.
|
96
|
+
#
|
97
|
+
# E.g.: If the data in the parsed JSON Hash is found in
|
98
|
+
# +json_parsed['data']['items']+, keys would be passed
|
99
|
+
# like this (as an Array):
|
100
|
+
#
|
101
|
+
# +bulk_helper.ingest_json(file, { keys: ['data', 'items'] })+
|
102
|
+
#
|
103
|
+
# or as a String:
|
104
|
+
#
|
105
|
+
# +bulk_helper.ingest_json(file, { keys: 'data, items' })+
|
106
|
+
#
|
107
|
+
# @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
|
108
|
+
# @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
|
109
|
+
#
|
110
|
+
def ingest_json(file, params = {}, &block)
|
111
|
+
data = JSON.parse(File.read(file))
|
112
|
+
if (keys = params.delete(:keys))
|
113
|
+
keys = keys.split(',') if keys.is_a?(String)
|
114
|
+
data = data.dig(*keys)
|
115
|
+
end
|
116
|
+
|
117
|
+
ingest(data, params, &block)
|
118
|
+
end
|
119
|
+
|
120
|
+
private
|
121
|
+
|
122
|
+
def bulk_request(ingest_docs, params, &block)
|
123
|
+
response = @client.bulk({ body: ingest_docs }.merge(params.merge(@params)))
|
124
|
+
yield response, ingest_docs if block_given?
|
125
|
+
response
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# Licensed to Elasticsearch B.V. under one or more contributor
|
2
|
+
# license agreements. See the NOTICE file distributed with
|
3
|
+
# this work for additional information regarding copyright
|
4
|
+
# ownership. Elasticsearch B.V. licenses this file to you under
|
5
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
6
|
+
# not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Elasticsearch
|
19
|
+
module Helpers
|
20
|
+
# Elasticsearch Client Helper for the ES|QL API
|
21
|
+
#
|
22
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-query-api.html
|
23
|
+
#
|
24
|
+
module ESQLHelper
|
25
|
+
# Query helper for ES|QL
|
26
|
+
#
|
27
|
+
# By default, the `esql.query` API returns a Hash response with the following keys:
|
28
|
+
#
|
29
|
+
# * `columns` with the value being an Array of `{ name: type }` Hashes for each column.
|
30
|
+
#
|
31
|
+
# * `values` with the value being an Array of Arrays with the values for each row.
|
32
|
+
#
|
33
|
+
# This helper function returns an Array of hashes with the columns as keys and the respective
|
34
|
+
# values: `{ column['name'] => value }`.
|
35
|
+
#
|
36
|
+
# @param client [Elasticsearch::Client] an instance of the Client to use for the query.
|
37
|
+
# @param query [Hash, String] The query to be passed to the ES|QL query API.
|
38
|
+
# @param params [Hash] options to pass to the ES|QL query API.
|
39
|
+
# @param parser [Hash] Hash of column name keys and Proc values to transform the value of
|
40
|
+
# a given column.
|
41
|
+
# @example Using the ES|QL helper
|
42
|
+
# require 'elasticsearch/helpers/esql_helper'
|
43
|
+
# query = <<~ESQL
|
44
|
+
# FROM sample_data
|
45
|
+
# | EVAL duration_ms = ROUND(event.duration / 1000000.0, 1)
|
46
|
+
# ESQL
|
47
|
+
# response = Elasticsearch::Helpers::ESQLHelper.query(client, query)
|
48
|
+
#
|
49
|
+
# @example Using the ES|QL helper with a parser
|
50
|
+
# response = Elasticsearch::Helpers::ESQLHelper.query(
|
51
|
+
# client,
|
52
|
+
# query,
|
53
|
+
# parser: { '@timestamp' => Proc.new { |t| DateTime.parse(t) } }
|
54
|
+
# )
|
55
|
+
#
|
56
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/Helpers.html#_esql_helper
|
57
|
+
#
|
58
|
+
def self.query(client, query, params = {}, parser: {})
|
59
|
+
response = client.esql.query({ body: { query: query }, format: 'json' }.merge(params))
|
60
|
+
|
61
|
+
columns = response['columns']
|
62
|
+
response['values'].map do |value|
|
63
|
+
(value.length - 1).downto(0).map do |index|
|
64
|
+
key = columns[index]['name']
|
65
|
+
value[index] = parser[key].call(value[index]) if value[index] && parser[key]
|
66
|
+
{ key => value[index] }
|
67
|
+
end.reduce({}, :merge)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# Licensed to Elasticsearch B.V. under one or more contributor
|
2
|
+
# license agreements. See the NOTICE file distributed with
|
3
|
+
# this work for additional information regarding copyright
|
4
|
+
# ownership. Elasticsearch B.V. licenses this file to you under
|
5
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
6
|
+
# not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Elasticsearch
|
19
|
+
module Helpers
|
20
|
+
# Elasticsearch Client Helper for the Scroll API
|
21
|
+
#
|
22
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/scroll-api.html
|
23
|
+
#
|
24
|
+
class ScrollHelper
|
25
|
+
include Enumerable
|
26
|
+
|
27
|
+
# Create a ScrollHelper
|
28
|
+
#
|
29
|
+
# @param [Elasticsearch::Client] client (Required) Instance of Elasticsearch client to use.
|
30
|
+
# @param [String] index (Required) Index on which to perform the Bulk actions.
|
31
|
+
# @param [Hash] body Body parameters to re-use in every scroll request
|
32
|
+
# @param [Time] scroll Specify how long a consistent view of the index should be maintained for scrolled search
|
33
|
+
#
|
34
|
+
def initialize(client, index, body, scroll = '1m')
|
35
|
+
@index = index
|
36
|
+
@client = client
|
37
|
+
@scroll = scroll
|
38
|
+
@body = body
|
39
|
+
end
|
40
|
+
|
41
|
+
# Implementation of +each+ for Enumerable module inclusion
|
42
|
+
#
|
43
|
+
# @yieldparam document [Hash] yields a document found in the search hits.
|
44
|
+
#
|
45
|
+
def each(&block)
|
46
|
+
@docs = []
|
47
|
+
@scroll_id = nil
|
48
|
+
refresh_docs
|
49
|
+
for doc in @docs do
|
50
|
+
refresh_docs
|
51
|
+
yield doc
|
52
|
+
end
|
53
|
+
clear
|
54
|
+
end
|
55
|
+
|
56
|
+
# Results from a scroll.
|
57
|
+
# Can be called repeatedly (e.g. in a loop) to get the scroll pages.
|
58
|
+
#
|
59
|
+
def results
|
60
|
+
if @scroll_id
|
61
|
+
scroll_request
|
62
|
+
else
|
63
|
+
initial_search
|
64
|
+
end
|
65
|
+
rescue StandardError => e
|
66
|
+
raise e
|
67
|
+
end
|
68
|
+
|
69
|
+
# Clear Scroll and resets inner documents collection
|
70
|
+
#
|
71
|
+
def clear
|
72
|
+
@client.clear_scroll(body: { scroll_id: @scroll_id }) if @scroll_id
|
73
|
+
@docs = []
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def refresh_docs
|
79
|
+
@docs ||= []
|
80
|
+
@docs << results
|
81
|
+
@docs.flatten!
|
82
|
+
end
|
83
|
+
|
84
|
+
def initial_search
|
85
|
+
response = @client.search(index: @index, scroll: @scroll, body: @body)
|
86
|
+
@scroll_id = response['_scroll_id']
|
87
|
+
response['hits']['hits']
|
88
|
+
end
|
89
|
+
|
90
|
+
def scroll_request
|
91
|
+
@client.scroll(body: {scroll: @scroll, scroll_id: @scroll_id})['hits']['hits']
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|